From a4d9e6d51fad61660aef986ca948defa4a2f14d3 Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Tue, 17 Nov 2026 00:51:11 +0400 Subject: [PATCH 1/7] chore(deps): pin agentOS 0.4.9-rc.1 --- package.json | 2 +- packages/runtime-node/package.json | 2 +- pnpm-lock.yaml | 64 +++++++++++++++--------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index 2be9ded..408ff25 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "license": "Apache-2.0", "dependencies": { "@aws-sdk/client-s3": "3.1153.0", - "@rivet-dev/agentos-core": "0.4.8", + "@rivet-dev/agentos-core": "0.4.9-rc.1", "@rivet-dev/durable-object-migrator": "1.0.3", "@rivet-dev/sandbox-adapter": "0.5.3", "capnweb": "0.12.0", diff --git a/packages/runtime-node/package.json b/packages/runtime-node/package.json index 4ba210d..a1cba7e 100644 --- a/packages/runtime-node/package.json +++ b/packages/runtime-node/package.json @@ -27,7 +27,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "3.1153.0", - "@rivet-dev/agentos-core": "0.4.8", + "@rivet-dev/agentos-core": "0.4.9-rc.1", "@rivet-dev/durable-object-migrator": "1.0.3", "@rivet-dev/sandbox-adapter": "0.5.3", "capnweb": "0.12.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6c85703..a3c82f7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: 3.1153.0 version: 3.1153.0 '@rivet-dev/agentos-core': - specifier: 0.4.8 - version: 0.4.8 + specifier: 0.4.9-rc.1 + version: 0.4.9-rc.1 '@rivet-dev/durable-object-migrator': specifier: 1.0.3 version: 1.0.3 @@ -172,8 +172,8 @@ importers: specifier: 3.1153.0 version: 3.1153.0 '@rivet-dev/agentos-core': - specifier: 0.4.8 - version: 0.4.8 + specifier: 0.4.9-rc.1 + version: 0.4.9-rc.1 '@rivet-dev/durable-object-migrator': specifier: 1.0.3 version: 1.0.3 @@ -1424,40 +1424,40 @@ packages: engines: {node: '>=18'} hasBin: true - '@rivet-dev/agentos-core@0.4.8': - resolution: {integrity: sha512-EekiSBrBGAR6VI41BlzQyaIDRXGxHillrojST5t8XnZ30jfzqpymWkPC997G+MT5NPyPcrv/q3duAjX06imzXQ==} + '@rivet-dev/agentos-core@0.4.9-rc.1': + resolution: {integrity: sha512-pqE3TSCSB5hXImPOHeQ5edXR6IBI2FhCT2lHUeLZMcEBY5rAAcplJFF3JdeGkBvaC0YSTBr3P5mmfyvUpUdmVw==} - '@rivet-dev/agentos-runtime-core@0.4.8': - resolution: {integrity: sha512-uCNXNPJXisctZrRonSvjzPxazwQErPZaOrtogibnONd11D6KxDT2u/p89BqihUiK3KCQs8NLkVr3WaYgwZ+0bA==} + '@rivet-dev/agentos-runtime-core@0.4.9-rc.1': + resolution: {integrity: sha512-yr2SUC6bNdv+R5rkJqq/4FPipP0OiE1kFJV6T+lAFjpoalsUn9X04ZaIvMHgvAgdL6Gug31dK6kLd+D/nhOx6A==} - '@rivet-dev/agentos-runtime-sidecar-darwin-arm64@0.4.8': - resolution: {integrity: sha512-ZaL+MEHqtbP1WRF575m2LQFP6tBcIPb6p4vKnZGYuUad2o9qqE1qeOhG3YhFdymrE8QUMHwFKDvL/+7Ep1k/7Q==} + '@rivet-dev/agentos-runtime-sidecar-darwin-arm64@0.4.9-rc.1': + resolution: {integrity: sha512-ofVhX5pWAP5OEdiv/MKoC/rzbGegIzBj9EIK8hP8KzRdVMq3uWPtTF+4QjPHmLUhOBT1qyOpmirenhkyafFsZg==} engines: {node: '>=20'} cpu: [arm64] os: [darwin] - '@rivet-dev/agentos-runtime-sidecar-darwin-x64@0.4.8': - resolution: {integrity: sha512-chpzdQjs4SdSGrlTj6MSAk14/E91kyb6zcrbw2CJCn2tE4CJi0BUqgFAdWHKfZznwbnWXNB7vU9oBxYLmHSQQw==} + '@rivet-dev/agentos-runtime-sidecar-darwin-x64@0.4.9-rc.1': + resolution: {integrity: sha512-/gTYCcB/AEvaEgdLU+bPdfRX0vqaKtmgTWdwAm77jENcdSuiFNuAuxYmhqXB15Pm9Bylo57tqhX/Dl5OePKVmw==} engines: {node: '>=20'} cpu: [x64] os: [darwin] - '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu@0.4.8': - resolution: {integrity: sha512-ysmQ8kfBNAhRGodTkSJUftb7PowrxBhhDil66mSKVOQwO6jN21/K6M+K8y0DGTS5d1azqf+fZe2UzewMb3dUbQ==} + '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu@0.4.9-rc.1': + resolution: {integrity: sha512-HSicsWKJRrFsCcjEW54ZTT+G07axN7cku5r0jVtGkx6DSSstSqMU8QSWj5mDO6dT6pVsTAHDXORqizbO0RcRWw==} engines: {node: '>=20'} cpu: [arm64] os: [linux] libc: [glibc] - '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu@0.4.8': - resolution: {integrity: sha512-DEJNL3VcxWqRwIDcOSWHQ3APTNFZRvv6wokWP4dV4dwHsNc8dCKMLf8B2qYKw3Qljb7loinppeOu6WRwDaJgqw==} + '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu@0.4.9-rc.1': + resolution: {integrity: sha512-1WS+G0Wj3pA+2qKME2R92Qjz0wFweF8R0Lss0BAtRrWmUmmHvb5mzjmcgDTkyJVwvC/MHKO0sNPmARRN9NapTA==} engines: {node: '>=20'} cpu: [x64] os: [linux] libc: [glibc] - '@rivet-dev/agentos-runtime-sidecar@0.4.8': - resolution: {integrity: sha512-SUeyGdmSKl6NyLhFoLo5KJa48b84Cg40Dcw80CxY+62u4tNIiops3XpZEp2Xzeh2USD7GaYfvVBfzYba9votmw==} + '@rivet-dev/agentos-runtime-sidecar@0.4.9-rc.1': + resolution: {integrity: sha512-Ag0uUmdZIrlajssHJ9F7RcGzkZCGL4c407e5nPv0fNV3FvBeqq43+RXdF+W4t0FhaTtkbLqh+04OIkpMUSw59A==} engines: {node: '>=20'} '@rivet-dev/durable-object-migrator@1.0.3': @@ -4263,40 +4263,40 @@ snapshots: - react-native-b4a - supports-color - '@rivet-dev/agentos-core@0.4.8': + '@rivet-dev/agentos-core@0.4.9-rc.1': dependencies: - '@rivet-dev/agentos-runtime-core': 0.4.8 - '@rivet-dev/agentos-runtime-sidecar': 0.4.8 + '@rivet-dev/agentos-runtime-core': 0.4.9-rc.1 + '@rivet-dev/agentos-runtime-sidecar': 0.4.9-rc.1 '@xterm/headless': 6.0.0 croner: 10.0.1 long-timeout: 0.1.1 zod: 4.5.4 - '@rivet-dev/agentos-runtime-core@0.4.8': + '@rivet-dev/agentos-runtime-core@0.4.9-rc.1': dependencies: '@bare-ts/lib': 0.4.1 - '@rivet-dev/agentos-runtime-sidecar': 0.4.8 + '@rivet-dev/agentos-runtime-sidecar': 0.4.9-rc.1 zod: 4.5.4 zod-to-json-schema: 3.25.2(zod@4.5.4) - '@rivet-dev/agentos-runtime-sidecar-darwin-arm64@0.4.8': + '@rivet-dev/agentos-runtime-sidecar-darwin-arm64@0.4.9-rc.1': optional: true - '@rivet-dev/agentos-runtime-sidecar-darwin-x64@0.4.8': + '@rivet-dev/agentos-runtime-sidecar-darwin-x64@0.4.9-rc.1': optional: true - '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu@0.4.8': + '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu@0.4.9-rc.1': optional: true - '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu@0.4.8': + '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu@0.4.9-rc.1': optional: true - '@rivet-dev/agentos-runtime-sidecar@0.4.8': + '@rivet-dev/agentos-runtime-sidecar@0.4.9-rc.1': optionalDependencies: - '@rivet-dev/agentos-runtime-sidecar-darwin-arm64': 0.4.8 - '@rivet-dev/agentos-runtime-sidecar-darwin-x64': 0.4.8 - '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu': 0.4.8 - '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu': 0.4.8 + '@rivet-dev/agentos-runtime-sidecar-darwin-arm64': 0.4.9-rc.1 + '@rivet-dev/agentos-runtime-sidecar-darwin-x64': 0.4.9-rc.1 + '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu': 0.4.9-rc.1 + '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu': 0.4.9-rc.1 '@rivet-dev/durable-object-migrator@1.0.3': dependencies: From 7e410fbe7775f152962b684035ab5dcdabb859ca Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Tue, 17 Nov 2026 01:35:50 +0400 Subject: [PATCH 2/7] chore(release): update version to 1.0.4-rc.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 408ff25..b20c025 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-orange", - "version": "1.0.3", + "version": "1.0.4-rc.1", "description": "Cloudflare Workers compatibility layer for Rivet", "packageManager": "pnpm@10.34.9", "engines": { From f23e95e7be96f47396880209fcf113b5ac745051 Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sat, 14 Nov 2026 23:30:27 +0400 Subject: [PATCH 3/7] perf(worker-loader): keep the dynamic Worker pool ahead of bursts - Size each network mode's VM pool to recent demand (POOL_SIZE..POOL_MAX), shrink back after the idle timeout - Refill with up to 4 parallel prewarms as soon as no Worker is loading, instead of one at a time 250 ms later - Boot on demand at once when the pool is empty; never queue a burst behind one prewarm - Test pool growth, idle shrink, parallel boots, and per-Worker VM isolation through orange dev --- docs/modules/worker-loader.mdx | 49 +++- .../src/bindings/worker-loader/agentos.ts | 265 +++++++++++++----- .../src/bindings/worker-loader/loader.ts | 18 +- packages/runtime-node/src/runner.ts | 6 +- tests/bindings/worker-loader.test.ts | 59 ++-- tests/fixtures/worker-loader/worker.js | 30 +- 6 files changed, 310 insertions(+), 117 deletions(-) diff --git a/docs/modules/worker-loader.mdx b/docs/modules/worker-loader.mdx index 6f18ce6..82527b3 100644 --- a/docs/modules/worker-loader.mdx +++ b/docs/modules/worker-loader.mdx @@ -8,7 +8,7 @@ description: Load and run dynamic Workers at runtime with the worker_loaders bin @@ -89,8 +89,9 @@ Hello from a dynamic Worker - Dynamic Worker code is untrusted, so each dynamic Worker gets its own VM, as it gets its own isolate on Cloudflare. It never runs on the host. -- Orange keeps a few VMs booted ahead of use, so a new dynamic Worker only - writes its own modules. Each VM runs one dynamic Worker and is never reused. +- Orange keeps VMs booted ahead of use, and more after a burst of loads, so a + new dynamic Worker only writes its own modules. Each VM runs one dynamic + Worker and is never reused. - A dynamic Worker stops when it is idle or nothing references it. Its VM is disposed, never returned to the pool. - Your Worker and the dynamic Worker talk Workers RPC, so stubs and bindings @@ -127,11 +128,15 @@ agentOS's limits apply: | `env` | No set limit. In a test with agentOS 0.4.1, 5 MiB worked. | At 10 MiB, the dynamic Worker exited with code 1. | | Running dynamic Workers, across all Worker Loader bindings | 32. Set `ORANGE_WORKER_LOADER_MAX_WORKERS` to change it. | A new dynamic Worker stops the least recently used idle one. If none is idle, `get()` or `load()` throws a `WorkerLoaderLimitError` that names the limit. At 80% of the limit, `orange dev` logs a warning. | | Idle time before a dynamic Worker stops | 30 seconds. Set `ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS` to change it, from 1 to 2147483647. | The dynamic Worker stops and frees its VM. The next call loads it again. A value out of range stops `orange dev` at startup with a `RangeError` that names the setting. | -| Prewarmed VMs | 2 for each network mode by default, from 0 to 16 with `ORANGE_WORKER_LOADER_POOL_SIZE` | A load that finds the pool empty boots its own VM. | +| Prewarmed VMs when no burst is recent | 2 for each network mode by default, from 0 to 64 with `ORANGE_WORKER_LOADER_POOL_SIZE` | A load that finds the pool empty boots its own VM. | +| Prewarmed VMs after a burst | 16 for each network mode by default, from 0 to 64 with `ORANGE_WORKER_LOADER_POOL_MAX`. It must not be less than `ORANGE_WORKER_LOADER_POOL_SIZE`. `0` boots every VM on demand. | The pool stops growing. A value out of range stops `orange dev` at startup with a `RangeError` that names the setting. | +| VMs that prewarm at once | 4, across both network modes | The other prewarms wait. Loads never wait for this limit. | -Each running dynamic Worker uses about 54 MiB. Prewarmed VMs that no Worker -has claimed do not count toward `ORANGE_WORKER_LOADER_MAX_WORKERS`, and the -limit never stops them. +Each running dynamic Worker uses about 54 MiB. Each prewarmed VM uses 26 to +54 MiB, depending on the agentOS version. After a burst of 16 or more loads, +a network mode can hold 16 prewarmed VMs, 0.4 to 0.9 GiB, until the idle +timeout passes. Prewarmed VMs that no Worker has claimed do not count toward +`ORANGE_WORKER_LOADER_MAX_WORKERS`, and the limit never stops them. #### Isolation @@ -150,11 +155,23 @@ limit never stops them. has its Node.js process started with Orange's runtime imported. It has seen no Worker code or `env`. - A load takes a prewarmed VM, writes its modules, and starts the Worker. If - the pool is empty, it waits for the VM that is booting, or boots its own. - The pool refills one VM at a time, shortly after the load finishes. + the pool is empty, it takes a VM that is booting and that no other load + waits for, or boots its own at once, beside the other loads. +- Each network mode's pool aims for the most loads seen within one prewarm + time, from `ORANGE_WORKER_LOADER_POOL_SIZE` to + `ORANGE_WORKER_LOADER_POOL_MAX`. So the next burst of the same size finds + its VMs ready, and a steady stream of loads keeps the pool ahead. +- The pool refills as soon as no dynamic Worker is loading, because a + prewarm that runs beside a load slows it. Workers that load at the same + time also hold the refill until their first calls finish. One Worker holds + the refill back for at most 1 second. +- When the idle timeout (`ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS`) passes + without a burst as large, the pool shrinks to + `ORANGE_WORKER_LOADER_POOL_SIZE` and disposes the other prewarmed VMs. - A VM that fails to prewarm is reported as a background error, and the next - load boots its own VM. Set `ORANGE_WORKER_LOADER_POOL_SIZE` to `0` to boot - every VM on demand. + load boots its own VM. The pool tries to prewarm again only after its next + load. Set `ORANGE_WORKER_LOADER_POOL_MAX` to `0` to boot every VM on + demand. - If the VM cannot boot, `get()` and `load()` calls fail with the boot error. Orange never falls back to the host. - A dynamic Worker is idle when no call to it is in flight, your Worker holds @@ -250,10 +267,12 @@ These add to the [Compatibility](#compatibility) table, the `tests/bindings/worker-loader.test.ts` drives a real `orange dev` over HTTP: fetch and RPC, env and props capabilities, isolation, `globalOutbound: null`, name caching, syntax errors, process crashes, module types, a `node:http` -server and an Express app in a dynamic Worker, and two bursts of loads larger -than the VM pool, the second after the first burst's Workers stop idle, with -each Worker isolated from the others and from the read-only runtime mount. A -second `orange dev` with a limit of 2 and a 1.5-second idle timeout checks that +server and an Express app in a dynamic Worker, and bursts of loads larger than +the VM pool whose Workers each wait in a call for all the others. The VM pool +grows to the last burst and shrinks to its idle size after the idle timeout, +and each Worker gets a fresh VM, isolated from the others and from the +read-only runtime mount. A second `orange dev` with a limit of 2 and a +1.5-second idle timeout checks that idle Workers stop and load again, that calls longer than the timeout finish, that busy Workers fail a new load with `WorkerLoaderLimitError`, and that more `load()` calls than the limit succeed. diff --git a/packages/runtime-node/src/bindings/worker-loader/agentos.ts b/packages/runtime-node/src/bindings/worker-loader/agentos.ts index 9210174..22e1ebc 100644 --- a/packages/runtime-node/src/bindings/worker-loader/agentos.ts +++ b/packages/runtime-node/src/bindings/worker-loader/agentos.ts @@ -1,8 +1,9 @@ // Runs each dynamic Worker as a Node.js process in its own agentOS VM, so two -// dynamic Workers never share a filesystem, process table, or loopback. A small -// pool keeps VMs booted ahead of use, each with its child process started and -// Orange's runtime imported, so a new dynamic Worker only writes its own modules. -// A pooled VM serves one dynamic Worker and is never reused. +// dynamic Workers never share a filesystem, process table, or loopback. A pool +// sized to recent demand keeps VMs booted ahead of use, each with its child +// process started and Orange's runtime imported, so a new dynamic Worker only +// writes its own modules. A pooled VM serves one dynamic Worker and is never +// reused. // // agentOS sets a VM's network policy in `AgentOs.create()` and has no API to // change it after boot. A Worker with the default `globalOutbound` gets a VM @@ -65,6 +66,8 @@ function assertFitsAgentOsFileMessage(path: string, content: string | Uint8Array type VmChild = WorkerChild & { exited: Promise }; +type LoadHooks = Pick; + // A VM with the runtime bundle mounted and the child process started. The // child has imported the runtime and waits for `start`. It has seen no user // code or env. @@ -75,33 +78,69 @@ interface ReadyVm { root: string; } +// The ready VMs of one network mode. Its target size follows demand: the most +// Workers claimed within one prewarm time, from `size` to `max`, so a burst +// that drained the pool finds it full the next time. It falls back to `size` +// once `idleMs` passes without such a peak. interface VmPool { mode: NetworkMode; ready: ReadyVm[]; - // An open that waits for the prewarm in flight instead of booting its own VM. - waiting: ((ready: ReadyVm | undefined) => void) | undefined; - warming: boolean; + // Opens that wait for a prewarm in flight, at most one for each prewarm. + // Every other open boots its own VM at once. + waiting: ((ready: ReadyVm | undefined) => void)[]; + warming: number; + // The latest claim times, oldest first, at most `max` of them. + claims: number[]; + peak: number; + shrink: NodeJS.Timeout | undefined; + // Set when a prewarm fails. The pool refills again only after its next + // claim, so a VM that cannot boot never loops in the background. + failed: boolean; } -// Ready VMs to keep per network mode. A mode's pool fills only after its first -// dynamic Worker, so an app without dynamic Workers boots no VM. +// A mode's pool fills only after its first dynamic Worker, so an app without +// dynamic Workers boots no VM. export const workerLoaderPoolSizeSetting = "ORANGE_WORKER_LOADER_POOL_SIZE"; +export const workerLoaderPoolMaxSetting = "ORANGE_WORKER_LOADER_POOL_MAX"; const defaultPoolSize = 2; -const maxPoolSize = 16; -// The pool refills a moment after a Worker loads, so the prewarm does not slow -// the Worker's first calls. -const refillDelayMs = 250; - -export function workerLoaderPoolSize(settings: Readonly>): number { - const value = settings[workerLoaderPoolSizeSetting]; - if (value === undefined || value === "") return defaultPoolSize; - const size = Number(value); - if (!Number.isInteger(size) || size < 0 || size > maxPoolSize) { +const defaultPoolMax = 16; +const poolSettingLimit = 64; +// Prewarms in flight across both modes. They share one agentOS sidecar with +// the running Workers. A load that finds its pool empty never waits for this. +const maxParallelPrewarms = 4; +// The longest one Worker holds back the refill while it loads. +const maxRefillHoldMs = 1_000; +// The prewarm time before the first one is measured. +const initialPrewarmMs = 250; + +export interface WorkerLoaderPoolLimits { + // Ready VMs kept per network mode when no burst is recent. + size: number; + // The most ready VMs per network mode. 0 boots every VM on demand. + max: number; +} + +function poolSetting(settings: Readonly>, name: string, fallback: number): number { + const value = settings[name]; + if (value === undefined || value === "") return fallback; + const count = Number(value); + if (!Number.isInteger(count) || count < 0 || count > poolSettingLimit) { + throw new RangeError( + `${name} must be an integer from 0 to ${poolSettingLimit}, got ${JSON.stringify(value)}. Set ${workerLoaderPoolMaxSetting} to 0 to boot every dynamic Worker's VM on demand.`, + ); + } + return count; +} + +export function workerLoaderPoolLimits(settings: Readonly>): WorkerLoaderPoolLimits { + const max = poolSetting(settings, workerLoaderPoolMaxSetting, defaultPoolMax); + const size = poolSetting(settings, workerLoaderPoolSizeSetting, Math.min(defaultPoolSize, max)); + if (size > max) { throw new RangeError( - `${workerLoaderPoolSizeSetting} must be an integer from 0 to ${maxPoolSize}, got ${JSON.stringify(value)}. Set it to 0 to boot every dynamic Worker's VM on demand.`, + `${workerLoaderPoolSizeSetting} (${size}) must not be larger than ${workerLoaderPoolMaxSetting} (${max}).`, ); } - return size; + return { max, size }; } export interface AgentOsWorkerLoaderBackendOptions { @@ -110,7 +149,10 @@ export interface AgentOsWorkerLoaderBackendOptions { bundlePath(): string; // Reports a failure off the request path, such as a failed prewarm. onDiagnostic(message: string): void; - poolSize: number; + pool: WorkerLoaderPoolLimits & { + // How long a pool keeps VMs above `size` after its last peak. + idleMs: number; + }; } export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { @@ -118,9 +160,14 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { readonly #vms = new Set(); readonly #children = new Set<{ exited: Promise; kill(): void }>(); readonly #pools = new Map(); - // The one prewarm in flight. Prewarms run one at a time, since every VM - // shares one agentOS sidecar with the dynamic Workers that are running. - #warming: Promise | undefined; + readonly #warming = new Set>(); + // Dynamic Workers that hold back the refill, since a prewarm that runs + // beside them slows them about 3x. A Worker holds it while it loads. A + // Worker that loads beside another holds it until its first calls finish + // too, so a burst's first calls never share the sidecar with its refill. + readonly #holds = new Set<{ beside: boolean }>(); + // A moving average of how long a prewarm takes. + #prewarmMs = initialPrewarmMs; #closed = false; constructor(options: AgentOsWorkerLoaderBackendOptions) { @@ -135,24 +182,47 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { const mode: NetworkMode = network ? "network" : "deny"; let pool = this.#pools.get(mode); if (pool === undefined) { - pool = { mode, ready: [], waiting: undefined, warming: false }; + pool = { claims: [], failed: false, mode, peak: 0, ready: [], shrink: undefined, waiting: [], warming: 0 }; this.#pools.set(mode, pool); } + const hold = { beside: this.#holds.size > 0 }; + for (const other of this.#holds) other.beside = true; + this.#holds.add(hold); + const release = (): void => { + if (!this.#holds.delete(hold)) return; + clearTimeout(timer); + this.#refill(); + }; + // A Worker whose load hangs must not stop the refill for good. + const timer = setTimeout(release, maxRefillHoldMs); + timer.unref(); + const hooks: LoadHooks = { + loaded: () => { + if (!hold.beside) release(); + }, + settled: release, + }; + try { + return await this.#openVm(pool, hooks); + } catch (error: unknown) { + release(); + throw error; + } + } + + async #openVm(pool: VmPool, hooks: LoadHooks): Promise { // A ready VM serves exactly one dynamic Worker and is disposed with it. const ready = await this.#claim(pool); if (this.#closed) { if (ready !== undefined) await this.#discard(ready); throw new Error("The Worker Loader has stopped."); } - const loaded = (): void => { - setTimeout(() => this.#refill(), refillDelayMs).unref(); - }; if (ready !== undefined) { const child = Promise.resolve(ready.child); // The child is running even if the Worker fails before it is used. - return this.#host(() => this.#discard(ready), ready.booted, ready.root, () => child, loaded); + return this.#host(() => this.#discard(ready), ready.booted, ready.root, () => child, hooks); } - const owned = this.#boot(mode); + const owned = this.#boot(pool.mode); let booted: BootedVm; try { booted = await owned.booted; @@ -161,60 +231,106 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { throw bootError(error); } const root = posix.join(workersRoot, randomUUID()); - return this.#host(() => owned.dispose(), booted, root, () => this.#spawn(booted.vm, root, owned), loaded); + return this.#host(() => owned.dispose(), booted, root, () => this.#spawn(booted.vm, root, owned), hooks); } - // A ready VM, or the prewarm in flight if no other open waits for it, since - // it finishes sooner than a new boot. Otherwise the open boots its own VM. + // A ready VM, or a prewarm in flight that no other open waits for, since it + // finishes sooner than a new boot. Otherwise the open boots its own VM at + // once, beside any other boot. #claim(pool: VmPool): ReadyVm | Promise | undefined { + this.#recordClaim(pool); const ready = pool.ready.shift(); - if (ready !== undefined || !pool.warming || pool.waiting !== undefined) return ready; - return new Promise((resolveReady) => { - pool.waiting = resolveReady; - }); + if (ready !== undefined || pool.warming <= pool.waiting.length) return ready; + return new Promise((resolveReady) => pool.waiting.push(resolveReady)); + } + + // Raises the pool's target to the claims seen within one prewarm time, so + // the refill keeps up with a steady stream of loads and the next burst of + // the same size finds its VMs ready. + #recordClaim(pool: VmPool): void { + const { idleMs, max } = this.#options.pool; + pool.failed = false; + if (max === 0) return; + const now = performance.now(); + pool.claims.push(now); + if (pool.claims.length > max) pool.claims.shift(); + const recent = pool.claims.filter((at) => now - at <= this.#prewarmMs).length; + if (recent < pool.peak) return; + pool.peak = recent; + clearTimeout(pool.shrink); + pool.shrink = setTimeout(() => { + pool.peak = 0; + pool.shrink = undefined; + const excess = pool.ready.splice(0, Math.max(0, pool.ready.length - this.#target(pool))); + for (const ready of excess) void this.#discard(ready); + }, idleMs).unref(); + } + + #target(pool: VmPool): number { + const { max, size } = this.#options.pool; + return Math.min(max, Math.max(size, pool.peak)); } - // Prewarms one VM for the first pool below its size, then the next. A - // failed prewarm is reported and stops the refill until the next Worker - // loads, so a VM that cannot boot never loops in the background. + // Ready VMs, plus prewarms in flight that no open waits for. + #supply(pool: VmPool): number { + return pool.ready.length + pool.warming - pool.waiting.length; + } + + // Starts prewarms for the pools furthest below their target, up to + // `maxParallelPrewarms` at once, while no dynamic Worker holds it back. #refill(): void { - if (this.#closed || this.#warming !== undefined) return; - const pool = [...this.#pools.values()].find((entry) => entry.ready.length < this.#options.poolSize); - if (pool === undefined) return; - pool.warming = true; - this.#warming = this.#prewarm(pool.mode).then( - (ready) => { - this.#warming = undefined; - pool.warming = false; - this.#offer(pool, ready); - this.#refill(); - }, - (error: unknown) => { - this.#warming = undefined; - pool.warming = false; - this.#takeWaiting(pool)?.(undefined); - // Closing disposes a booting VM, which fails its prewarm on purpose. - if (this.#closed) return; - this.#options.onDiagnostic( - `OpenOrange cannot prewarm an agentOS VM for dynamic Workers (${pool.mode}): ${errorMessage(error)}. Dynamic Workers boot their VM on demand instead.`, - ); - }, - ); + while (!this.#closed && this.#holds.size === 0 && this.#warming.size < maxParallelPrewarms) { + let neediest: VmPool | undefined; + let missing = 0; + for (const pool of this.#pools.values()) { + const gap = this.#target(pool) - this.#supply(pool); + if (!pool.failed && gap > missing) { + neediest = pool; + missing = gap; + } + } + if (neediest === undefined) return; + this.#startPrewarm(neediest); + } } - #takeWaiting(pool: VmPool): ((ready: ReadyVm | undefined) => void) | undefined { - const waiting = pool.waiting; - pool.waiting = undefined; - return waiting; + #startPrewarm(pool: VmPool): void { + pool.warming += 1; + const started = performance.now(); + const warming = this.#prewarm(pool.mode) + .then( + (ready) => { + pool.warming -= 1; + this.#prewarmMs += (performance.now() - started - this.#prewarmMs) / 4; + this.#offer(pool, ready); + }, + (error: unknown) => { + pool.warming -= 1; + // An open that waited for this prewarm boots its own VM. + if (pool.waiting.length > pool.warming) pool.waiting.pop()?.(undefined); + // Closing disposes a booting VM, which fails its prewarm on purpose. + if (this.#closed || pool.failed) return; + pool.failed = true; + this.#options.onDiagnostic( + `OpenOrange cannot prewarm an agentOS VM for dynamic Workers (${pool.mode}): ${errorMessage(error)}. Dynamic Workers boot their VM on demand instead.`, + ); + }, + ) + .finally(() => { + this.#warming.delete(warming); + this.#refill(); + }); + this.#warming.add(warming); } #offer(pool: VmPool, ready: ReadyVm): void { - const waiting = this.#takeWaiting(pool); + const waiting = pool.waiting.shift(); if (waiting !== undefined) { waiting(ready); return; } - if (this.#closed) { + // The target may have shrunk while the VM booted. + if (this.#closed || pool.ready.length >= this.#target(pool)) { void this.#discard(ready); return; } @@ -238,11 +354,11 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { booted: BootedVm, root: string, spawn: () => Promise, - loaded: () => void, + hooks: LoadHooks, ): WorkerHost { const { moduleRewrites, vm } = booted; return { - loaded, + ...hooks, moduleRewrites, remove, root, @@ -345,10 +461,13 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { // A prewarm that is booting finishes or fails first, so none starts a // child after the children below are stopped. await Promise.race([ - this.#warming, + Promise.all(this.#warming), new Promise((resolveWait) => setTimeout(resolveWait, exitWaitMs).unref()), ]); - for (const pool of this.#pools.values()) this.#takeWaiting(pool)?.(undefined); + for (const pool of this.#pools.values()) { + clearTimeout(pool.shrink); + for (const waiting of pool.waiting.splice(0)) waiting(undefined); + } // Disposing a VM with a running process fails to observe its exit, so // stop the children first. const children = [...this.#children]; diff --git a/packages/runtime-node/src/bindings/worker-loader/loader.ts b/packages/runtime-node/src/bindings/worker-loader/loader.ts index 99bbce1..5b4306d 100644 --- a/packages/runtime-node/src/bindings/worker-loader/loader.ts +++ b/packages/runtime-node/src/bindings/worker-loader/loader.ts @@ -47,6 +47,9 @@ export interface WorkerHost { spawn(): Promise; // Runs once the dynamic Worker has loaded or failed to load. loaded(): void; + // Runs once the loaded dynamic Worker first has no call in flight, or when + // it fails to load. + settled(): void; // Disposes the VM. Runs when the process exits or fails to start. remove(): Promise; } @@ -244,6 +247,10 @@ function positiveIntegerSetting( return value; } +export function workerLoaderIdleTimeoutMs(settings: Readonly>): number { + return positiveIntegerSetting(settings, idleTimeoutSetting, defaultIdleTimeoutMs, maxTimerDelayMs); +} + // Bounds the dynamic Workers that run at once across every Worker Loader binding of the runtime. // Each one is an agentOS VM and a Node.js process, about 54 MiB. At the limit, a new Worker stops // the least recently used idle one. A Worker with a call in flight is never stopped. @@ -267,7 +274,7 @@ export class DynamicWorkerPool { ): DynamicWorkerPool { return new DynamicWorkerPool( { - idleTimeoutMs: positiveIntegerSetting(settings, idleTimeoutSetting, defaultIdleTimeoutMs, maxTimerDelayMs), + idleTimeoutMs: workerLoaderIdleTimeoutMs(settings), maxWorkers: positiveIntegerSetting(settings, maxWorkersSetting, defaultMaxWorkers, Number.MAX_SAFE_INTEGER), }, warn, @@ -335,6 +342,8 @@ class DynamicWorker { #exited = false; #idleTimer: NodeJS.Timeout | undefined; #inFlight = 0; + // The host of a Worker that has loaded and whose first calls are in flight. + #settling: WorkerHost | undefined; #lastUsed = 0; #orphaned = false; #session: LoaderSession | undefined; @@ -433,8 +442,11 @@ class DynamicWorker { this.#session = session; this.#lastUsed = performance.now(); this.#armIdleTimer(this.#options.pool.idleTimeoutMs); + if (this.#inFlight === 0) host.settled(); + else this.#settling = host; return session; } catch (error: unknown) { + host?.settled(); this.terminate(); // A running child frees the slot when it exits. Without one, nothing else frees it. if (this.#child === undefined) { @@ -486,6 +498,10 @@ class DynamicWorker { const settle = (): void => { this.#inFlight -= 1; this.#lastUsed = performance.now(); + if (this.#inFlight === 0) { + this.#settling?.settled(); + this.#settling = undefined; + } if (this.#orphaned && this.idle) this.evict(); }; void call.then(settle, settle); diff --git a/packages/runtime-node/src/runner.ts b/packages/runtime-node/src/runner.ts index dfc1bc3..6405e52 100644 --- a/packages/runtime-node/src/runner.ts +++ b/packages/runtime-node/src/runner.ts @@ -55,10 +55,11 @@ import { type WorkflowEngineObject, workflowEngineClassName, } from "./bindings/workflows/engine-object.js"; -import { AgentOsWorkerLoaderBackend, workerLoaderPoolSize } from "./bindings/worker-loader/agentos.js"; +import { AgentOsWorkerLoaderBackend, workerLoaderPoolLimits } from "./bindings/worker-loader/agentos.js"; import { DynamicWorkerPool, WorkerLoader, + workerLoaderIdleTimeoutMs, workerLoaderVmBundlePath, type WorkerLoaderBackend, } from "./bindings/worker-loader/loader.js"; @@ -226,7 +227,8 @@ function workerLoaderBackend(): WorkerLoaderBackend { onDiagnostic(message): void { send({ message, type: "background-error" }); }, - poolSize: workerLoaderPoolSize(runtimeSettings()), + // The pool keeps VMs above its idle size as long as a Worker stays idle before it stops. + pool: { ...workerLoaderPoolLimits(runtimeSettings()), idleMs: workerLoaderIdleTimeoutMs(runtimeSettings()) }, }); return sharedWorkerLoaderBackend; } diff --git a/tests/bindings/worker-loader.test.ts b/tests/bindings/worker-loader.test.ts index ee2cf93..bf548a2 100644 --- a/tests/bindings/worker-loader.test.ts +++ b/tests/bindings/worker-loader.test.ts @@ -326,33 +326,33 @@ test("refuses to start with an idle timeout that Node.js timers cannot hold", as } }); -test("loads bursts of dynamic Workers larger than the VM pool, each in a fresh VM even after idle Workers stop", async () => { +test("grows the VM pool to the last burst, shrinks it when idle, and boots each Worker in a fresh VM", async () => { + // The idle timeout leaves time to refill the pool between the first two + // bursts, and also sets how long the pool keeps VMs above its idle size. + const idleMs = 8_000; const environment = { - ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS: "1000", - ORANGE_WORKER_LOADER_POOL_SIZE: "2", + ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS: String(idleMs), + ORANGE_WORKER_LOADER_POOL_MAX: "8", + ORANGE_WORKER_LOADER_POOL_SIZE: "1", }; await withOrangeDev("worker-loader", async (dev) => { - const loadAtOnce = async (n: number): Promise => { - const results = await dev.json(`/pool?n=${n}`); + const sleep = (ms: number): Promise => new Promise((resolveWait) => setTimeout(resolveWait, ms)); + // Loads n Workers at once. Each Worker's call waits for all n, so the + // burst completes only if no Worker's boot waits for another's call. + // Returns how many found their VM booted before the load. + const loadAtOnce = async (n: number, outbound = ""): Promise => { + const results = await dev.json(`/pool?n=${n}&outbound=${outbound}`, { + signal: AbortSignal.timeout(30_000), + }); assert.ok(Array.isArray(results) && results.length === n, JSON.stringify(results)); - return results; - }; - // The first dynamic Worker of each network mode boots on demand and - // starts filling that mode's pool. - await loadAtOnce(2); - for (let round = 0; round < 2; round += 1) { - // Leaves time for the pool to fill and, after round 0, for every - // Worker to pass its idle timeout and stop. A stopped Worker's VM - // must be disposed, never handed to the next Worker. - await new Promise((resolveWait) => setTimeout(resolveWait, 5_000)); - // A burst larger than the pool loads every Worker: some from the pool, - // the rest on demand. None sees another's files, globals, or env, each - // VM holds only its own Worker, and none can change Orange's runtime. - const results = await loadAtOnce(10); + let prewarmed = 0; for (const result of results) { assert.ok(isRecord(result) && isRecord(result.value), JSON.stringify(result)); - const id = String(result.value.id); - assert.deepEqual(result.value, { + const { prewarmed: wasPrewarmed, ...seen } = result.value; + const id = String(seen.id); + // None sees another's files, globals, or env, each VM holds only + // its own Worker, and none can change Orange's runtime. + assert.deepEqual(seen, { env: [`POOL_SECRET_${id}`], file: "ENOENT", global: null, @@ -360,7 +360,22 @@ test("loads bursts of dynamic Workers larger than the VM pool, each in a fresh V runtime: "EROFS", workerRoots: 1, }); + if (wasPrewarmed === true) prewarmed += 1; } - } + return prewarmed; + }; + // Both network modes load beside each other. + await loadAtOnce(4); + // A burst larger than the idle pool loads every Worker, most on VMs + // that boot on demand. + await loadAtOnce(8, "blocked"); + await sleep(idleMs / 2); + // The pool refilled to the size of the last burst. + assert.equal(await loadAtOnce(8, "blocked"), 8); + // After the idle timeout, the first burst's Workers have stopped, and + // the pool is back to its idle size. Stopped Workers' VMs are never + // handed to new Workers. + await sleep(idleMs + 2_000); + assert.equal(await loadAtOnce(8, "blocked"), 1); }, { environment }); }); diff --git a/tests/fixtures/worker-loader/worker.js b/tests/fixtures/worker-loader/worker.js index b93c6c9..b5bd2e9 100644 --- a/tests/fixtures/worker-loader/worker.js +++ b/tests/fixtures/worker-loader/worker.js @@ -24,6 +24,9 @@ const filesystemCode = { // Reports what a dynamic Worker can see of any other dynamic Worker: a file, // a global, process.env, and the VM's Worker roots. Then it leaves its own, // and tries to change Orange's runtime, which every VM reads from the host. +// It also reports whether its VM had its Worker root before the parent asked +// for the Worker at `requestedAt`, as a prewarmed VM does, and waits at +// `arrive` for the rest of its burst. const poolProbeCode = (id, globalOutbound) => ({ compatibilityDate: "2026-09-03", mainModule: "main.js", @@ -36,7 +39,9 @@ const poolProbeCode = (id, globalOutbound) => ({ const seenGlobal = globalThis.poolProbe ?? null; globalThis.poolProbe = ${JSON.stringify(id)}; export default { - probe() { + async probe({ arrive, requestedAt }) { + const rootCreatedAt = fs.statSync(process.cwd()).birthtimeMs; + await arrive(); let file; try { file = fs.readFileSync("/tmp/pool-probe.txt", "utf8"); } catch (error) { file = error.code; } fs.writeFileSync("/tmp/pool-probe.txt", ${JSON.stringify(id)}); @@ -47,6 +52,7 @@ const poolProbeCode = (id, globalOutbound) => ({ file, global: seenGlobal, id: ${JSON.stringify(id)}, + prewarmed: rootCreatedAt < requestedAt, runtime, workerRoots: fs.readdirSync("/open-orange/workers").length, }; @@ -386,12 +392,28 @@ const routes = { }, // Loads `n` dynamic Workers at once, alternating the VM network mode. async pool(env, _ctx, request) { - const n = Number(new URL(request.url).searchParams.get("n")); + const url = new URL(request.url); + const n = Number(url.searchParams.get("n")); + // With `blocked`, every Worker gets a VM with no network, so the burst + // draws from one VM pool. + const blocked = url.searchParams.get("outbound") === "blocked"; const tag = crypto.randomUUID().slice(0, 8); + // Every Worker's call waits until all n are in their call, so the + // burst completes only if its Workers boot beside each other. + let arrived = 0; + let release; + const everyone = new Promise((resolve) => { + release = resolve; + }); + const arrive = () => { + arrived += 1; + if (arrived === n) release(); + return everyone; + }; return Promise.all( Array.from({ length: n }, (_, i) => { - const code = poolProbeCode(`${tag}_${i}`, i % 2 === 0 ? null : undefined); - return settle(env.LOADER.load(code).getEntrypoint().probe(null)); + const code = poolProbeCode(`${tag}_${i}`, blocked || i % 2 === 0 ? null : undefined); + return settle(env.LOADER.load(code).getEntrypoint().probe({ arrive, requestedAt: Date.now() })); }), ); }, From 418456e8e61b2635cc1bbfcbf917cf2c02e6cd6a Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sun, 15 Nov 2026 04:17:27 +0400 Subject: [PATCH 4/7] fix(worker-loader): queue dynamic Workers at the slot limit and bound every VM - Wait in a bounded FIFO queue for a slot instead of failing at once; refuse past the deadline or queue limit with a typed WorkerLoaderLimitError that the runtime answers with 503 and Retry-After - Bound running, pooled, booting, and stopping VMs with one VM budget; prewarmed VMs stop first - Keep refilling the pool under a steady stream of loads, at the rate the stream needs - Export slot, queue, pool, VM, memory, and refusal metrics; document autoscaling --- docs/modules/worker-loader.mdx | 46 +++- docs/operations/monitoring.mdx | 64 ++++- .../src/bindings/worker-loader/agentos.ts | 191 +++++++++++++- .../src/bindings/worker-loader/loader.ts | 245 +++++++++++++++--- .../src/bindings/worker-loader/metrics.ts | 175 +++++++++++++ packages/runtime-node/src/runner.ts | 77 +++++- src/commands/d1.ts | 2 + src/commands/dev.ts | 2 + tests/bindings/worker-loader.test.ts | 150 ++++++++++- tests/fixtures/worker-loader/worker.js | 76 ++++-- 10 files changed, 927 insertions(+), 101 deletions(-) create mode 100644 packages/runtime-node/src/bindings/worker-loader/metrics.ts diff --git a/docs/modules/worker-loader.mdx b/docs/modules/worker-loader.mdx index 82527b3..f640a27 100644 --- a/docs/modules/worker-loader.mdx +++ b/docs/modules/worker-loader.mdx @@ -126,17 +126,44 @@ agentOS's limits apply: | One module file | 16 MiB | The load fails with a `RangeError` that names the limit. Split the module. | | All files in the VM | 64 MiB by default | agentOS's limit. Orange adds no check. | | `env` | No set limit. In a test with agentOS 0.4.1, 5 MiB worked. | At 10 MiB, the dynamic Worker exited with code 1. | -| Running dynamic Workers, across all Worker Loader bindings | 32. Set `ORANGE_WORKER_LOADER_MAX_WORKERS` to change it. | A new dynamic Worker stops the least recently used idle one. If none is idle, `get()` or `load()` throws a `WorkerLoaderLimitError` that names the limit. At 80% of the limit, `orange dev` logs a warning. | +| Running dynamic Workers, across all Worker Loader bindings | 32. Set `ORANGE_WORKER_LOADER_MAX_WORKERS` to change it. | A new dynamic Worker stops the least recently used idle one. If none is idle, it waits in line for a slot, first come first served, and its calls wait with it. At 80% of the limit, Orange logs a warning. | +| Time a dynamic Worker waits for a slot | 5 seconds. Set `ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS` to change it, from 1 to 2147483647. | Its calls reject with a `WorkerLoaderLimitError` whose `code` is `queue_timeout`. | +| Dynamic Workers that wait for a slot | 256. Set `ORANGE_WORKER_LOADER_MAX_QUEUED` to change it; `0` refuses at once. | Its calls reject with a `WorkerLoaderLimitError` whose `code` is `queue_full`. At 80% of the limit, Orange logs a warning. | +| agentOS VMs, counting running Workers, prewarmed VMs, and VMs that boot or stop | `ORANGE_WORKER_LOADER_MAX_WORKERS` plus `ORANGE_WORKER_LOADER_POOL_MAX`, 48 by default. Set `ORANGE_WORKER_LOADER_MAX_VMS` to change it; it must not be less than `ORANGE_WORKER_LOADER_MAX_WORKERS`. | Prewarmed VMs stop first, and the pool stops refilling. A Worker that needs a new VM waits for one to stop. At 80% of the budget, Orange logs a warning. | | Idle time before a dynamic Worker stops | 30 seconds. Set `ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS` to change it, from 1 to 2147483647. | The dynamic Worker stops and frees its VM. The next call loads it again. A value out of range stops `orange dev` at startup with a `RangeError` that names the setting. | | Prewarmed VMs when no burst is recent | 2 for each network mode by default, from 0 to 64 with `ORANGE_WORKER_LOADER_POOL_SIZE` | A load that finds the pool empty boots its own VM. | | Prewarmed VMs after a burst | 16 for each network mode by default, from 0 to 64 with `ORANGE_WORKER_LOADER_POOL_MAX`. It must not be less than `ORANGE_WORKER_LOADER_POOL_SIZE`. `0` boots every VM on demand. | The pool stops growing. A value out of range stops `orange dev` at startup with a `RangeError` that names the setting. | -| VMs that prewarm at once | 4, across both network modes | The other prewarms wait. Loads never wait for this limit. | +| VMs that prewarm at once | 4, across both network modes, while no Worker loads. While Workers load, only as many as keep up with a steady rate of loads. | The other prewarms wait. Loads never wait for this limit. | Each running dynamic Worker uses about 54 MiB. Each prewarmed VM uses 26 to 54 MiB, depending on the agentOS version. After a burst of 16 or more loads, a network mode can hold 16 prewarmed VMs, 0.4 to 0.9 GiB, until the idle -timeout passes. Prewarmed VMs that no Worker has claimed do not count toward -`ORANGE_WORKER_LOADER_MAX_WORKERS`, and the limit never stops them. +timeout passes. Prewarmed VMs do not count toward +`ORANGE_WORKER_LOADER_MAX_WORKERS`, but they count toward +`ORANGE_WORKER_LOADER_MAX_VMS`, which bounds the memory of all VMs. + +##### Refusals + +- A `WorkerLoaderLimitError` is a `RangeError` with `code` (`queue_full` or + `queue_timeout`), `overloaded: true`, `retryable: true`, and + `retryAfterSeconds: 1`, like the overloaded error of a Cloudflare Durable + Object. `get()` and `load()` never throw it; the Worker's calls reject with it. +- If your Worker does not catch it, the response is `503 Service Unavailable` + with `Retry-After: 1`, not 500. A WebSocket upgrade gets the same 503. On + Cloudflare, an uncaught exception is a 500 (error 1101). Cloudflare has no + per-runner Worker limit, so it has no such refusal. +- A held stub whose Worker was refused asks for a slot again on its next call, + and `get()` with the same name calls `getCode` again. +- Under load that stays above what the runner can start, the line stays full: + a Worker that gets a slot has waited close to the queue timeout, and the + rest are refused. Lower `ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS` to trade + refusals for latency, and scale out on the queue wait (see + [Monitoring](../operations/monitoring#autoscaling)). +- agentOS bounds the requests in flight to its shared sidecar at 128, and + each running Worker keeps some open. In a test that started 80 Workers at + once with `ORANGE_WORKER_LOADER_MAX_WORKERS` at 96, 33 to 38 started; the + rest failed with agentOS's `ERR_AGENTOS_IN_FLIGHT_REQUEST_LIMIT`. When a + running Worker's process fails this way, its call fails with a 500. #### Isolation @@ -161,10 +188,13 @@ timeout passes. Prewarmed VMs that no Worker has claimed do not count toward time, from `ORANGE_WORKER_LOADER_POOL_SIZE` to `ORANGE_WORKER_LOADER_POOL_MAX`. So the next burst of the same size finds its VMs ready, and a steady stream of loads keeps the pool ahead. -- The pool refills as soon as no dynamic Worker is loading, because a - prewarm that runs beside a load slows it. Workers that load at the same - time also hold the refill until their first calls finish. One Worker holds - the refill back for at most 1 second. +- The pool refills at full speed as soon as no dynamic Worker is loading, + because a prewarm that runs beside a load slows it. Workers that load at the + same time also hold the refill until their first calls finish. One Worker + holds the refill back for at most 1 second. +- While Workers keep loading, the pool still refills, with as many prewarms + as the rate of loads needs. Only loads that keep arriving count, so a burst + does not refill beside its own loads. - When the idle timeout (`ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS`) passes without a burst as large, the pool shrinks to `ORANGE_WORKER_LOADER_POOL_SIZE` and disposes the other prewarmed VMs. diff --git a/docs/operations/monitoring.mdx b/docs/operations/monitoring.mdx index fda2733..651e471 100644 --- a/docs/operations/monitoring.mdx +++ b/docs/operations/monitoring.mdx @@ -68,6 +68,12 @@ $ docker run \ -e RIVET_ENDPOINT=... my-worker ``` +## Autoscale dynamic Workers + +Scale runners on `orange_worker_loader_slot_utilization` and the slot queue +wait, not on CPU. See [Dynamic Worker metrics](#dynamic-worker-metrics) for an +example. + ## Learn more - [OpenTelemetry](https://rivet.dev/docs/tracing/): span names, sampling, and application spans @@ -95,7 +101,7 @@ RivetKit serves Prometheus metrics at `http://127.0.0.1:$RIVETKIT_METRICS_PORT/m them in the Rivet dashboard. `orange deploy` refuses to set it with `--var`. - **Self-hosted:** the endpoint listens on `127.0.0.1` only, so `docker run -p` does not expose it. Scrape it from the same network namespace, such as a sidecar in the same Kubernetes pod. -- `orange dev` serves no metrics. +- `orange dev` serves metrics when `RIVETKIT_METRICS_PORT` is set, from its first Worker. - With several Workers, only the first Worker's runtime serves metrics, so they cover that Worker's objects only. @@ -105,6 +111,62 @@ RivetKit serves Prometheus metrics at `http://127.0.0.1:$RIVETKIT_METRICS_PORT/m | `rivet_rivetkit_actor_http_requests_active` | Requests and RPC calls in progress | | `rivet_rivetkit_actor_inbox_depth` | Work waiting to run | +#### Dynamic Worker metrics + +A Worker with a `worker_loaders` binding adds these to the same endpoint. + +| Metric | Type | What it shows | +|---|---|---| +| `orange_worker_loader_workers` | gauge | Dynamic Workers that hold a slot | +| `orange_worker_loader_workers_max` | gauge | `ORANGE_WORKER_LOADER_MAX_WORKERS` | +| `orange_worker_loader_slot_utilization` | gauge | Slots in use divided by slots, from 0 to 1 | +| `orange_worker_loader_queue_depth` | gauge | Dynamic Workers that wait for a slot | +| `orange_worker_loader_queue_wait_seconds` | histogram | How long each dynamic Worker waited for a slot | +| `orange_worker_loader_start_seconds` | histogram | From a slot to a loaded dynamic Worker | +| `orange_worker_loader_vm_boot_seconds` | histogram | How long one agentOS VM takes to boot | +| `orange_worker_loader_pool_claims_total` | counter | VM claims by `mode` (`network`, `deny`) and `result`: `hit` (prewarmed), `warming` (waited for a prewarm), `miss` (booted on demand) | +| `orange_worker_loader_pool_ready` | gauge | Prewarmed VMs by `mode` | +| `orange_worker_loader_vms` | gauge | agentOS VMs by `state`: `live`, `pooled`, `booting`, `stopping` | +| `orange_worker_loader_vms_max` | gauge | `ORANGE_WORKER_LOADER_MAX_VMS` | +| `orange_worker_loader_vm_queue_depth` | gauge | Dynamic Workers with a slot that wait for room in the VM budget | +| `orange_worker_loader_rejections_total` | counter | `WorkerLoaderLimitError` refusals by `code`: `queue_full`, `queue_timeout` | +| `orange_runtime_resident_memory_bytes` | gauge | RSS by `process`: `runner`, and on Linux `children`, which includes the agentOS sidecar | +| `orange_runtime_memory_limit_bytes` | gauge | The runner's cgroup memory limit, when it has one | + +Useful queries: + +```promql +# Pool miss rate +sum(rate(orange_worker_loader_pool_claims_total{result="miss"}[5m])) + / sum(rate(orange_worker_loader_pool_claims_total[5m])) +# Slot queue wait, p95 +histogram_quantile(0.95, sum by (le) (rate(orange_worker_loader_queue_wait_seconds_bucket[5m]))) +# Memory headroom +sum(orange_runtime_resident_memory_bytes) / orange_runtime_memory_limit_bytes +``` + +The agentOS sidecar's own request queue is not visible to Orange, so it has no metric. + +#### Autoscaling + +With a Kubernetes HorizontalPodAutoscaler and a Prometheus adapter: + +- Scale on `orange_worker_loader_slot_utilization`, with a target of about + 0.7, and on the p95 of `orange_worker_loader_queue_wait_seconds`. A busy + slot can mean a dynamic Worker that waits on I/O, so CPU does not show + dynamic Worker demand. Do not scale on CPU. +- Set `behavior.scaleDown.stabilizationWindowSeconds` longer than + `ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS` (30 s by default), such as 300 s. An + idle dynamic Worker holds its slot until the timeout, so a shorter window + scales down while slots still look busy, and then scales up again. +- Alert on `orange_worker_loader_rejections_total`. Each refusal is a 503 or + a `WorkerLoaderLimitError` your Worker caught. +- Size the pod's memory limit for `ORANGE_WORKER_LOADER_MAX_VMS` VMs, and + watch `orange_runtime_resident_memory_bytes` against + `orange_runtime_memory_limit_bytes`. +- Orange has no readiness endpoint, so a full runner still receives + requests. Its 503s carry `Retry-After`. + #### Logs - `orange dev` prints each request and your Worker's `console` output. diff --git a/packages/runtime-node/src/bindings/worker-loader/agentos.ts b/packages/runtime-node/src/bindings/worker-loader/agentos.ts index 22e1ebc..306b9ca 100644 --- a/packages/runtime-node/src/bindings/worker-loader/agentos.ts +++ b/packages/runtime-node/src/bindings/worker-loader/agentos.ts @@ -21,7 +21,8 @@ import type { AgentOs } from "@rivet-dev/agentos-core"; import { onRuntimeClose } from "../../internal/close-hooks.js"; import { errorMessage, isRecord } from "../unknown-values.js"; -import type { WorkerChild, WorkerHost, WorkerLoaderBackend } from "./loader.js"; +import { positiveIntegerSetting, type WorkerChild, type WorkerHost, type WorkerLoaderBackend } from "./loader.js"; +import type { VmGauges, WorkerLoaderMetrics } from "./metrics.js"; // Where the child and runtime modules live inside each VM. const runtimeRoot = "/open-orange/runtime"; @@ -110,8 +111,13 @@ const poolSettingLimit = 64; const maxParallelPrewarms = 4; // The longest one Worker holds back the refill while it loads. const maxRefillHoldMs = 1_000; +// How far back the claim rate looks when loads keep the refill held. +const claimRateWindowMs = 1_000; +// Enough claim times to show a rate that needs every parallel prewarm. +const maxRecentClaims = 512; // The prewarm time before the first one is measured. const initialPrewarmMs = 250; +const warnRatio = 0.8; export interface WorkerLoaderPoolLimits { // Ready VMs kept per network mode when no burst is recent. @@ -120,6 +126,26 @@ export interface WorkerLoaderPoolLimits { max: number; } +// Every agentOS VM of dynamic Workers counts: running Workers, prewarmed VMs, VMs that boot, +// and VMs that are still stopping. A running Worker's slot frees as soon as it stops, before its +// VM is gone, so without this budget a churn of Workers could hold more VMs than the slots. +export const workerLoaderMaxVmsSetting = "ORANGE_WORKER_LOADER_MAX_VMS"; + +// Defaults to the slots plus one pool's most, so the pool has room beside full slots. +export function workerLoaderMaxVms( + settings: Readonly>, + maxWorkers: number, + pool: WorkerLoaderPoolLimits, +): number { + return positiveIntegerSetting( + settings, + workerLoaderMaxVmsSetting, + maxWorkers + pool.max, + Number.MAX_SAFE_INTEGER, + maxWorkers, + ); +} + function poolSetting(settings: Readonly>, name: string, fallback: number): number { const value = settings[name]; if (value === undefined || value === "") return fallback; @@ -149,6 +175,10 @@ export interface AgentOsWorkerLoaderBackendOptions { bundlePath(): string; // Reports a failure off the request path, such as a failed prewarm. onDiagnostic(message: string): void; + onWarning(message: string): void; + // At least the slots of the DynamicWorkerPool, so every Worker with a slot gets a VM. + maxVms: number; + metrics: WorkerLoaderMetrics; pool: WorkerLoaderPoolLimits & { // How long a pool keeps VMs above `size` after its last peak. idleMs: number; @@ -157,7 +187,13 @@ export interface AgentOsWorkerLoaderBackendOptions { export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { readonly #options: AgentOsWorkerLoaderBackendOptions; + // Every VM from its boot until its disposal finishes. readonly #vms = new Set(); + readonly #stopping = new Set(); + // Opens that wait for room in the VM budget, first in line first. Each boots its VM in the + // callback, so no prewarm takes the room first. + readonly #vmWaiters: { boot(): void; fail(error: Error): void }[] = []; + #warnedVms = false; readonly #children = new Set<{ exited: Promise; kill(): void }>(); readonly #pools = new Map(); readonly #warming = new Set>(); @@ -168,15 +204,37 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { readonly #holds = new Set<{ beside: boolean }>(); // A moving average of how long a prewarm takes. #prewarmMs = initialPrewarmMs; + // Claim times across both modes, oldest first, within one prewarm time plus the rate window. + readonly #recentClaims: number[] = []; + // VMs that boot for a load that found no prewarmed VM. + #bootingOnDemand = 0; #closed = false; constructor(options: AgentOsWorkerLoaderBackendOptions) { this.#options = options; + options.metrics.vmGauges = () => this.#gauges(); // Each VM owns a sidecar process, so stop every child and dispose the // VMs when the runtime stops. onRuntimeClose(() => this.#close()); } + #gauges(): VmGauges { + let pooled = 0; + const poolReady: Record = {}; + for (const pool of this.#pools.values()) { + pooled += pool.ready.length; + poolReady[pool.mode] = pool.ready.length; + } + const booting = this.#warming.size + this.#bootingOnDemand; + const stopping = this.#stopping.size; + return { + maxVms: this.#options.maxVms, + poolReady, + waiting: this.#vmWaiters.length, + vms: { booting, live: Math.max(0, this.#vms.size - pooled - booting - stopping), pooled, stopping }, + }; + } + async open(network: boolean): Promise { if (this.#closed) throw new Error("The Worker Loader has stopped."); const mode: NetworkMode = network ? "network" : "deny"; @@ -212,7 +270,13 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { async #openVm(pool: VmPool, hooks: LoadHooks): Promise { // A ready VM serves exactly one dynamic Worker and is disposed with it. - const ready = await this.#claim(pool); + const claim = this.#claim(pool); + this.#refill(); + const ready = await claim; + this.#options.metrics.claim( + pool.mode, + ready === undefined ? "miss" : claim instanceof Promise ? "warming" : "hit", + ); if (this.#closed) { if (ready !== undefined) await this.#discard(ready); throw new Error("The Worker Loader has stopped."); @@ -222,16 +286,64 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { // The child is running even if the Worker fails before it is used. return this.#host(() => this.#discard(ready), ready.booted, ready.root, () => child, hooks); } - const owned = this.#boot(pool.mode); + let owned: OwnedVm | undefined; let booted: BootedVm; try { - booted = await owned.booted; + owned = await this.#bootWithinBudget(pool.mode); + this.#bootingOnDemand += 1; + try { + booted = await owned.booted; + } finally { + this.#bootingOnDemand -= 1; + } } catch (error: unknown) { - await owned.dispose(); + await owned?.dispose(); throw bootError(error); } + const bootedVm = owned; const root = posix.join(workersRoot, randomUUID()); - return this.#host(() => owned.dispose(), booted, root, () => this.#spawn(booted.vm, root, owned), hooks); + return this.#host(() => bootedVm.dispose(), booted, root, () => this.#spawn(booted.vm, root, bootedVm), hooks); + } + + // Boots a VM for a load once the VM budget has room. At the budget, prewarmed VMs stop to + // make room, and the load waits for a VM that stops. + #bootWithinBudget(mode: NetworkMode): Promise { + if (this.#closed) return Promise.reject(new Error("The Worker Loader has stopped.")); + if (this.#vmWaiters.length === 0 && this.#vms.size < this.#options.maxVms) { + return Promise.resolve(this.#boot(mode)); + } + return new Promise((resolveBoot, rejectBoot) => { + this.#vmWaiters.push({ boot: () => resolveBoot(this.#boot(mode)), fail: rejectBoot }); + this.#makeRoom(); + }); + } + + // Stops prewarmed VMs, from the fullest pool first, while more loads wait for room than the + // VMs that already stop will free. + #makeRoom(): void { + let short = this.#vmWaiters.length - this.#stopping.size; + while (short > 0) { + let fullest: VmPool | undefined; + for (const pool of this.#pools.values()) { + if (pool.ready.length > (fullest?.ready.length ?? 0)) fullest = pool; + } + const ready = fullest?.ready.shift(); + if (ready === undefined) return; + void this.#discard(ready); + short -= 1; + } + } + + // A VM finished stopping, so a load that waits for room may boot. + #vmFreed(): void { + if (this.#closed) { + for (const waiter of this.#vmWaiters.splice(0)) waiter.fail(new Error("The Worker Loader has stopped.")); + } + while (this.#vmWaiters.length > 0 && this.#vms.size < this.#options.maxVms) { + this.#vmWaiters.shift()?.boot(); + } + if (this.#vms.size < Math.ceil(this.#options.maxVms * warnRatio)) this.#warnedVms = false; + this.#refill(); } // A ready VM, or a prewarm in flight that no other open waits for, since it @@ -252,6 +364,11 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { pool.failed = false; if (max === 0) return; const now = performance.now(); + this.#recentClaims.push(now); + const oldest = now - this.#prewarmMs - claimRateWindowMs; + while (this.#recentClaims.length > maxRecentClaims || (this.#recentClaims[0] ?? now) < oldest) { + this.#recentClaims.shift(); + } pool.claims.push(now); if (pool.claims.length > max) pool.claims.shift(); const recent = pool.claims.filter((at) => now - at <= this.#prewarmMs).length; @@ -276,10 +393,34 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { return pool.ready.length + pool.warming - pool.waiting.length; } - // Starts prewarms for the pools furthest below their target, up to - // `maxParallelPrewarms` at once, while no dynamic Worker holds it back. + // How many prewarms may run now. With no dynamic Worker loading, `maxParallelPrewarms`. While + // Workers load, a prewarm beside them slows them, so only a steady stream of claims earns + // prewarms: as many as keep up with its rate. A steady stream claims VMs both within the last + // prewarm time and in the second before it; a burst does only once, so it never refills + // beside its own loads. + #refillLimit(): number { + if (this.#holds.size === 0) return maxParallelPrewarms; + const now = performance.now(); + let recent = 0; + let earlier = 0; + for (const at of this.#recentClaims) { + const age = now - at; + if (age <= this.#prewarmMs) recent += 1; + else if (age <= this.#prewarmMs + claimRateWindowMs) earlier += 1; + } + const ratePerMs = Math.min(recent / this.#prewarmMs, earlier / claimRateWindowMs); + return Math.min(maxParallelPrewarms, Math.ceil(ratePerMs * this.#prewarmMs)); + } + + // Starts prewarms for the pools furthest below their target, up to the refill limit at once, + // while the VM budget has room and no load waits for it. #refill(): void { - while (!this.#closed && this.#holds.size === 0 && this.#warming.size < maxParallelPrewarms) { + while ( + !this.#closed && + this.#warming.size < this.#refillLimit() && + this.#vmWaiters.length === 0 && + this.#vms.size < this.#options.maxVms + ) { let neediest: VmPool | undefined; let missing = 0; for (const pool of this.#pools.values()) { @@ -329,8 +470,8 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { waiting(ready); return; } - // The target may have shrunk while the VM booted. - if (this.#closed || pool.ready.length >= this.#target(pool)) { + // The target may have shrunk while the VM booted, or a load may wait for its room. + if (this.#closed || pool.ready.length >= this.#target(pool) || this.#vmWaiters.length > this.#stopping.size) { void this.#discard(ready); return; } @@ -377,10 +518,18 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { } async #spawn(vm: AgentOs, root: string, owned: OwnedVm): Promise { - const child = await spawnVmChild(vm, root, { + const spawned = await spawnVmChild(vm, root, { dispose: () => owned.dispose(), report: (message) => this.#options.onDiagnostic(message), }); + // A VM counts as stopping from the kill on, so room that it will free is not made twice. + const child: VmChild = { + ...spawned, + kill: () => { + this.#stopping.add(owned); + spawned.kill(); + }, + }; const entry = { exited: child.exited, kill: () => child.kill() }; this.#children.add(entry); void child.exited.then(() => this.#children.delete(entry)); @@ -411,13 +560,18 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { } #boot(mode: NetworkMode): OwnedVm { + const startedAt = performance.now(); const booted = this.#create(mode); + void booted.then( + () => this.#options.metrics.vmBoot.observe((performance.now() - startedAt) / 1_000), + () => undefined, + ); let disposed: Promise | undefined; const owned: OwnedVm = { booted, dispose: () => { disposed ??= (async () => { - this.#vms.delete(owned); + this.#stopping.add(owned); // A VM that failed to boot has nothing to dispose, and its boot error reaches the // caller that booted it. const vm = await booted.then(({ vm: value }) => value).catch(() => undefined); @@ -426,11 +580,21 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { `OpenOrange cannot dispose the agentOS VM of a dynamic Worker: ${errorMessage(error)}`, ); }); + this.#stopping.delete(owned); + this.#vms.delete(owned); + this.#vmFreed(); })(); return disposed; }, }; this.#vms.add(owned); + const { maxVms } = this.#options; + if (!this.#warnedVms && this.#vms.size >= Math.ceil(maxVms * warnRatio)) { + this.#warnedVms = true; + this.#options.onWarning( + `${this.#vms.size} of ${maxVms} agentOS VMs for dynamic Workers are in use, counting running Workers, prewarmed VMs, and VMs that boot or stop. At the limit, prewarmed VMs stop first, and a new Worker waits for a VM to stop. Raise ${workerLoaderMaxVmsSetting} to allow more.`, + ); + } return owned; } @@ -468,6 +632,7 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { clearTimeout(pool.shrink); for (const waiting of pool.waiting.splice(0)) waiting(undefined); } + for (const waiter of this.#vmWaiters.splice(0)) waiter.fail(new Error("The Worker Loader has stopped.")); // Disposing a VM with a running process fails to observe its exit, so // stop the children first. const children = [...this.#children]; diff --git a/packages/runtime-node/src/bindings/worker-loader/loader.ts b/packages/runtime-node/src/bindings/worker-loader/loader.ts index 5b4306d..e3fac41 100644 --- a/packages/runtime-node/src/bindings/worker-loader/loader.ts +++ b/packages/runtime-node/src/bindings/worker-loader/loader.ts @@ -22,6 +22,7 @@ import { } from "./session.js"; import type { CapnWebStub } from "../durable-objects/rpc/capnweb.js"; import { errorMessage, isRecord } from "../unknown-values.js"; +import { WorkerLoaderMetrics } from "./metrics.js"; export interface WorkerFile { content: string | Uint8Array; @@ -217,32 +218,51 @@ interface LoaderSession { export const maxWorkersSetting = "ORANGE_WORKER_LOADER_MAX_WORKERS"; export const idleTimeoutSetting = "ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS"; +export const queueTimeoutSetting = "ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS"; +export const maxQueuedSetting = "ORANGE_WORKER_LOADER_MAX_QUEUED"; const defaultMaxWorkers = 32; const defaultIdleTimeoutMs = 30_000; +const defaultQueueTimeoutMs = 5_000; +const defaultMaxQueued = 256; const warnRatio = 0.8; +const retryAfterSeconds = 1; -// A new dynamic Worker needs a slot, and every running one holds a slot until it stops. +export type WorkerLoaderLimitCode = "queue_full" | "queue_timeout"; + +// A new dynamic Worker needs a slot, and every running one holds a slot until it stops. With every +// slot busy, a new one waits in line; this error refuses it when the line is full or its wait +// passes the queue timeout. Like the overloaded error of a Cloudflare Durable Object, it carries +// `overloaded` and `retryable`, and the runtime answers it with 503 and Retry-After when the +// Worker does not catch it. +// https://developers.cloudflare.com/durable-objects/best-practices/error-handling/ export class WorkerLoaderLimitError extends RangeError { - constructor(message: string) { + readonly code: WorkerLoaderLimitCode; + readonly overloaded = true; + readonly retryable = true; + readonly retryAfterSeconds = retryAfterSeconds; + + constructor(code: WorkerLoaderLimitCode, message: string) { super(message); this.name = "WorkerLoaderLimitError"; + this.code = code; } } // Node.js runs a setTimeout() delay above this after 1 ms. const maxTimerDelayMs = 2_147_483_647; -function positiveIntegerSetting( +export function positiveIntegerSetting( settings: Readonly>, name: string, fallback: number, max: number, + min = 1, ): number { const raw = settings[name]; if (raw === undefined || raw.trim() === "") return fallback; const value = Number(raw); - if (!Number.isSafeInteger(value) || value < 1 || value > max) { - throw new RangeError(`${name} must be an integer from 1 to ${max}. Got: ${JSON.stringify(raw)}`); + if (!Number.isSafeInteger(value) || value < min || value > max) { + throw new RangeError(`${name} must be an integer from ${min} to ${max}. Got: ${JSON.stringify(raw)}`); } return value; } @@ -251,67 +271,189 @@ export function workerLoaderIdleTimeoutMs(settings: Readonly>): number { + return positiveIntegerSetting(settings, maxWorkersSetting, defaultMaxWorkers, Number.MAX_SAFE_INTEGER); +} + +interface SlotWaiter { + grant(): void; + fail(error: Error): void; +} + +export interface DynamicWorkerPoolLimits { + idleTimeoutMs: number; + maxQueued: number; + maxWorkers: number; + queueTimeoutMs: number; +} + // Bounds the dynamic Workers that run at once across every Worker Loader binding of the runtime. -// Each one is an agentOS VM and a Node.js process, about 54 MiB. At the limit, a new Worker stops -// the least recently used idle one. A Worker with a call in flight is never stopped. +// Each one is an agentOS VM and a Node.js process. At the limit, a new Worker stops the least +// recently used idle one. A Worker with a call in flight is never stopped. With none idle, new +// Workers wait in line, first come first served, until a slot frees or the queue timeout passes. export class DynamicWorkerPool { readonly idleTimeoutMs: number; + readonly maxQueued: number; readonly maxWorkers: number; + readonly queueTimeoutMs: number; + readonly metrics: WorkerLoaderMetrics; // Least recently used first. readonly #live = new Set(); + // First in line first. + readonly #queue = new Map(); readonly #warn: (message: string) => void; - #warned = false; + #pumping = false; + #warnedLive = false; + #warnedQueue = false; - constructor(limits: { idleTimeoutMs: number; maxWorkers: number }, warn: (message: string) => void) { + constructor(limits: DynamicWorkerPoolLimits, warn: (message: string) => void, metrics = new WorkerLoaderMetrics()) { this.idleTimeoutMs = limits.idleTimeoutMs; + this.maxQueued = limits.maxQueued; this.maxWorkers = limits.maxWorkers; + this.queueTimeoutMs = limits.queueTimeoutMs; + this.metrics = metrics; + metrics.slotGauges = () => ({ live: this.#live.size, maxWorkers: this.maxWorkers, queued: this.#queue.size }); this.#warn = warn; } static fromSettings( settings: Readonly>, warn: (message: string) => void, + metrics?: WorkerLoaderMetrics, ): DynamicWorkerPool { return new DynamicWorkerPool( { idleTimeoutMs: workerLoaderIdleTimeoutMs(settings), - maxWorkers: positiveIntegerSetting(settings, maxWorkersSetting, defaultMaxWorkers, Number.MAX_SAFE_INTEGER), + maxQueued: positiveIntegerSetting(settings, maxQueuedSetting, defaultMaxQueued, Number.MAX_SAFE_INTEGER, 0), + maxWorkers: workerLoaderMaxWorkers(settings), + queueTimeoutMs: positiveIntegerSetting(settings, queueTimeoutSetting, defaultQueueTimeoutMs, maxTimerDelayMs), }, warn, + metrics, ); } - admit(worker: DynamicWorker): void { - if (this.#live.size >= this.maxWorkers) { - let evicted = false; - for (const candidate of this.#live) { - if (!candidate.idle) continue; - candidate.evict(); - evicted = true; - break; + get queued(): number { + return this.#queue.size; + } + + // Resolves once `worker` holds a slot. + acquire(worker: DynamicWorker): Promise { + const queuedAt = performance.now(); + return new Promise((resolveSlot, rejectSlot) => { + let timer: NodeJS.Timeout | undefined; + this.#queue.set(worker, { + fail: (error) => { + clearTimeout(timer); + rejectSlot(error); + }, + grant: () => { + clearTimeout(timer); + this.metrics.queueWait.observe((performance.now() - queuedAt) / 1_000); + resolveSlot(); + }, + }); + this.#pump(); + if (!this.#queue.has(worker)) return; + // Only the Workers behind those that got a slot count toward the queue limit. + if (this.#queue.size > this.maxQueued) { + this.#queue.delete(worker); + rejectSlot( + this.#refuse( + "queue_full", + `OpenOrange cannot start another dynamic Worker: all ${this.maxWorkers} slots are busy and ${this.#queue.size} Workers already wait for one, the most ${maxQueuedSetting} allows. Retry later, or raise ${maxWorkersSetting} or ${maxQueuedSetting}.`, + ), + ); + return; } - if (!evicted) { - throw new WorkerLoaderLimitError( - `OpenOrange cannot start another dynamic Worker: ${this.#live.size} are running, none is idle, and the limit is ${this.maxWorkers}. Raise ${maxWorkersSetting} or let calls finish first.`, + if (!this.#warnedQueue && this.#queue.size >= Math.ceil(this.maxQueued * warnRatio)) { + this.#warnedQueue = true; + this.#warn( + `${this.#queue.size} dynamic Workers wait for a slot, and past ${this.maxQueued} new ones fail with WorkerLoaderLimitError. Raise ${maxWorkersSetting} or ${maxQueuedSetting}, or add runners.`, ); } - } - this.#live.add(worker); - if (!this.#warned && this.#live.size >= Math.ceil(this.maxWorkers * warnRatio)) { - this.#warned = true; - this.#warn( - `${this.#live.size} of ${this.maxWorkers} dynamic Workers are running. At the limit, a new one stops the least recently used idle one, or fails with WorkerLoaderLimitError if none is idle. Raise ${maxWorkersSetting} to allow more.`, - ); - } + timer = setTimeout(() => { + if (!this.#queue.delete(worker)) return; + this.#queueShrank(); + rejectSlot( + this.#refuse( + "queue_timeout", + `OpenOrange cannot start another dynamic Worker: it waited ${this.queueTimeoutMs} ms for one of ${this.maxWorkers} busy slots. Retry later, or raise ${maxWorkersSetting} or ${queueTimeoutSetting}.`, + ), + ); + }, this.queueTimeoutMs); + timer.unref(); + }); + } + + // Takes a Worker that stops before it gets a slot out of line. + cancel(worker: DynamicWorker, error: Error): void { + const waiter = this.#queue.get(worker); + if (waiter === undefined) return; + this.#queue.delete(worker); + this.#queueShrank(); + waiter.fail(error); } touch(worker: DynamicWorker): void { if (this.#live.delete(worker)) this.#live.add(worker); } + // A Worker may have become idle, so a Worker in line may take its slot. + idle(): void { + if (this.#queue.size > 0) this.#pump(); + } + remove(worker: DynamicWorker): void { this.#live.delete(worker); - if (this.#live.size < Math.ceil(this.maxWorkers * warnRatio)) this.#warned = false; + if (this.#live.size < Math.ceil(this.maxWorkers * warnRatio)) this.#warnedLive = false; + this.#pump(); + } + + #refuse(code: WorkerLoaderLimitCode, message: string): WorkerLoaderLimitError { + this.metrics.reject(code); + return new WorkerLoaderLimitError(code, message); + } + + #queueShrank(): void { + if (this.#queue.size < Math.ceil(this.maxQueued * warnRatio)) this.#warnedQueue = false; + } + + // Hands free slots to Workers in line, in order. Stopping an idle Worker frees its slot, and + // its removal must not start a second pass. + #pump(): void { + if (this.#pumping) return; + this.#pumping = true; + try { + for (const [worker, waiter] of this.#queue) { + if (!this.#freeSlot()) break; + this.#queue.delete(worker); + this.#live.add(worker); + this.#warnLive(); + waiter.grant(); + } + this.#queueShrank(); + } finally { + this.#pumping = false; + } + } + + #freeSlot(): boolean { + if (this.#live.size < this.maxWorkers) return true; + for (const candidate of this.#live) { + if (!candidate.idle) continue; + candidate.evict(); + return this.#live.size < this.maxWorkers; + } + return false; + } + + #warnLive(): void { + if (this.#warnedLive || this.#live.size < Math.ceil(this.maxWorkers * warnRatio)) return; + this.#warnedLive = true; + this.#warn( + `${this.#live.size} of ${this.maxWorkers} dynamic Workers are running. At the limit, a new one stops the least recently used idle one, or waits up to ${this.queueTimeoutMs} ms for a slot and then fails with WorkerLoaderLimitError. Raise ${maxWorkersSetting} to allow more.`, + ); } } @@ -340,6 +482,7 @@ class DynamicWorker { #baseExports = 0; #evicted = false; #exited = false; + #idleCheck = false; #idleTimer: NodeJS.Timeout | undefined; #inFlight = 0; // The host of a Worker that has loaded and whose first calls are in flight. @@ -371,7 +514,7 @@ class DynamicWorker { return this.#running ?? Promise.resolve(); } - // Stopped by the pool, so a held stub may load the Worker again. + // Stopped by the pool or refused a slot, so a held stub may load the Worker again. get evicted(): boolean { return this.#evicted; } @@ -385,7 +528,16 @@ class DynamicWorker { async #start(): Promise { let host: WorkerHost | undefined; + const { pool } = this.#options; try { + try { + await pool.acquire(this); + } catch (error: unknown) { + // A stub that is still held asks for a slot again on its next call. + if (error instanceof WorkerLoaderLimitError) this.#evicted = true; + throw error; + } + const startedAt = performance.now(); const prepared = await this.#getCode(); host = await this.#options.backend.open(prepared.outbound === "network"); const bundleRoot = posix.join(host.root, "bundle"); @@ -413,6 +565,7 @@ class DynamicWorker { if (this.#terminated) child.kill(); const frames = readFrames(child.stdout, (line) => process.stdout.write(`${line}\n`)); frames.on("message", (message: unknown) => { + this.#checkIdle(); if (typeof message !== "object" || message === null) return; const stdout: unknown = Reflect.get(message, "openOrangeWorkerLoaderStdout"); if (typeof stdout === "string") process.stdout.write(stdout); @@ -424,6 +577,7 @@ class DynamicWorker { } }); const transport = new FramedRpcTransport(frames, (message) => { + this.#checkIdle(); child.write(`${JSON.stringify(message)}\n`); }); const session = openLoaderSession(transport); @@ -441,7 +595,9 @@ class DynamicWorker { this.#baseExports = session.stats().exports; this.#session = session; this.#lastUsed = performance.now(); - this.#armIdleTimer(this.#options.pool.idleTimeoutMs); + pool.metrics.workerStart.observe((this.#lastUsed - startedAt) / 1_000); + this.#armIdleTimer(pool.idleTimeoutMs); + this.#checkIdle(); if (this.#inFlight === 0) host.settled(); else this.#settling = host; return session; @@ -465,6 +621,18 @@ class DynamicWorker { } } + // A message in either direction can end the last call or stream, which leaves the Worker + // idle. Only Workers in line for a slot need to know at once, so the check runs only then, + // at most once per turn of the event loop. + #checkIdle(): void { + if (this.#idleCheck || this.#options.pool.queued === 0) return; + this.#idleCheck = true; + setImmediate(() => { + this.#idleCheck = false; + if (this.idle) this.#options.pool.idle(); + }); + } + // Like workerd, an isolate that abortIsolate() ended fails its calls with an internal error, // and the reason goes only to the log under the same reference. #exitError(reason: string): Error { @@ -503,14 +671,19 @@ class DynamicWorker { this.#settling = undefined; } if (this.#orphaned && this.idle) this.evict(); + else this.#checkIdle(); }; void call.then(settle, settle); return call; } - // No stub references this unnamed Worker any more, so it stops once idle. + // No stub references this unnamed Worker any more, so it stops once idle. A Worker in line for + // a slot leaves the line, unless a call waits on it: the call holds no stub. release(): void { this.#orphaned = true; + if (this.#inFlight === 0) { + this.#options.pool.cancel(this, new Error(`Dynamic Worker ${this.#label} has no stub left.`)); + } if (this.idle) this.evict(); } @@ -546,6 +719,7 @@ class DynamicWorker { capability.dispose(); this.#entrypointImports -= 1; if (this.#orphaned && this.idle) this.evict(); + else this.#checkIdle(); }, () => undefined, ); @@ -553,6 +727,7 @@ class DynamicWorker { terminate(): void { this.#terminated = true; + this.#options.pool.cancel(this, new Error("The Worker Loader has stopped.")); if (this.#child !== undefined && !this.#exited) this.#child.kill(); } } @@ -665,7 +840,8 @@ export class WorkerStub { constructor(handle: WorkerHandle) { this.#handle = handle; - // The Worker starts loading at once, and a full pool throws here. + // The Worker starts loading at once. With every slot busy, it waits in line, and its calls + // wait with it. handle.acquire(); handle.hold(this); } @@ -775,7 +951,6 @@ export class WorkerLoader { onExit(); }, ); - this.#options.pool.admit(worker); this.#all.add(worker); worker.start(); return worker; diff --git a/packages/runtime-node/src/bindings/worker-loader/metrics.ts b/packages/runtime-node/src/bindings/worker-loader/metrics.ts new file mode 100644 index 0000000..2ccb059 --- /dev/null +++ b/packages/runtime-node/src/bindings/worker-loader/metrics.ts @@ -0,0 +1,175 @@ +// Prometheus metrics for dynamic Workers. The runtime appends them to the runner's +// metrics endpoint, so an autoscaler can scale on slot use and queue wait. + +import { readdirSync, readFileSync } from "node:fs"; + +// Seconds. Covers a warm claim (about 10 ms) up to the default queue timeout. +const secondsBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30]; + +class Histogram { + readonly #counts = secondsBuckets.map(() => 0); + #count = 0; + #sum = 0; + + observe(seconds: number): void { + this.#count += 1; + this.#sum += seconds; + for (const [index, bound] of secondsBuckets.entries()) { + if (seconds <= bound) this.#counts[index] = (this.#counts[index] ?? 0) + 1; + } + } + + render(name: string, labels = ""): string[] { + const join = labels === "" ? "" : `${labels},`; + const lines = secondsBuckets.map((bound, index) => `${name}_bucket{${join}le="${bound}"} ${this.#counts[index] ?? 0}`); + lines.push(`${name}_bucket{${join}le="+Inf"} ${this.#count}`); + const suffix = labels === "" ? "" : `{${labels}}`; + lines.push(`${name}_sum${suffix} ${this.#sum}`, `${name}_count${suffix} ${this.#count}`); + return lines; + } +} + +export interface SlotGauges { + live: number; + maxWorkers: number; + queued: number; +} + +export interface VmGauges { + vms: { booting: number; live: number; pooled: number; stopping: number }; + maxVms: number; + poolReady: Record; + // Loads that wait for room in the VM budget. + waiting: number; +} + +// One instance per runtime. The slot pool and the agentOS backend record into it. +export class WorkerLoaderMetrics { + readonly queueWait = new Histogram(); + readonly workerStart = new Histogram(); + readonly vmBoot = new Histogram(); + readonly #claims = new Map(); + readonly #rejections = new Map(); + slotGauges: (() => SlotGauges) | undefined; + vmGauges: (() => VmGauges) | undefined; + + // `warming` waited for a prewarm in flight, and `miss` booted a VM on demand. + claim(mode: string, result: "hit" | "miss" | "warming"): void { + const key = `mode="${mode}",result="${result}"`; + this.#claims.set(key, (this.#claims.get(key) ?? 0) + 1); + } + + reject(code: string): void { + this.#rejections.set(code, (this.#rejections.get(code) ?? 0) + 1); + } + + render(): string { + const out: string[] = []; + const metric = (name: string, type: string, help: string): void => { + out.push(`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`); + }; + const slots = this.slotGauges?.(); + if (slots !== undefined) { + metric("orange_worker_loader_workers", "gauge", "Dynamic Workers that hold a slot."); + out.push(`orange_worker_loader_workers ${slots.live}`); + metric("orange_worker_loader_workers_max", "gauge", "Slots: ORANGE_WORKER_LOADER_MAX_WORKERS."); + out.push(`orange_worker_loader_workers_max ${slots.maxWorkers}`); + metric("orange_worker_loader_slot_utilization", "gauge", "Slots in use divided by slots."); + out.push(`orange_worker_loader_slot_utilization ${slots.live / slots.maxWorkers}`); + metric("orange_worker_loader_queue_depth", "gauge", "Dynamic Workers that wait for a slot."); + out.push(`orange_worker_loader_queue_depth ${slots.queued}`); + } + const gauges = this.vmGauges?.(); + if (gauges !== undefined) { + metric("orange_worker_loader_vms", "gauge", "agentOS VMs of dynamic Workers, by state."); + for (const [state, count] of Object.entries(gauges.vms)) { + out.push(`orange_worker_loader_vms{state="${state}"} ${count}`); + } + metric("orange_worker_loader_vms_max", "gauge", "VM budget: ORANGE_WORKER_LOADER_MAX_VMS."); + out.push(`orange_worker_loader_vms_max ${gauges.maxVms}`); + metric("orange_worker_loader_vm_queue_depth", "gauge", "Dynamic Workers with a slot that wait for room in the VM budget."); + out.push(`orange_worker_loader_vm_queue_depth ${gauges.waiting}`); + metric("orange_worker_loader_pool_ready", "gauge", "Prewarmed VMs ready to claim, by network mode."); + for (const [mode, count] of Object.entries(gauges.poolReady)) { + out.push(`orange_worker_loader_pool_ready{mode="${mode}"} ${count}`); + } + } + metric("orange_worker_loader_queue_wait_seconds", "histogram", "Time a dynamic Worker waited for a slot."); + out.push(...this.queueWait.render("orange_worker_loader_queue_wait_seconds")); + metric("orange_worker_loader_start_seconds", "histogram", "Time from a slot to a loaded dynamic Worker."); + out.push(...this.workerStart.render("orange_worker_loader_start_seconds")); + metric("orange_worker_loader_vm_boot_seconds", "histogram", "Time to boot a VM and start its child process."); + out.push(...this.vmBoot.render("orange_worker_loader_vm_boot_seconds")); + metric("orange_worker_loader_pool_claims_total", "counter", "VM claims, by network mode and pool hit or miss."); + for (const [labels, count] of this.#claims) out.push(`orange_worker_loader_pool_claims_total{${labels}} ${count}`); + metric("orange_worker_loader_rejections_total", "counter", "Dynamic Workers refused with WorkerLoaderLimitError, by code."); + for (const [code, count] of this.#rejections) out.push(`orange_worker_loader_rejections_total{code="${code}"} ${count}`); + metric("orange_runtime_resident_memory_bytes", "gauge", "Resident memory of the runner and of its child processes, such as the agentOS sidecar."); + const memory = residentMemory(); + out.push(`orange_runtime_resident_memory_bytes{process="runner"} ${memory.runner}`); + if (memory.children !== undefined) { + out.push(`orange_runtime_resident_memory_bytes{process="children"} ${memory.children}`); + } + const limit = memoryLimitBytes(); + if (limit !== undefined) { + metric("orange_runtime_memory_limit_bytes", "gauge", "The cgroup memory limit of the runner."); + out.push(`orange_runtime_memory_limit_bytes ${limit}`); + } + return `${out.join("\n")}\n`; + } +} + +// Linux only: the children's RSS comes from /proc, which other systems lack. +function residentMemory(): { children: number | undefined; runner: number } { + const runner = process.memoryUsage.rss(); + if (process.platform !== "linux") return { children: undefined, runner }; + let children = 0; + const pending = [process.pid]; + const seen = new Set(); + while (pending.length > 0) { + const pid = pending.pop(); + if (pid === undefined || seen.has(pid)) continue; + seen.add(pid); + if (pid !== process.pid) children += rssOf(pid); + for (const child of childrenOf(pid)) pending.push(child); + } + return { children, runner }; +} + +// A process can exit while it is read, so a missing file counts as nothing. +function readProc(path: string): string { + try { + return readFileSync(path, "utf8"); + } catch { + return ""; + } +} + +function childrenOf(pid: number): number[] { + let tasks: string[]; + try { + tasks = readdirSync(`/proc/${pid}/task`); + } catch { + return []; + } + return tasks.flatMap((task) => + readProc(`/proc/${pid}/task/${task}/children`) + .split(" ") + .filter((value) => value !== "") + .map(Number), + ); +} + +function rssOf(pid: number): number { + const match = /^VmRSS:\s+(\d+) kB/m.exec(readProc(`/proc/${pid}/status`)); + return match?.[1] === undefined ? 0 : Number(match[1]) * 1024; +} + +function memoryLimitBytes(): number | undefined { + if (process.platform !== "linux") return undefined; + const cgroup = /^0::(.*)$/m.exec(readProc("/proc/self/cgroup"))?.[1]; + if (cgroup === undefined) return undefined; + const raw = readProc(`/sys/fs/cgroup${cgroup}/memory.max`).trim(); + const value = Number(raw); + return raw === "" || raw === "max" || !Number.isFinite(value) ? undefined : value; +} diff --git a/packages/runtime-node/src/runner.ts b/packages/runtime-node/src/runner.ts index 6405e52..ec8b91e 100644 --- a/packages/runtime-node/src/runner.ts +++ b/packages/runtime-node/src/runner.ts @@ -55,14 +55,21 @@ import { type WorkflowEngineObject, workflowEngineClassName, } from "./bindings/workflows/engine-object.js"; -import { AgentOsWorkerLoaderBackend, workerLoaderPoolLimits } from "./bindings/worker-loader/agentos.js"; +import { + AgentOsWorkerLoaderBackend, + workerLoaderMaxVms, + workerLoaderPoolLimits, +} from "./bindings/worker-loader/agentos.js"; import { DynamicWorkerPool, WorkerLoader, + WorkerLoaderLimitError, workerLoaderIdleTimeoutMs, + workerLoaderMaxWorkers, workerLoaderVmBundlePath, type WorkerLoaderBackend, } from "./bindings/worker-loader/loader.js"; +import { WorkerLoaderMetrics } from "./bindings/worker-loader/metrics.js"; import { createDispatchNamespace } from "./bindings/dispatch-namespace.js"; import type { EnvironmentBinding } from "./bindings/create-env.js"; import type { ConfiguredDurableObjectClass } from "./bindings/durable-objects/classes.js"; @@ -219,16 +226,24 @@ function requestBody( } // Dynamic Workers run only in agentOS VMs. All Worker Loader bindings share one -// backend. +// backend, one VM budget, and one set of metrics. +const workerLoaderMetrics = new WorkerLoaderMetrics(); let sharedWorkerLoaderBackend: WorkerLoaderBackend | undefined; function workerLoaderBackend(): WorkerLoaderBackend { + const settings = runtimeSettings(); + const pool = workerLoaderPoolLimits(settings); sharedWorkerLoaderBackend ??= new AgentOsWorkerLoaderBackend({ bundlePath: () => workerLoaderVmBundlePath(import.meta.url), + maxVms: workerLoaderMaxVms(settings, workerLoaderMaxWorkers(settings), pool), + metrics: workerLoaderMetrics, onDiagnostic(message): void { send({ message, type: "background-error" }); }, + onWarning(message): void { + send({ message, type: "warning" }); + }, // The pool keeps VMs above its idle size as long as a Worker stays idle before it stops. - pool: { ...workerLoaderPoolLimits(runtimeSettings()), idleMs: workerLoaderIdleTimeoutMs(runtimeSettings()) }, + pool: { ...pool, idleMs: workerLoaderIdleTimeoutMs(settings) }, }); return sharedWorkerLoaderBackend; } @@ -236,10 +251,17 @@ function workerLoaderBackend(): WorkerLoaderBackend { // One pool bounds the dynamic Workers of every Worker Loader binding. let sharedWorkerLoaderPool: DynamicWorkerPool | undefined; function workerLoaderPool(warn: (message: string) => void): DynamicWorkerPool { - sharedWorkerLoaderPool ??= DynamicWorkerPool.fromSettings(runtimeSettings(), warn); + sharedWorkerLoaderPool ??= DynamicWorkerPool.fromSettings(runtimeSettings(), warn, workerLoaderMetrics); return sharedWorkerLoaderPool; } +// A Worker that does not catch a WorkerLoaderLimitError is out of dynamic Worker capacity, not +// broken, so the client gets 503 and when to retry. Cloudflare answers a Worker that cannot run +// for platform limits with 503 too. +function capacityFailure(error: unknown): { retryAfterSeconds: number } | undefined { + return error instanceof WorkerLoaderLimitError ? { retryAfterSeconds: error.retryAfterSeconds } : undefined; +} + function requestHeaders(request: IncomingMessage): Headers { const headers = new Headers(); for (const [name, value] of Object.entries(request.headers)) { @@ -1207,14 +1229,19 @@ export async function startRuntime( reportWaitUntil(dispatch.waitUntil); } catch (error: unknown) { send({ ...errorDetails(error), type: "request-error" }); - if (!outgoing.headersSent) { - outgoing.statusCode = 500; - outgoing.setHeader("content-type", "text/plain; charset=utf-8"); - } if (outgoing.headersSent) { outgoing.destroy(error instanceof Error ? error : undefined); - } else { + return; + } + outgoing.setHeader("content-type", "text/plain; charset=utf-8"); + const capacity = capacityFailure(error); + if (capacity === undefined) { + outgoing.statusCode = 500; outgoing.end("Internal Server Error"); + } else { + outgoing.statusCode = 503; + outgoing.setHeader("retry-after", String(capacity.retryAfterSeconds)); + outgoing.end("Service Unavailable"); } } })(); @@ -1259,8 +1286,20 @@ export async function startRuntime( } catch (error: unknown) { if (workerSocket !== null) closeUncoupledWorkerWebSocket(workerSocket); send({ ...errorDetails(error), type: "request-error" }); - writeUpgradeFailure(socket); - reportRequest(incoming, 500, startedAt); + const capacity = capacityFailure(error); + if (capacity === undefined) { + writeUpgradeFailure(socket); + reportRequest(incoming, 500, startedAt); + } else { + await writeRejectedUpgrade( + socket, + new Response("Service Unavailable", { + headers: { "retry-after": String(capacity.retryAfterSeconds) }, + status: 503, + }), + ); + reportRequest(incoming, 503, startedAt); + } } })(); }); @@ -1276,16 +1315,26 @@ export async function startRuntime( await closeRegistry(); throw error; } - // A Rivet cluster scrapes the runner's metrics. A local Engine has no scraper. - const metricsPort = externalEngine ? metricsPortFromEnvironment(process.env) : undefined; + // A Rivet cluster scrapes the runner's metrics. `orange dev` serves them too when the port is set. + const metricsPort = metricsPortFromEnvironment(process.env); const metricsServer = metricsPort === undefined ? undefined : await startMetricsServer(metricsPort, async () => { + const loaderMetrics = runtimeBindings.workerLoaders.length > 0 ? workerLoaderMetrics.render() : ""; if (durableObjectRegistry === undefined) { + if (loaderMetrics !== "") { + return new Response(loaderMetrics, { headers: { "content-type": "text/plain; version=0.0.4" } }); + } return new Response("registry not started\n", { status: 503 }); } - return await durableObjectRegistry.prometheusMetrics(); + const rivet = await durableObjectRegistry.prometheusMetrics(); + if (loaderMetrics === "" || !rivet.ok) return rivet; + // Only the content type carries over: the body is longer than RivetKit's. + return new Response(`${await rivet.text()}${loaderMetrics}`, { + headers: { "content-type": rivet.headers.get("content-type") ?? "text/plain; version=0.0.4" }, + status: rivet.status, + }); }); // The cron actor runs in this process. const cronSync = new AbortController(); diff --git a/src/commands/d1.ts b/src/commands/d1.ts index ac738b7..b4be930 100644 --- a/src/commands/d1.ts +++ b/src/commands/d1.ts @@ -194,6 +194,8 @@ async function openDatabase( deploymentIdentity, host: "127.0.0.1", normalizedConfiguration: "{}", + // A one-shot D1 command has nothing to scrape. + serveMetrics: false, port: 0, projectRoot: config.projectRoot, ...(endpoint === undefined diff --git a/src/commands/dev.ts b/src/commands/dev.ts index 667889b..4466035 100644 --- a/src/commands/dev.ts +++ b/src/commands/dev.ts @@ -432,6 +432,8 @@ async function executeDevCommand( ...(index === 0 ? { onRequest: (request) => reporter.request(request) } : {}), port: index === 0 ? parsed.port : 0, projectRoot: config.projectRoot, + // One port can have one metrics server. + serveMetrics: index === 0, queueConsumers: config.queueConsumers, tailConsumers: config.tailConsumers, ...(config.workerName === undefined ? {} : { workerName: config.workerName }), diff --git a/tests/bindings/worker-loader.test.ts b/tests/bindings/worker-loader.test.ts index bf548a2..12eb08d 100644 --- a/tests/bindings/worker-loader.test.ts +++ b/tests/bindings/worker-loader.test.ts @@ -2,12 +2,13 @@ import assert from "node:assert/strict"; import { glob, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { builtinModules } from "node:module"; +import { createServer, type AddressInfo } from "node:net"; import { dirname, join } from "node:path"; import test from "node:test"; import { build, type Plugin } from "esbuild"; -import { copyProject, isRecord, repositoryRoot, startOrangeDev, withOrangeDev } from "../harness/orange-dev.js"; +import { copyProject, isRecord, repositoryRoot, startOrangeDev, waitFor, withOrangeDev } from "../harness/orange-dev.js"; const fixtureRoot = join(repositoryRoot, "tests", "fixtures", "worker-loader"); @@ -254,10 +255,7 @@ test("stops idle dynamic Workers without interrupting calls in flight", async () ORANGE_WORKER_LOADER_MAX_WORKERS: "2", }; await withOrangeDev("worker-loader", async (dev) => { - const result = await dev.json("/eviction"); - assert.ok(isRecord(result), JSON.stringify(result)); - const { full, ...rest } = result; - assert.deepEqual(rest, { + assert.deepEqual(await dev.json("/eviction"), { // After the idle timeout, get() with the same name calls getCode again and // the Worker starts with fresh module state. named: [1, 2, 1], @@ -266,20 +264,17 @@ test("stops idle dynamic Workers without interrupting calls in flight", async () // call loads the same code again. heldCounts: [1, 1], // Calls that outlast the idle timeout finish, and a full pool of busy - // Workers does not stop them. + // Workers does not stop them. A Worker loaded meanwhile waits for a slot. slept: [{ value: 3_000 }, { value: 3_000 }], + full: { value: 1 }, // Idle Workers give up their slots, so more load() calls than the limit // all succeed. sequential: [1, 1, 1, 1, 1, 1], - // With both Workers handling a call made in the same tick, load() throws - // and both calls finish. - sameTick: "WorkerLoaderLimitError", + // With both Workers handling a call made in the same tick, a third Worker + // waits for one of them to finish, and both calls finish. + sameTick: { value: 1 }, sameTickCalls: [{ value: 2 }, { value: 2 }], }); - // With every slot busy, load() throws the typed error that names the limit. - assert.ok(isRecord(full) && isRecord(full.error), JSON.stringify(full)); - assert.equal(full.error.name, "WorkerLoaderLimitError"); - assert.match(String(full.error.message), /the limit is 2\. Raise ORANGE_WORKER_LOADER_MAX_WORKERS/); }, { environment }); }); @@ -312,6 +307,135 @@ test("keeps a dynamic Worker running until every body it streams ends, and frees }, { environment }); }); +async function freePort(): Promise { + const server = createServer(); + await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); + const { port } = server.address() as AddressInfo; + await new Promise((resolveClose) => server.close(resolveClose)); + return port; +} + +// Reads the runner's Prometheus metrics into `name{labels}` -> value. +async function scrape(port: number): Promise> { + const response = await fetch(`http://127.0.0.1:${port}/metrics`); + assert.equal(response.status, 200); + const values = new Map(); + for (const line of (await response.text()).split("\n")) { + if (line === "" || line.startsWith("#")) continue; + const space = line.lastIndexOf(" "); + values.set(line.slice(0, space), Number(line.slice(space + 1))); + } + return values; +} + +test("runs a burst larger than the slot limit by queueing it, across both network modes", async () => { + const environment = { + // No Worker stops for its idle timeout, so each slot frees only because its Worker is idle. + ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS: "60000", + ORANGE_WORKER_LOADER_MAX_WORKERS: "2", + // Long enough for a slow machine to boot every Worker of the burst in turn. + ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS: "30000", + }; + await withOrangeDev("worker-loader", async (dev) => { + assert.deepEqual(await dev.json("/queued?n=8&ms=100"), Array.from({ length: 8 }, () => ({ value: 100 }))); + }, { environment }); +}); + +test("refuses a dynamic Worker past the queue timeout or the queue limit with 503 and Retry-After, and reports it in metrics", async () => { + const metricsPort = await freePort(); + const environment = { + ORANGE_WORKER_LOADER_MAX_QUEUED: "1", + ORANGE_WORKER_LOADER_MAX_WORKERS: "1", + ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS: "1500", + RIVETKIT_METRICS_PORT: String(metricsPort), + }; + await withOrangeDev("worker-loader", async (dev) => { + const url = dev.origin; + const metric = async (name: string): Promise => (await scrape(metricsPort)).get(name); + // One Worker holds the only slot for 10 s. + const busy = fetch(`${url}/one?ms=10000`); + await waitFor("the busy Worker holds the slot", async () => (await metric("orange_worker_loader_workers")) === 1, 15_000); + assert.equal(await metric("orange_worker_loader_slot_utilization"), 1); + + // The next Worker waits in line, and the one after it finds the line full. + const timedOut = fetch(`${url}/one`); + await waitFor("a Worker waits for a slot", async () => (await metric("orange_worker_loader_queue_depth")) === 1, 15_000); + const full = await fetch(`${url}/one`); + assert.equal(full.status, 503); + assert.equal(full.headers.get("retry-after"), "1"); + const late = await timedOut; + assert.equal(late.status, 503, await late.clone().text()); + assert.equal(late.headers.get("retry-after"), "1"); + + // A Worker that catches the refusal sees a typed, retryable error. + assert.deepEqual(await (await fetch(`${url}/caught`)).json(), { + error: { code: "queue_timeout", name: "WorkerLoaderLimitError", overloaded: true, retryable: true }, + }); + const metrics = await scrape(metricsPort); + assert.equal(metrics.get('orange_worker_loader_rejections_total{code="queue_full"}'), 1); + assert.equal(metrics.get('orange_worker_loader_rejections_total{code="queue_timeout"}'), 2); + assert.equal(metrics.get("orange_worker_loader_queue_depth"), 0); + + // Once the busy Worker finishes, a retry loads. + assert.equal((await busy).status, 200); + await dev.json("/one"); + }, { environment }); +}); + +test("keeps every agentOS VM, running, prewarmed, or stopping, within the VM budget while Workers churn", async () => { + const metricsPort = await freePort(); + const environment = { + ORANGE_WORKER_LOADER_MAX_VMS: "3", + ORANGE_WORKER_LOADER_MAX_WORKERS: "2", + ORANGE_WORKER_LOADER_POOL_MAX: "2", + ORANGE_WORKER_LOADER_POOL_SIZE: "2", + ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS: "60000", + RIVETKIT_METRICS_PORT: String(metricsPort), + }; + await withOrangeDev("worker-loader", async (dev) => { + let most = 0; + let sampling = true; + const sampler = (async () => { + while (sampling) { + const metrics = await scrape(metricsPort); + let vms = 0; + for (const [name, value] of metrics) if (name.startsWith("orange_worker_loader_vms{")) vms += value; + assert.equal(metrics.get("orange_worker_loader_vms_max"), 3); + most = Math.max(most, vms); + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } + })(); + // Each round's Workers stop the last round's idle ones, in both network modes, so VMs boot + // and stop beside each other while both pools refill. + for (let round = 0; round < 4; round++) { + assert.deepEqual(await dev.json("/queued?n=6&ms=20"), Array.from({ length: 6 }, () => ({ value: 20 }))); + } + sampling = false; + await sampler; + assert.ok(most > 0 && most <= 3, `${most} VMs at once`); + }, { environment }); +}); + +test("refills the VM pool while a steady stream of loads keeps Workers loading", async () => { + const environment = { + ORANGE_WORKER_LOADER_POOL_MAX: "4", + ORANGE_WORKER_LOADER_POOL_SIZE: "1", + }; + await withOrangeDev("worker-loader", async (dev) => { + // Arrivals every 50 ms keep at least one Worker loading at any time. + const results = await Promise.all( + Array.from({ length: 80 }, async (_, i) => { + await new Promise((resolveWait) => setTimeout(resolveWait, i * 50)); + return (await dev.json("/prewarmed")) === true; + }), + ); + // After the first second, most loads find a prewarmed VM. + const steady = results.slice(20); + const prewarmed = steady.filter(Boolean).length; + assert.ok(prewarmed >= steady.length / 2, `${prewarmed} of ${steady.length} loads found a prewarmed VM`); + }, { environment }); +}); + test("refuses to start with an idle timeout that Node.js timers cannot hold", async () => { const project = await copyProject(fixtureRoot); try { diff --git a/tests/fixtures/worker-loader/worker.js b/tests/fixtures/worker-loader/worker.js index b5bd2e9..6e320cd 100644 --- a/tests/fixtures/worker-loader/worker.js +++ b/tests/fixtures/worker-loader/worker.js @@ -225,6 +225,27 @@ async function settle(promise) { } } +const prewarmedCode = { + compatibilityDate: "2026-09-03", + mainModule: "main.js", + globalOutbound: null, + modules: { + "main.js": ` + import { createRequire } from "node:module"; + const fs = createRequire(import.meta.url)("fs"); + export default { prewarmed(requestedAt) { return fs.statSync(process.cwd()).birthtimeMs < requestedAt; } }; + `, + }, +}; + +const sleepCode = { + compatibilityDate: "2026-09-03", + mainModule: "main.js", + modules: { + "main.js": `export default { async sleep(ms) { await new Promise((r) => setTimeout(r, ms)); return ms; } };`, + }, +}; + const routes = { async basic(env, ctx) { const worker = env.LOADER.get("basic", () => ({ @@ -476,6 +497,37 @@ const routes = { } return { received: "timed out", started }; }, + // Starts `n` named Workers at once whose calls each take `ms`, half of them with network access. + // The garbage collector never stops a named Worker, so only idle Workers free their slots. + async queued(env, _ctx, request) { + const url = new URL(request.url); + const n = Number(url.searchParams.get("n")); + const ms = Number(url.searchParams.get("ms")); + const tag = crypto.randomUUID(); + return Promise.all( + Array.from({ length: n }, (_, i) => { + const code = { ...sleepCode, globalOutbound: i % 2 === 0 ? null : undefined }; + return settle(env.LOADER.get(`${tag}-${i}`, () => code).getEntrypoint().sleep(ms)); + }), + ); + }, + // Whether a new Worker's VM had its Worker root before the load, as a prewarmed VM does. + async prewarmed(env) { + return env.LOADER.load(prewarmedCode).getEntrypoint().prewarmed(Date.now()); + }, + // One Worker and one call. Its errors reach the runtime uncaught. + async one(env, _ctx, request) { + const ms = Number(new URL(request.url).searchParams.get("ms") ?? 0); + return env.LOADER.load(sleepCode).getEntrypoint().sleep(ms); + }, + // One Worker and one call, with the fields of its error. + async caught(env) { + try { + return { value: await env.LOADER.load(sleepCode).getEntrypoint().sleep(0) }; + } catch (error) { + return { error: { code: error.code, name: error.name, overloaded: error.overloaded, retryable: error.retryable } }; + } + }, // Runs with ORANGE_WORKER_LOADER_MAX_WORKERS=2: more failed loads than the limit, then a load. async failedStarts(env) { const errors = []; @@ -505,19 +557,14 @@ const routes = { named.push(await env.LOADER.get("idle", getCode).getEntrypoint().increment()); heldCounts.push(await held.increment()); - const slowCode = { - compatibilityDate: "2026-09-03", - mainModule: "main.js", - modules: { - "main.js": `export default { async sleep(ms) { await new Promise((r) => setTimeout(r, ms)); return ms; } };`, - }, - }; const slow = [ - env.LOADER.load(slowCode).getEntrypoint().sleep(3_000), - env.LOADER.load(slowCode).getEntrypoint().sleep(3_000), + env.LOADER.load(sleepCode).getEntrypoint().sleep(3_000), + env.LOADER.load(sleepCode).getEntrypoint().sleep(3_000), ]; - const full = await settle(Promise.resolve().then(() => env.LOADER.load(counterCode))); + // Every slot is busy, so this Worker waits for one. + const waiting = settle(env.LOADER.load(counterCode).getEntrypoint().increment()); const slept = await Promise.all(slow.map(settle)); + const full = await waiting; const sequential = []; for (let i = 0; i < 6; i++) { @@ -528,14 +575,9 @@ const routes = { const b = env.LOADER.load(counterCode).getEntrypoint(); await Promise.all([a.increment(), b.increment()]); const calls = [a.increment(), b.increment()]; - let sameTick; - try { - env.LOADER.load(counterCode); - sameTick = "loaded"; - } catch (error) { - sameTick = error.name; - } + const third = settle(env.LOADER.load(counterCode).getEntrypoint().increment()); const sameTickCalls = await Promise.all(calls.map(settle)); + const sameTick = await third; return { full, heldCounts, loads, named, sameTick, sameTickCalls, sequential, slept }; }, async syntax(env) { From 06bae9086bd472c154462070dcb4ab6bf712c08b Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sun, 15 Nov 2026 04:09:13 +0400 Subject: [PATCH 5/7] perf(worker-loader): pipeline a new dynamic Worker's first call and batch its frames - Send entrypoint calls right behind `start` with Cap'n Web pipelining; a Worker that fails to load still fails them with its startup error - Batch the parent's stdin frames per event-loop turn and the child's stdout frames per microtask checkpoint - Test: calls made while a Worker loads, beside a 3 MiB body, each get their own result --- .../src/bindings/worker-loader/agentos.ts | 43 +++++++++--- .../src/bindings/worker-loader/child.ts | 70 +++++++++++++++---- .../src/bindings/worker-loader/loader.ts | 33 ++++++--- .../src/bindings/worker-loader/session.ts | 17 ++++- tests/bindings/worker-loader.test.ts | 4 ++ tests/fixtures/worker-loader/worker.js | 28 ++++++++ 6 files changed, 161 insertions(+), 34 deletions(-) diff --git a/packages/runtime-node/src/bindings/worker-loader/agentos.ts b/packages/runtime-node/src/bindings/worker-loader/agentos.ts index 306b9ca..40005e7 100644 --- a/packages/runtime-node/src/bindings/worker-loader/agentos.ts +++ b/packages/runtime-node/src/bindings/worker-loader/agentos.ts @@ -28,6 +28,8 @@ import type { VmGauges, WorkerLoaderMetrics } from "./metrics.js"; const runtimeRoot = "/open-orange/runtime"; const workersRoot = "/open-orange/workers"; const exitWaitMs = 5_000; +// The most stdin bytes sent to a dynamic Worker process in one agentOS write. +const maxStdinWriteBytes = 1024 * 1024; type NetworkMode = "deny" | "network"; @@ -693,8 +695,33 @@ async function spawnVmChild(vm: AgentOs, root: string, options: VmChildOptions): stdout.end(); for (const listener of exitListeners.splice(0)) listener(exitReason ?? "unknown"); }); - // The VM takes stdin writes in order only if each waits for the last. - let writes = Promise.resolve(); + // The VM takes stdin writes in order only if each waits for the last. Frames written in one + // turn of the event loop, or while the last write is in flight, go in one write of at most + // `maxStdinWriteBytes`. + let unsent: string[] = []; + let writing = false; + const writeUnsent = async (): Promise => { + while (unsent.length > 0 && exitReason === undefined) { + let bytes = 0; + let count = 0; + for (const frame of unsent) { + const frameBytes = Buffer.byteLength(frame); + if (count > 0 && bytes + frameBytes > maxStdinWriteBytes) break; + bytes += frameBytes; + count += 1; + } + try { + await vm.process.writeStdin(pid, unsent.splice(0, count).join("")); + } catch (error: unknown) { + // Writes fail once the process exits, and its exit fails its calls. + if (exitReason === undefined && !killing) { + options.report(`OpenOrange cannot write to dynamic Worker process ${pid}: ${errorMessage(error)}`); + } + } + } + unsent = []; + writing = false; + }; return { exited: done, kill(): void { @@ -720,14 +747,10 @@ async function spawnVmChild(vm: AgentOs, root: string, options: VmChildOptions): stdout, write(text): void { if (exitReason !== undefined) return; - writes = writes - .then(() => vm.process.writeStdin(pid, text)) - .catch((error: unknown) => { - // Writes fail once the process exits, and its exit fails its calls. - if (exitReason === undefined && !killing) { - options.report(`OpenOrange cannot write to dynamic Worker process ${pid}: ${errorMessage(error)}`); - } - }); + unsent.push(text); + if (writing) return; + writing = true; + setImmediate(() => void writeUnsent()); }, }; } diff --git a/packages/runtime-node/src/bindings/worker-loader/child.ts b/packages/runtime-node/src/bindings/worker-loader/child.ts index 439ae92..3eb6fbb 100644 --- a/packages/runtime-node/src/bindings/worker-loader/child.ts +++ b/packages/runtime-node/src/bindings/worker-loader/child.ts @@ -92,11 +92,20 @@ function startupError(error: unknown, start: DynamicWorkerStart | undefined): un class DynamicWorkerHost extends DurableObjectRpcTarget { #worker: LoadedWorkerEntrypoint | undefined; + // The first `start`. The parent pipelines its entrypoint calls behind it. + #started: Promise | undefined; - async start(envelope: unknown): Promise { + start(envelope: unknown): Promise { + if (this.#started !== undefined) { + return Promise.resolve(encodeLoaderError(new Error("Dynamic Worker already started"))); + } + this.#started = this.#load(envelope); + return this.#started; + } + + async #load(envelope: unknown): Promise { let start: DynamicWorkerStart | undefined; try { - if (this.#worker !== undefined) throw new Error("Dynamic Worker already started"); start = parseDynamicWorkerStart(decodeLoaderValue(envelope, true)); const { bundleRoot, mainFile, moduleRoot, outbound } = start; // node:fs reads `/bundle` from the dynamic Worker's own modules. @@ -122,8 +131,11 @@ class DynamicWorkerHost extends DurableObjectRpcTarget { } } - entrypoint(name: unknown, props: unknown): LoaderCapabilityTarget { - if (this.#worker === undefined) throw new Error("Dynamic Worker is not started"); + async entrypoint(name: unknown, props: unknown): Promise { + if (this.#started === undefined) throw new Error("Dynamic Worker is not started"); + await this.#started; + // The parent fails this call with the startup error. + if (this.#worker === undefined) throw new Error("Dynamic Worker failed to start"); if (name !== undefined && typeof name !== "string") { throw new TypeError("Entrypoint name must be a string"); } @@ -132,13 +144,34 @@ class DynamicWorkerHost extends DurableObjectRpcTarget { } } +interface FrameWriter { + send(message: object): void; + // Returns the frames not yet written and forgets them. + take(): string; +} + // Stdout carries the frames to the parent, so the Worker's own stdout writes -// travel as frames too. The parent writes them to its stdout. Returns the -// writer for every other frame. -function frameStdoutWrites(): (message: object) => void { +// travel as frames too. The parent writes them to its stdout. Frames sent in +// one microtask checkpoint leave in one write, so the messages of a call reach +// the parent together. Returns the writer for every other frame. +function frameStdoutWrites(): FrameWriter { const stdout = process.stdout; - const frameStream = { write: stdout.write.bind(stdout) }; - const sendFrame = (message: object): void => writeFrame(frameStream, message); + const write = stdout.write.bind(stdout); + let pending = ""; + const buffer = { + write(text: string): boolean { + if (pending === "") { + queueMicrotask(() => { + const frames = pending; + pending = ""; + if (frames !== "") write(frames); + }); + } + pending += text; + return true; + }, + }; + const sendFrame = (message: object): void => writeFrame(buffer, message); stdout.write = ( chunk: string | Uint8Array, encoding?: BufferEncoding | ((error?: Error | null) => void), @@ -153,11 +186,19 @@ function frameStdoutWrites(): (message: object) => void { if (done !== undefined) process.nextTick(done); return true; }; - return sendFrame; + return { + send: sendFrame, + take(): string { + const frames = pending; + pending = ""; + return frames; + }, + }; } if (process.argv[1] === fileURLToPath(import.meta.url)) { - const sendFrame = frameStdoutWrites(); + const frames = frameStdoutWrites(); + const sendFrame = frames.send; const report = (error: unknown): void => { sendFrame({ openOrangeWorkerLoaderDiagnostic: @@ -171,10 +212,15 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { // The dynamic Worker never outlives the runtime process that loaded it. process.stdin.once("end", () => process.exit(0)); process.stdout.on("error", () => process.exit(0)); + // Frames sent just before an exit still reach the parent. + process.on("exit", () => { + const unsent = frames.take(); + if (unsent !== "") writeSync(1, unsent); + }); // abortIsolate() reports its reason and ends this process at once, so nothing else from the // aborted isolate reaches the parent. The frame is written synchronously for that reason. Reflect.set(globalThis, abortIsolateHook, (reason: string): never => { - writeSync(1, `${JSON.stringify({ openOrangeWorkerLoaderAbort: reason })}\n`); + writeSync(1, `${frames.take()}${JSON.stringify({ openOrangeWorkerLoaderAbort: reason })}\n`); process.exit(1); }); const transport = new FramedRpcTransport(readFrames(process.stdin, () => undefined), sendFrame); diff --git a/packages/runtime-node/src/bindings/worker-loader/loader.ts b/packages/runtime-node/src/bindings/worker-loader/loader.ts index e3fac41..6d99c11 100644 --- a/packages/runtime-node/src/bindings/worker-loader/loader.ts +++ b/packages/runtime-node/src/bindings/worker-loader/loader.ts @@ -489,6 +489,11 @@ class DynamicWorker { #settling: WorkerHost | undefined; #lastUsed = 0; #orphaned = false; + // The session's main stub once `start` is sent. Entrypoint calls pipeline behind `start`, so a + // new Worker's first call reaches its process together with `start` instead of after its reply. + readonly #sent: Promise; + #markSent: (main: CapnWebStub) => void = () => undefined; + #failSent: (error: unknown) => void = () => undefined; #session: LoaderSession | undefined; #terminated = false; @@ -502,6 +507,11 @@ class DynamicWorker { this.#label = label; this.#onExit = onExit; this.#options = options; + this.#sent = new Promise((resolveSent, rejectSent) => { + this.#markSent = resolveSent; + this.#failSent = rejectSent; + }); + this.#sent.catch(() => undefined); } start(): void { @@ -590,9 +600,12 @@ class DynamicWorker { this.#onExit(); void removeHost.remove(); }); - const reply = await session.main.start(envelope); - if (reply !== undefined && reply !== null) decodeLoaderReply(reply); + const started = session.main.start(envelope); + // The envelope's capabilities are exported once `start` is sent, before any other call. this.#baseExports = session.stats().exports; + this.#markSent(session.main); + const reply = await started; + if (reply !== undefined && reply !== null) decodeLoaderReply(reply); this.#session = session; this.#lastUsed = performance.now(); pool.metrics.workerStart.observe((this.#lastUsed - startedAt) / 1_000); @@ -602,6 +615,7 @@ class DynamicWorker { else this.#settling = host; return session; } catch (error: unknown) { + this.#failSent(error); host?.settled(); this.terminate(); // A running child frees the slot when it exits. Without one, nothing else frees it. @@ -698,15 +712,16 @@ class DynamicWorker { // A capability for `getEntrypoint(name, { props })`. entrypoint(name: string | undefined, props: unknown): Promise { - const running = this.#running ?? Promise.reject(new Error("Dynamic Worker is not started")); + const running = this.#running; + if (running === undefined) return Promise.reject(new Error("Dynamic Worker is not started")); const propsEnvelope = props === undefined ? undefined : encodeLoaderValue(props); - const remote = running.then(async ({ main }) => { - const stub = await main.entrypoint(name, propsEnvelope); - if ((typeof stub !== "object" || stub === null) && typeof stub !== "function") { - throw new Error("Dynamic Worker returned an invalid entrypoint"); - } + const remote = this.#sent.then((main) => { + // A promise for the entrypoint, so calls on it pipeline too. The process resolves it once + // the Worker has loaded. A call settles only after `start` does, so a Worker that fails to + // load fails its calls with the startup error. + const stub: object = main.entrypoint(name, propsEnvelope); this.#entrypointImports += 1; - return new RemoteCapability(stub); + return new RemoteCapability(stub, running); }); remote.catch(() => undefined); return remote; diff --git a/packages/runtime-node/src/bindings/worker-loader/session.ts b/packages/runtime-node/src/bindings/worker-loader/session.ts index 75dd584..e5ca276 100644 --- a/packages/runtime-node/src/bindings/worker-loader/session.ts +++ b/packages/runtime-node/src/bindings/worker-loader/session.ts @@ -340,10 +340,14 @@ export class LoaderCapabilityTarget extends DurableObjectRpcTarget { // process's LoaderCapabilityTarget. export class RemoteCapability implements Capability { readonly #stub: object; + // Settles before each call's result does, and its rejection replaces the call's. See + // `DynamicWorker.entrypoint()`. + readonly #ready: Promise | undefined; #released = false; - constructor(stub: object) { + constructor(stub: object, ready?: Promise) { this.#stub = stub; + this.#ready = ready; } call(path: readonly string[], args: readonly unknown[] | undefined): CallOutcome { @@ -353,7 +357,14 @@ export class RemoteCapability implements Capability { const envelope = args === undefined ? undefined : encodeLoaderValue([...args]); const invoke: unknown = Reflect.get(this.#stub, "invoke"); if (typeof invoke !== "function") throw new Error("Dynamic Worker capability is invalid"); - const reply: unknown = await Reflect.apply(invoke, this.#stub, [[...path], envelope]); + let reply: unknown; + try { + reply = await Reflect.apply(invoke, this.#stub, [[...path], envelope]); + } catch (error: unknown) { + await this.#ready; + throw error; + } + await this.#ready; return decodeLoaderReply(reply); })(); value.catch(() => undefined); @@ -363,7 +374,7 @@ export class RemoteCapability implements Capability { } dup(): Capability { - return new RemoteCapability(duplicate(this.#stub)); + return new RemoteCapability(duplicate(this.#stub), this.#ready); } dispose(): void { diff --git a/tests/bindings/worker-loader.test.ts b/tests/bindings/worker-loader.test.ts index 12eb08d..7544249 100644 --- a/tests/bindings/worker-loader.test.ts +++ b/tests/bindings/worker-loader.test.ts @@ -159,6 +159,10 @@ test("loads dynamic Workers with workerd Worker Loader semantics in agentOS VMs" // name or no name loads a fresh Worker and calls the code callback. assert.deepEqual(await get("cache"), { counts: [1, 2, 3, 1, 1], loads: 3 }); + // Calls made while a Worker loads reach it in order and each gets its own result, even + // beside a 3 MiB request body. + assert.deepEqual(await get("pipelined"), ["/a:3:one", `/large:${3 * 1024 * 1024}:end`, 42, "/b:3:two"]); + // A syntax error in a module rejects the call with V8's SyntaxError. assert.deepEqual(await get("syntax"), { error: { message: "Unexpected token ';'", name: "SyntaxError" }, diff --git a/tests/fixtures/worker-loader/worker.js b/tests/fixtures/worker-loader/worker.js index 6e320cd..d866173 100644 --- a/tests/fixtures/worker-loader/worker.js +++ b/tests/fixtures/worker-loader/worker.js @@ -580,6 +580,34 @@ const routes = { const sameTick = await third; return { full, heldCounts, loads, named, sameTick, sameTickCalls, sequential, slept }; }, + // Calls made while the Worker still loads, one of them with a body larger than one + // stdin write to its process, each get their own result. + async pipelined(env) { + const worker = env.LOADER.load({ + compatibilityDate: "2026-09-03", + mainModule: "main.js", + modules: { + "main.js": ` + export default { + async fetch(request) { + const body = await request.text(); + return new Response(new URL(request.url).pathname + ":" + body.length + ":" + body.slice(-3)); + }, + double(n) { return n * 2; }, + }; + `, + }, + }); + const large = "x".repeat(3 * 1024 * 1024 - 3) + "end"; + const send = (path, body) => + worker.getEntrypoint().fetch(`http://x/${path}`, { body, method: "POST" }).then((response) => response.text()); + return await Promise.all([ + send("a", "one"), + send("large", large), + worker.getEntrypoint().double(21), + send("b", "two"), + ]); + }, async syntax(env) { const worker = env.LOADER.get("syntax", () => ({ compatibilityDate: "2026-09-03", From 866f18a366c58dde0964b4e8b82bdde6cbb6060c Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sun, 15 Nov 2026 01:49:04 +0400 Subject: [PATCH 6/7] perf(worker-loader): boot dynamic Worker children from an agentOS module snapshot - Read WebAssembly on use, not at module load, in the VM runtime bundle - Move child startup into startChild() behind a thin worker-loader-child-main.mjs entry - Pass snapshotEntrypoints for that entry when the agentOS client supports it (0.4.8 does not) - Check runtime module state, cloudflare:workers env, and randomness per pooled Worker --- docs/modules/worker-loader.mdx | 10 ++++++++-- .../src/bindings/queues/workerd-wire.ts | 16 +++++++++++----- .../runtime-node/src/bindings/unsafe-eval.ts | 4 ++-- .../src/bindings/worker-loader/agentos.ts | 13 +++++++++++-- .../src/bindings/worker-loader/child.ts | 8 ++++++-- .../src/globals/implementations.ts | 18 ++++++++++++------ .../runtime-node/src/globals/webassembly.ts | 15 +++++++++------ packages/runtime-node/src/index.ts | 2 +- src/build/build-package.ts | 8 ++++++++ tests/bindings/worker-loader.test.ts | 18 +++++++++++++++--- tests/fixtures/worker-loader/worker.js | 13 +++++++++++-- 11 files changed, 94 insertions(+), 31 deletions(-) diff --git a/docs/modules/worker-loader.mdx b/docs/modules/worker-loader.mdx index f640a27..0ec998c 100644 --- a/docs/modules/worker-loader.mdx +++ b/docs/modules/worker-loader.mdx @@ -181,6 +181,11 @@ timeout passes. Prewarmed VMs do not count toward Worker loads, so an app without dynamic Workers boots none. A prewarmed VM has its Node.js process started with Orange's runtime imported. It has seen no Worker code or `env`. +- With an agentOS client that supports `snapshotEntrypoints`, each Node.js + process boots from a V8 snapshot of Orange's runtime, built once per agentOS + sidecar, instead of importing it. Each process gets its own copy of the + runtime's state. agentOS 0.4.8 lacks the option, so there each process + imports the runtime. - A load takes a prewarmed VM, writes its modules, and starts the Worker. If the pool is empty, it takes a VM that is booting and that no other load waits for, or boots its own at once, beside the other loads. @@ -301,8 +306,9 @@ server and an Express app in a dynamic Worker, and bursts of loads larger than the VM pool whose Workers each wait in a call for all the others. The VM pool grows to the last burst and shrinks to its idle size after the idle timeout, and each Worker gets a fresh VM, isolated from the others and from the -read-only runtime mount. A second `orange dev` with a limit of 2 and a -1.5-second idle timeout checks that +read-only runtime mount, with its own runtime module state and randomness +when its process boots from the agentOS snapshot. A second `orange dev` with +a limit of 2 and a 1.5-second idle timeout checks that idle Workers stop and load again, that calls longer than the timeout finish, that busy Workers fail a new load with `WorkerLoaderLimitError`, and that more `load()` calls than the limit succeed. diff --git a/packages/runtime-node/src/bindings/queues/workerd-wire.ts b/packages/runtime-node/src/bindings/queues/workerd-wire.ts index dfe060f..3c726fe 100644 --- a/packages/runtime-node/src/bindings/queues/workerd-wire.ts +++ b/packages/runtime-node/src/bindings/queues/workerd-wire.ts @@ -35,13 +35,19 @@ const maxHeadersGuard = 3; // The index is workerd's ErrorTag. Unknown (0) carries its own name. // https://github.com/cloudflare/workerd/blob/b73ce3ae105c35bafaaad864095079524573a465/src/workerd/jsg/ser.c%2B%2B#L23 -const webAssemblyError = (name: string): ErrorConstructor => - Reflect.get(Reflect.get(globalThis, "WebAssembly") as object, name) as ErrorConstructor; -const errorClasses: readonly (ErrorConstructor | undefined)[] = [ +// A string names a WebAssembly class, read on use: V8 has no WebAssembly +// global while agentOS builds a module snapshot of the dynamic Worker child. +const errorClasses: readonly (ErrorConstructor | string | undefined)[] = [ Error, Error, TypeError, RangeError, ReferenceError, SyntaxError, - webAssemblyError("CompileError"), webAssemblyError("LinkError"), webAssemblyError("RuntimeError"), + "CompileError", "LinkError", "RuntimeError", undefined, EvalError, URIError, undefined, undefined, ]; +const errorClass = (tag: number): ErrorConstructor | undefined => { + const entry = errorClasses[tag]; + return typeof entry === "string" + ? (Reflect.get(Reflect.get(globalThis, "WebAssembly") as object, entry) as ErrorConstructor) + : entry; +}; const errorNames: Readonly> = { 9: "SuspendError", 12: "AggregateError", @@ -111,7 +117,7 @@ class WorkerdWireDeserializer extends Deserializer { const name: unknown = errorTag === 0 ? this.readValue() : errorNames[errorTag]; const message: unknown = this.readValue(); if (typeof message !== "string") throw cloneError("error message is not a string"); - const error = new (errorClasses[errorTag] ?? Error)(message); + const error = new (errorClass(errorTag) ?? Error)(message); if (name !== undefined) Object.defineProperty(error, "name", { configurable: true, value: name, writable: true }); const properties: unknown = this.readValue(); if (isObject(properties)) { diff --git a/packages/runtime-node/src/bindings/unsafe-eval.ts b/packages/runtime-node/src/bindings/unsafe-eval.ts index ba2cbf4..d70143e 100644 --- a/packages/runtime-node/src/bindings/unsafe-eval.ts +++ b/packages/runtime-node/src/bindings/unsafe-eval.ts @@ -1,6 +1,6 @@ import { compileFunction, runInThisContext } from "node:vm"; -import { WebAssembly } from "../globals/webassembly.js"; +import { nativeWebAssembly } from "../globals/webassembly.js"; function bytesFrom(source: unknown): Uint8Array { if (source instanceof ArrayBuffer) return new Uint8Array(source); @@ -48,7 +48,7 @@ export class UnsafeEval { } newWasmModule(source: unknown): unknown { - return new WebAssembly.Module(bytesFrom(source)); + return new (nativeWebAssembly().Module)(bytesFrom(source)); } } diff --git a/packages/runtime-node/src/bindings/worker-loader/agentos.ts b/packages/runtime-node/src/bindings/worker-loader/agentos.ts index 40005e7..0be48ba 100644 --- a/packages/runtime-node/src/bindings/worker-loader/agentos.ts +++ b/packages/runtime-node/src/bindings/worker-loader/agentos.ts @@ -26,6 +26,9 @@ import type { VmGauges, WorkerLoaderMetrics } from "./metrics.js"; // Where the child and runtime modules live inside each VM. const runtimeRoot = "/open-orange/runtime"; +// The thin process entry that `buildWorkerLoaderVmBundle` writes. It imports +// worker-loader-child.mjs, which holds the child and the whole runtime. +const childEntry = posix.join(runtimeRoot, "worker-loader-child-main.mjs"); const workersRoot = "/open-orange/workers"; const exitWaitMs = 5_000; // The most stdin bytes sent to a dynamic Worker process in one agentOS write. @@ -612,12 +615,18 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend { return [specifier, posix.join(runtimeRoot, file)]; }), ); - const { AgentOs, hostDirMount } = await import("@rivet-dev/agentos-core"); + const { AgentOs, agentOsOptionFieldSchemas, hostDirMount } = await import("@rivet-dev/agentos-core"); + // agentOS boots each child from a snapshot of the runtime that the + // entry imports, built once per sidecar. agentOS 0.4.8 lacks the option + // and rejects options it does not know, so it is set only when present. + const snapshot = + "snapshotEntrypoints" in agentOsOptionFieldSchemas ? { snapshotEntrypoints: [childEntry] } : {}; // The guest reads Orange's runtime from the host, read-only, so a VM // boots without a copy of it. The mount shows only the bundle directory. const vm = await AgentOs.create({ mounts: [hostDirMount(runtimeRoot, bundlePath, { readOnly: true })], permissions: { network: mode === "network" ? "allow" : "deny" }, + ...snapshot, }); return { moduleRewrites, vm }; } @@ -659,7 +668,7 @@ async function spawnVmChild(vm: AgentOs, root: string, options: VmChildOptions): let exitReason: string | undefined; let killing = false; let forceExit: (reason: string) => void = () => undefined; - const descriptor = await vm.process.spawn("node", [posix.join(runtimeRoot, "worker-loader-child.mjs")], { + const descriptor = await vm.process.spawn("node", [childEntry], { cwd: root, env: { HOME: "/home/user", PATH: "/usr/local/bin:/usr/bin:/bin" }, onStderr: (chunk) => process.stderr.write(chunk), diff --git a/packages/runtime-node/src/bindings/worker-loader/child.ts b/packages/runtime-node/src/bindings/worker-loader/child.ts index 3eb6fbb..66082b3 100644 --- a/packages/runtime-node/src/bindings/worker-loader/child.ts +++ b/packages/runtime-node/src/bindings/worker-loader/child.ts @@ -3,7 +3,7 @@ // env, modules, and outbound access that the parent passed. import { writeSync } from "node:fs"; -import { fileURLToPath, pathToFileURL } from "node:url"; +import { pathToFileURL } from "node:url"; import { DurableObjectRpcTarget } from "../durable-objects/rpc/target.js"; import type { Request, Response } from "../../globals/http.js"; @@ -196,7 +196,11 @@ function frameStdoutWrites(): FrameWriter { }; } -if (process.argv[1] === fileURLToPath(import.meta.url)) { +// Runs the child. The thin entry `worker-loader-child-main.mjs` calls it, so +// agentOS can boot the child from a snapshot that holds this module and the +// whole runtime, evaluated once. Read per-process state such as `process.env` +// in here or later, never at module top level. +export function startChild(): void { const frames = frameStdoutWrites(); const sendFrame = frames.send; const report = (error: unknown): void => { diff --git a/packages/runtime-node/src/globals/implementations.ts b/packages/runtime-node/src/globals/implementations.ts index 9b17f11..215b606 100644 --- a/packages/runtime-node/src/globals/implementations.ts +++ b/packages/runtime-node/src/globals/implementations.ts @@ -37,7 +37,7 @@ import { import { Navigator, navigator } from "./navigator.js"; import { nodeWebGlobals } from "./node-web.js"; import { scheduler } from "./scheduler.js"; -import { WebAssembly } from "./webassembly.js"; +import { nativeWebAssembly } from "./webassembly.js"; import { WebSocket, WebSocketPair, @@ -96,7 +96,10 @@ const scopeGlobals = { URL, URLPattern, URLSearchParams, - WebAssembly, + // A getter, so loading this module never reads WebAssembly. See nativeWebAssembly(). + get WebAssembly() { + return nativeWebAssembly(); + }, WebSocket, WebSocketPair, WebSocketRequestResponsePair, @@ -142,10 +145,13 @@ type ScopeGlobals = typeof scopeGlobals; // assertion. The global-scope differential fixture reads these globals from self. const self = globalThis as unknown as GlobalScope; -export const globalImplementations = { - ...scopeGlobals, - self, -}; +// Copies the descriptors, so the WebAssembly getter is not called here. +// SAFETY: defineProperties adds every scopeGlobals property to the object, +// but TypeScript types its result as the first argument. +export const globalImplementations = Object.defineProperties( + { self }, + Object.getOwnPropertyDescriptors(scopeGlobals), +) as ScopeGlobals & { self: GlobalScope }; export const globalImplementationStatuses: Record< keyof typeof globalImplementations, diff --git a/packages/runtime-node/src/globals/webassembly.ts b/packages/runtime-node/src/globals/webassembly.ts index 4dc264c..77dbfe4 100644 --- a/packages/runtime-node/src/globals/webassembly.ts +++ b/packages/runtime-node/src/globals/webassembly.ts @@ -141,10 +141,13 @@ function isWebAssemblyNamespace(value: unknown): value is WebAssemblyNamespace { } // Preserve Node's full namespace (including host-only helpers used by Undici), -// while exposing only the Cloudflare-visible contract to Orange code. -const nativeWebAssembly: unknown = Reflect.get(globalThis, "WebAssembly"); -if (!isWebAssemblyNamespace(nativeWebAssembly)) { - throw new Error("Node.js does not provide the required WebAssembly global"); +// while exposing only the Cloudflare-visible contract to Orange code. Read on +// use, not at module load: V8 has no WebAssembly global while agentOS builds +// a module snapshot of the dynamic Worker child. +export function nativeWebAssembly(): WebAssemblyNamespace { + const value: unknown = Reflect.get(globalThis, "WebAssembly"); + if (!isWebAssemblyNamespace(value)) { + throw new Error("Node.js does not provide the required WebAssembly global"); + } + return value; } - -export const WebAssembly = nativeWebAssembly; diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index a8a07e8..cca917c 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -78,7 +78,7 @@ export { WritableStream, } from "./globals/native.js"; export { Event } from "./globals/event.js"; -export { WebAssembly } from "./globals/webassembly.js"; +export { nativeWebAssembly } from "./globals/webassembly.js"; export { WebSocket, WebSocketPair } from "./globals/websocket.js"; export { clearInterval, diff --git a/src/build/build-package.ts b/src/build/build-package.ts index b3b9f7d..126e3dc 100644 --- a/src/build/build-package.ts +++ b/src/build/build-package.ts @@ -74,6 +74,14 @@ export async function buildWorkerLoaderVmBundle( join(outputDirectory, "manifest.json"), `${JSON.stringify(manifest, null, "\t")}\n`, ); + // The process entry agentOS runs for each dynamic Worker (`childEntry` in + // worker-loader/agentos.ts). It stays this small so agentOS can boot it + // from a snapshot of everything it imports, and only these two lines run + // in each process. + await writeFile( + join(outputDirectory, "worker-loader-child-main.mjs"), + 'import { startChild } from "./worker-loader-child.mjs";\nstartChild();\n', + ); } export async function buildPackage( diff --git a/tests/bindings/worker-loader.test.ts b/tests/bindings/worker-loader.test.ts index 7544249..065c804 100644 --- a/tests/bindings/worker-loader.test.ts +++ b/tests/bindings/worker-loader.test.ts @@ -474,22 +474,34 @@ test("grows the VM pool to the last burst, shrinks it when idle, and boots each }); assert.ok(Array.isArray(results) && results.length === n, JSON.stringify(results)); let prewarmed = 0; + const randoms = new Set(); + const uuids = new Set(); for (const result of results) { assert.ok(isRecord(result) && isRecord(result.value), JSON.stringify(result)); - const { prewarmed: wasPrewarmed, ...seen } = result.value; + const { prewarmed: wasPrewarmed, random, ...seen } = result.value; const id = String(seen.id); - // None sees another's files, globals, or env, each VM holds only - // its own Worker, and none can change Orange's runtime. + // None sees another's files, globals, runtime module state, or env, + // each VM holds only its own Worker, and none can change Orange's + // runtime. This holds for children that boot from an agentOS + // snapshot of the runtime too. assert.deepEqual(seen, { env: [`POOL_SECRET_${id}`], file: "ENOENT", global: null, id, runtime: "EROFS", + runtimeState: null, workerRoots: 1, + workersEnv: [`POOL_SECRET_${id}`], }); + assert.ok(Array.isArray(random), JSON.stringify(result)); + randoms.add(random[0]); + uuids.add(random[1]); if (wasPrewarmed === true) prewarmed += 1; } + // No two Workers draw the same random values. + assert.equal(randoms.size, n); + assert.equal(uuids.size, n); return prewarmed; }; // Both network modes load beside each other. diff --git a/tests/fixtures/worker-loader/worker.js b/tests/fixtures/worker-loader/worker.js index d866173..4b4a1e6 100644 --- a/tests/fixtures/worker-loader/worker.js +++ b/tests/fixtures/worker-loader/worker.js @@ -22,8 +22,10 @@ const filesystemCode = { }; // Reports what a dynamic Worker can see of any other dynamic Worker: a file, -// a global, process.env, and the VM's Worker roots. Then it leaves its own, -// and tries to change Orange's runtime, which every VM reads from the host. +// a global, state in Orange's runtime modules, process.env, the env of +// cloudflare:workers, and the VM's Worker roots. Then it leaves its own, and +// tries to change Orange's runtime, which every VM reads from the host. A VM +// boots its child from one agentOS snapshot of the runtime when it can. // It also reports whether its VM had its Worker root before the parent asked // for the Worker at `requestedAt`, as a prewarmed VM does, and waits at // `arrive` for the rest of its burst. @@ -35,9 +37,13 @@ const poolProbeCode = (id, globalOutbound) => ({ modules: { "main.js": ` import { createRequire } from "node:module"; + import { env as workersEnv } from "cloudflare:workers"; const fs = createRequire(import.meta.url)("fs"); const seenGlobal = globalThis.poolProbe ?? null; globalThis.poolProbe = ${JSON.stringify(id)}; + // Response is a class of Orange's runtime, not of Node.js. + const seenRuntime = Response.prototype.poolProbe ?? null; + Response.prototype.poolProbe = ${JSON.stringify(id)}; export default { async probe({ arrive, requestedAt }) { const rootCreatedAt = fs.statSync(process.cwd()).birthtimeMs; @@ -53,8 +59,11 @@ const poolProbeCode = (id, globalOutbound) => ({ global: seenGlobal, id: ${JSON.stringify(id)}, prewarmed: rootCreatedAt < requestedAt, + random: [Math.random(), crypto.randomUUID()], runtime, + runtimeState: seenRuntime, workerRoots: fs.readdirSync("/open-orange/workers").length, + workersEnv: Object.keys(workersEnv), }; }, }; From 89309853ce939328670c138312fc54864fdaa1d8 Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Tue, 17 Nov 2026 02:49:47 +0400 Subject: [PATCH 7/7] chore(release): update version to 1.0.4-rc.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b20c025..b5e70fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-orange", - "version": "1.0.4-rc.1", + "version": "1.0.4-rc.2", "description": "Cloudflare Workers compatibility layer for Rivet", "packageManager": "pnpm@10.34.9", "engines": {