diff --git a/docs/modules/worker-loader.mdx b/docs/modules/worker-loader.mdx
index 6f18ce6..0ec998c 100644
--- a/docs/modules/worker-loader.mdx
+++ b/docs/modules/worker-loader.mdx
@@ -8,7 +8,7 @@ description: Load and run dynamic Workers at runtime with the worker_loaders bin
@@ -89,8 +89,9 @@ Hello from a dynamic Worker
- Dynamic Worker code is untrusted, so each dynamic Worker gets its own VM,
as it gets its own isolate on Cloudflare. It never runs on the host.
-- Orange keeps a few VMs booted ahead of use, so a new dynamic Worker only
- writes its own modules. Each VM runs one dynamic Worker and is never reused.
+- Orange keeps VMs booted ahead of use, and more after a burst of loads, so a
+ new dynamic Worker only writes its own modules. Each VM runs one dynamic
+ Worker and is never reused.
- A dynamic Worker stops when it is idle or nothing references it. Its VM is
disposed, never returned to the pool.
- Your Worker and the dynamic Worker talk Workers RPC, so stubs and bindings
@@ -125,13 +126,44 @@ agentOS's limits apply:
| One module file | 16 MiB | The load fails with a `RangeError` that names the limit. Split the module. |
| All files in the VM | 64 MiB by default | agentOS's limit. Orange adds no check. |
| `env` | No set limit. In a test with agentOS 0.4.1, 5 MiB worked. | At 10 MiB, the dynamic Worker exited with code 1. |
-| Running dynamic Workers, across all Worker Loader bindings | 32. Set `ORANGE_WORKER_LOADER_MAX_WORKERS` to change it. | A new dynamic Worker stops the least recently used idle one. If none is idle, `get()` or `load()` throws a `WorkerLoaderLimitError` that names the limit. At 80% of the limit, `orange dev` logs a warning. |
+| Running dynamic Workers, across all Worker Loader bindings | 32. Set `ORANGE_WORKER_LOADER_MAX_WORKERS` to change it. | A new dynamic Worker stops the least recently used idle one. If none is idle, it waits in line for a slot, first come first served, and its calls wait with it. At 80% of the limit, Orange logs a warning. |
+| Time a dynamic Worker waits for a slot | 5 seconds. Set `ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS` to change it, from 1 to 2147483647. | Its calls reject with a `WorkerLoaderLimitError` whose `code` is `queue_timeout`. |
+| Dynamic Workers that wait for a slot | 256. Set `ORANGE_WORKER_LOADER_MAX_QUEUED` to change it; `0` refuses at once. | Its calls reject with a `WorkerLoaderLimitError` whose `code` is `queue_full`. At 80% of the limit, Orange logs a warning. |
+| agentOS VMs, counting running Workers, prewarmed VMs, and VMs that boot or stop | `ORANGE_WORKER_LOADER_MAX_WORKERS` plus `ORANGE_WORKER_LOADER_POOL_MAX`, 48 by default. Set `ORANGE_WORKER_LOADER_MAX_VMS` to change it; it must not be less than `ORANGE_WORKER_LOADER_MAX_WORKERS`. | Prewarmed VMs stop first, and the pool stops refilling. A Worker that needs a new VM waits for one to stop. At 80% of the budget, Orange logs a warning. |
| Idle time before a dynamic Worker stops | 30 seconds. Set `ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS` to change it, from 1 to 2147483647. | The dynamic Worker stops and frees its VM. The next call loads it again. A value out of range stops `orange dev` at startup with a `RangeError` that names the setting. |
-| Prewarmed VMs | 2 for each network mode by default, from 0 to 16 with `ORANGE_WORKER_LOADER_POOL_SIZE` | A load that finds the pool empty boots its own VM. |
-
-Each running dynamic Worker uses about 54 MiB. Prewarmed VMs that no Worker
-has claimed do not count toward `ORANGE_WORKER_LOADER_MAX_WORKERS`, and the
-limit never stops them.
+| Prewarmed VMs when no burst is recent | 2 for each network mode by default, from 0 to 64 with `ORANGE_WORKER_LOADER_POOL_SIZE` | A load that finds the pool empty boots its own VM. |
+| Prewarmed VMs after a burst | 16 for each network mode by default, from 0 to 64 with `ORANGE_WORKER_LOADER_POOL_MAX`. It must not be less than `ORANGE_WORKER_LOADER_POOL_SIZE`. `0` boots every VM on demand. | The pool stops growing. A value out of range stops `orange dev` at startup with a `RangeError` that names the setting. |
+| VMs that prewarm at once | 4, across both network modes, while no Worker loads. While Workers load, only as many as keep up with a steady rate of loads. | The other prewarms wait. Loads never wait for this limit. |
+
+Each running dynamic Worker uses about 54 MiB. Each prewarmed VM uses 26 to
+54 MiB, depending on the agentOS version. After a burst of 16 or more loads,
+a network mode can hold 16 prewarmed VMs, 0.4 to 0.9 GiB, until the idle
+timeout passes. Prewarmed VMs do not count toward
+`ORANGE_WORKER_LOADER_MAX_WORKERS`, but they count toward
+`ORANGE_WORKER_LOADER_MAX_VMS`, which bounds the memory of all VMs.
+
+##### Refusals
+
+- A `WorkerLoaderLimitError` is a `RangeError` with `code` (`queue_full` or
+ `queue_timeout`), `overloaded: true`, `retryable: true`, and
+ `retryAfterSeconds: 1`, like the overloaded error of a Cloudflare Durable
+ Object. `get()` and `load()` never throw it; the Worker's calls reject with it.
+- If your Worker does not catch it, the response is `503 Service Unavailable`
+ with `Retry-After: 1`, not 500. A WebSocket upgrade gets the same 503. On
+ Cloudflare, an uncaught exception is a 500 (error 1101). Cloudflare has no
+ per-runner Worker limit, so it has no such refusal.
+- A held stub whose Worker was refused asks for a slot again on its next call,
+ and `get()` with the same name calls `getCode` again.
+- Under load that stays above what the runner can start, the line stays full:
+ a Worker that gets a slot has waited close to the queue timeout, and the
+ rest are refused. Lower `ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS` to trade
+ refusals for latency, and scale out on the queue wait (see
+ [Monitoring](../operations/monitoring#autoscaling)).
+- agentOS bounds the requests in flight to its shared sidecar at 128, and
+ each running Worker keeps some open. In a test that started 80 Workers at
+ once with `ORANGE_WORKER_LOADER_MAX_WORKERS` at 96, 33 to 38 started; the
+ rest failed with agentOS's `ERR_AGENTOS_IN_FLIGHT_REQUEST_LIMIT`. When a
+ running Worker's process fails this way, its call fails with a 500.
#### Isolation
@@ -149,12 +181,32 @@ limit never stops them.
Worker loads, so an app without dynamic Workers boots none. A prewarmed VM
has its Node.js process started with Orange's runtime imported. It has seen
no Worker code or `env`.
+- With an agentOS client that supports `snapshotEntrypoints`, each Node.js
+ process boots from a V8 snapshot of Orange's runtime, built once per agentOS
+ sidecar, instead of importing it. Each process gets its own copy of the
+ runtime's state. agentOS 0.4.8 lacks the option, so there each process
+ imports the runtime.
- A load takes a prewarmed VM, writes its modules, and starts the Worker. If
- the pool is empty, it waits for the VM that is booting, or boots its own.
- The pool refills one VM at a time, shortly after the load finishes.
+ the pool is empty, it takes a VM that is booting and that no other load
+ waits for, or boots its own at once, beside the other loads.
+- Each network mode's pool aims for the most loads seen within one prewarm
+ time, from `ORANGE_WORKER_LOADER_POOL_SIZE` to
+ `ORANGE_WORKER_LOADER_POOL_MAX`. So the next burst of the same size finds
+ its VMs ready, and a steady stream of loads keeps the pool ahead.
+- The pool refills at full speed as soon as no dynamic Worker is loading,
+ because a prewarm that runs beside a load slows it. Workers that load at the
+ same time also hold the refill until their first calls finish. One Worker
+ holds the refill back for at most 1 second.
+- While Workers keep loading, the pool still refills, with as many prewarms
+ as the rate of loads needs. Only loads that keep arriving count, so a burst
+ does not refill beside its own loads.
+- When the idle timeout (`ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS`) passes
+ without a burst as large, the pool shrinks to
+ `ORANGE_WORKER_LOADER_POOL_SIZE` and disposes the other prewarmed VMs.
- A VM that fails to prewarm is reported as a background error, and the next
- load boots its own VM. Set `ORANGE_WORKER_LOADER_POOL_SIZE` to `0` to boot
- every VM on demand.
+ load boots its own VM. The pool tries to prewarm again only after its next
+ load. Set `ORANGE_WORKER_LOADER_POOL_MAX` to `0` to boot every VM on
+ demand.
- If the VM cannot boot, `get()` and `load()` calls fail with the boot error.
Orange never falls back to the host.
- A dynamic Worker is idle when no call to it is in flight, your Worker holds
@@ -250,10 +302,13 @@ These add to the [Compatibility](#compatibility) table, the
`tests/bindings/worker-loader.test.ts` drives a real `orange dev` over HTTP:
fetch and RPC, env and props capabilities, isolation, `globalOutbound: null`,
name caching, syntax errors, process crashes, module types, a `node:http`
-server and an Express app in a dynamic Worker, and two bursts of loads larger
-than the VM pool, the second after the first burst's Workers stop idle, with
-each Worker isolated from the others and from the read-only runtime mount. A
-second `orange dev` with a limit of 2 and a 1.5-second idle timeout checks that
+server and an Express app in a dynamic Worker, and bursts of loads larger than
+the VM pool whose Workers each wait in a call for all the others. The VM pool
+grows to the last burst and shrinks to its idle size after the idle timeout,
+and each Worker gets a fresh VM, isolated from the others and from the
+read-only runtime mount, with its own runtime module state and randomness
+when its process boots from the agentOS snapshot. A second `orange dev` with
+a limit of 2 and a 1.5-second idle timeout checks that
idle Workers stop and load again, that calls longer than the timeout finish,
that busy Workers fail a new load with `WorkerLoaderLimitError`, and that more
`load()` calls than the limit succeed.
diff --git a/docs/operations/monitoring.mdx b/docs/operations/monitoring.mdx
index fda2733..651e471 100644
--- a/docs/operations/monitoring.mdx
+++ b/docs/operations/monitoring.mdx
@@ -68,6 +68,12 @@ $ docker run \
-e RIVET_ENDPOINT=... my-worker
```
+## Autoscale dynamic Workers
+
+Scale runners on `orange_worker_loader_slot_utilization` and the slot queue
+wait, not on CPU. See [Dynamic Worker metrics](#dynamic-worker-metrics) for an
+example.
+
## Learn more
- [OpenTelemetry](https://rivet.dev/docs/tracing/): span names, sampling, and application spans
@@ -95,7 +101,7 @@ RivetKit serves Prometheus metrics at `http://127.0.0.1:$RIVETKIT_METRICS_PORT/m
them in the Rivet dashboard. `orange deploy` refuses to set it with `--var`.
- **Self-hosted:** the endpoint listens on `127.0.0.1` only, so `docker run -p` does not expose
it. Scrape it from the same network namespace, such as a sidecar in the same Kubernetes pod.
-- `orange dev` serves no metrics.
+- `orange dev` serves metrics when `RIVETKIT_METRICS_PORT` is set, from its first Worker.
- With several Workers, only the first Worker's runtime serves metrics, so they cover that
Worker's objects only.
@@ -105,6 +111,62 @@ RivetKit serves Prometheus metrics at `http://127.0.0.1:$RIVETKIT_METRICS_PORT/m
| `rivet_rivetkit_actor_http_requests_active` | Requests and RPC calls in progress |
| `rivet_rivetkit_actor_inbox_depth` | Work waiting to run |
+#### Dynamic Worker metrics
+
+A Worker with a `worker_loaders` binding adds these to the same endpoint.
+
+| Metric | Type | What it shows |
+|---|---|---|
+| `orange_worker_loader_workers` | gauge | Dynamic Workers that hold a slot |
+| `orange_worker_loader_workers_max` | gauge | `ORANGE_WORKER_LOADER_MAX_WORKERS` |
+| `orange_worker_loader_slot_utilization` | gauge | Slots in use divided by slots, from 0 to 1 |
+| `orange_worker_loader_queue_depth` | gauge | Dynamic Workers that wait for a slot |
+| `orange_worker_loader_queue_wait_seconds` | histogram | How long each dynamic Worker waited for a slot |
+| `orange_worker_loader_start_seconds` | histogram | From a slot to a loaded dynamic Worker |
+| `orange_worker_loader_vm_boot_seconds` | histogram | How long one agentOS VM takes to boot |
+| `orange_worker_loader_pool_claims_total` | counter | VM claims by `mode` (`network`, `deny`) and `result`: `hit` (prewarmed), `warming` (waited for a prewarm), `miss` (booted on demand) |
+| `orange_worker_loader_pool_ready` | gauge | Prewarmed VMs by `mode` |
+| `orange_worker_loader_vms` | gauge | agentOS VMs by `state`: `live`, `pooled`, `booting`, `stopping` |
+| `orange_worker_loader_vms_max` | gauge | `ORANGE_WORKER_LOADER_MAX_VMS` |
+| `orange_worker_loader_vm_queue_depth` | gauge | Dynamic Workers with a slot that wait for room in the VM budget |
+| `orange_worker_loader_rejections_total` | counter | `WorkerLoaderLimitError` refusals by `code`: `queue_full`, `queue_timeout` |
+| `orange_runtime_resident_memory_bytes` | gauge | RSS by `process`: `runner`, and on Linux `children`, which includes the agentOS sidecar |
+| `orange_runtime_memory_limit_bytes` | gauge | The runner's cgroup memory limit, when it has one |
+
+Useful queries:
+
+```promql
+# Pool miss rate
+sum(rate(orange_worker_loader_pool_claims_total{result="miss"}[5m]))
+ / sum(rate(orange_worker_loader_pool_claims_total[5m]))
+# Slot queue wait, p95
+histogram_quantile(0.95, sum by (le) (rate(orange_worker_loader_queue_wait_seconds_bucket[5m])))
+# Memory headroom
+sum(orange_runtime_resident_memory_bytes) / orange_runtime_memory_limit_bytes
+```
+
+The agentOS sidecar's own request queue is not visible to Orange, so it has no metric.
+
+#### Autoscaling
+
+With a Kubernetes HorizontalPodAutoscaler and a Prometheus adapter:
+
+- Scale on `orange_worker_loader_slot_utilization`, with a target of about
+ 0.7, and on the p95 of `orange_worker_loader_queue_wait_seconds`. A busy
+ slot can mean a dynamic Worker that waits on I/O, so CPU does not show
+ dynamic Worker demand. Do not scale on CPU.
+- Set `behavior.scaleDown.stabilizationWindowSeconds` longer than
+ `ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS` (30 s by default), such as 300 s. An
+ idle dynamic Worker holds its slot until the timeout, so a shorter window
+ scales down while slots still look busy, and then scales up again.
+- Alert on `orange_worker_loader_rejections_total`. Each refusal is a 503 or
+ a `WorkerLoaderLimitError` your Worker caught.
+- Size the pod's memory limit for `ORANGE_WORKER_LOADER_MAX_VMS` VMs, and
+ watch `orange_runtime_resident_memory_bytes` against
+ `orange_runtime_memory_limit_bytes`.
+- Orange has no readiness endpoint, so a full runner still receives
+ requests. Its 503s carry `Retry-After`.
+
#### Logs
- `orange dev` prints each request and your Worker's `console` output.
diff --git a/package.json b/package.json
index 2be9ded..b5e70fd 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "open-orange",
- "version": "1.0.3",
+ "version": "1.0.4-rc.2",
"description": "Cloudflare Workers compatibility layer for Rivet",
"packageManager": "pnpm@10.34.9",
"engines": {
@@ -50,7 +50,7 @@
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/client-s3": "3.1153.0",
- "@rivet-dev/agentos-core": "0.4.8",
+ "@rivet-dev/agentos-core": "0.4.9-rc.1",
"@rivet-dev/durable-object-migrator": "1.0.3",
"@rivet-dev/sandbox-adapter": "0.5.3",
"capnweb": "0.12.0",
diff --git a/packages/runtime-node/package.json b/packages/runtime-node/package.json
index 4ba210d..a1cba7e 100644
--- a/packages/runtime-node/package.json
+++ b/packages/runtime-node/package.json
@@ -27,7 +27,7 @@
},
"dependencies": {
"@aws-sdk/client-s3": "3.1153.0",
- "@rivet-dev/agentos-core": "0.4.8",
+ "@rivet-dev/agentos-core": "0.4.9-rc.1",
"@rivet-dev/durable-object-migrator": "1.0.3",
"@rivet-dev/sandbox-adapter": "0.5.3",
"capnweb": "0.12.0",
diff --git a/packages/runtime-node/src/bindings/queues/workerd-wire.ts b/packages/runtime-node/src/bindings/queues/workerd-wire.ts
index dfe060f..3c726fe 100644
--- a/packages/runtime-node/src/bindings/queues/workerd-wire.ts
+++ b/packages/runtime-node/src/bindings/queues/workerd-wire.ts
@@ -35,13 +35,19 @@ const maxHeadersGuard = 3;
// The index is workerd's ErrorTag. Unknown (0) carries its own name.
// https://github.com/cloudflare/workerd/blob/b73ce3ae105c35bafaaad864095079524573a465/src/workerd/jsg/ser.c%2B%2B#L23
-const webAssemblyError = (name: string): ErrorConstructor =>
- Reflect.get(Reflect.get(globalThis, "WebAssembly") as object, name) as ErrorConstructor;
-const errorClasses: readonly (ErrorConstructor | undefined)[] = [
+// A string names a WebAssembly class, read on use: V8 has no WebAssembly
+// global while agentOS builds a module snapshot of the dynamic Worker child.
+const errorClasses: readonly (ErrorConstructor | string | undefined)[] = [
Error, Error, TypeError, RangeError, ReferenceError, SyntaxError,
- webAssemblyError("CompileError"), webAssemblyError("LinkError"), webAssemblyError("RuntimeError"),
+ "CompileError", "LinkError", "RuntimeError",
undefined, EvalError, URIError, undefined, undefined,
];
+const errorClass = (tag: number): ErrorConstructor | undefined => {
+ const entry = errorClasses[tag];
+ return typeof entry === "string"
+ ? (Reflect.get(Reflect.get(globalThis, "WebAssembly") as object, entry) as ErrorConstructor)
+ : entry;
+};
const errorNames: Readonly> = {
9: "SuspendError",
12: "AggregateError",
@@ -111,7 +117,7 @@ class WorkerdWireDeserializer extends Deserializer {
const name: unknown = errorTag === 0 ? this.readValue() : errorNames[errorTag];
const message: unknown = this.readValue();
if (typeof message !== "string") throw cloneError("error message is not a string");
- const error = new (errorClasses[errorTag] ?? Error)(message);
+ const error = new (errorClass(errorTag) ?? Error)(message);
if (name !== undefined) Object.defineProperty(error, "name", { configurable: true, value: name, writable: true });
const properties: unknown = this.readValue();
if (isObject(properties)) {
diff --git a/packages/runtime-node/src/bindings/unsafe-eval.ts b/packages/runtime-node/src/bindings/unsafe-eval.ts
index ba2cbf4..d70143e 100644
--- a/packages/runtime-node/src/bindings/unsafe-eval.ts
+++ b/packages/runtime-node/src/bindings/unsafe-eval.ts
@@ -1,6 +1,6 @@
import { compileFunction, runInThisContext } from "node:vm";
-import { WebAssembly } from "../globals/webassembly.js";
+import { nativeWebAssembly } from "../globals/webassembly.js";
function bytesFrom(source: unknown): Uint8Array {
if (source instanceof ArrayBuffer) return new Uint8Array(source);
@@ -48,7 +48,7 @@ export class UnsafeEval {
}
newWasmModule(source: unknown): unknown {
- return new WebAssembly.Module(bytesFrom(source));
+ return new (nativeWebAssembly().Module)(bytesFrom(source));
}
}
diff --git a/packages/runtime-node/src/bindings/worker-loader/agentos.ts b/packages/runtime-node/src/bindings/worker-loader/agentos.ts
index 9210174..0be48ba 100644
--- a/packages/runtime-node/src/bindings/worker-loader/agentos.ts
+++ b/packages/runtime-node/src/bindings/worker-loader/agentos.ts
@@ -1,8 +1,9 @@
// Runs each dynamic Worker as a Node.js process in its own agentOS VM, so two
-// dynamic Workers never share a filesystem, process table, or loopback. A small
-// pool keeps VMs booted ahead of use, each with its child process started and
-// Orange's runtime imported, so a new dynamic Worker only writes its own modules.
-// A pooled VM serves one dynamic Worker and is never reused.
+// dynamic Workers never share a filesystem, process table, or loopback. A pool
+// sized to recent demand keeps VMs booted ahead of use, each with its child
+// process started and Orange's runtime imported, so a new dynamic Worker only
+// writes its own modules. A pooled VM serves one dynamic Worker and is never
+// reused.
//
// agentOS sets a VM's network policy in `AgentOs.create()` and has no API to
// change it after boot. A Worker with the default `globalOutbound` gets a VM
@@ -20,12 +21,18 @@ import type { AgentOs } from "@rivet-dev/agentos-core";
import { onRuntimeClose } from "../../internal/close-hooks.js";
import { errorMessage, isRecord } from "../unknown-values.js";
-import type { WorkerChild, WorkerHost, WorkerLoaderBackend } from "./loader.js";
+import { positiveIntegerSetting, type WorkerChild, type WorkerHost, type WorkerLoaderBackend } from "./loader.js";
+import type { VmGauges, WorkerLoaderMetrics } from "./metrics.js";
// Where the child and runtime modules live inside each VM.
const runtimeRoot = "/open-orange/runtime";
+// The thin process entry that `buildWorkerLoaderVmBundle` writes. It imports
+// worker-loader-child.mjs, which holds the child and the whole runtime.
+const childEntry = posix.join(runtimeRoot, "worker-loader-child-main.mjs");
const workersRoot = "/open-orange/workers";
const exitWaitMs = 5_000;
+// The most stdin bytes sent to a dynamic Worker process in one agentOS write.
+const maxStdinWriteBytes = 1024 * 1024;
type NetworkMode = "deny" | "network";
@@ -65,6 +72,8 @@ function assertFitsAgentOsFileMessage(path: string, content: string | Uint8Array
type VmChild = WorkerChild & { exited: Promise };
+type LoadHooks = Pick;
+
// A VM with the runtime bundle mounted and the child process started. The
// child has imported the runtime and waits for `start`. It has seen no user
// code or env.
@@ -75,33 +84,94 @@ interface ReadyVm {
root: string;
}
+// The ready VMs of one network mode. Its target size follows demand: the most
+// Workers claimed within one prewarm time, from `size` to `max`, so a burst
+// that drained the pool finds it full the next time. It falls back to `size`
+// once `idleMs` passes without such a peak.
interface VmPool {
mode: NetworkMode;
ready: ReadyVm[];
- // An open that waits for the prewarm in flight instead of booting its own VM.
- waiting: ((ready: ReadyVm | undefined) => void) | undefined;
- warming: boolean;
+ // Opens that wait for a prewarm in flight, at most one for each prewarm.
+ // Every other open boots its own VM at once.
+ waiting: ((ready: ReadyVm | undefined) => void)[];
+ warming: number;
+ // The latest claim times, oldest first, at most `max` of them.
+ claims: number[];
+ peak: number;
+ shrink: NodeJS.Timeout | undefined;
+ // Set when a prewarm fails. The pool refills again only after its next
+ // claim, so a VM that cannot boot never loops in the background.
+ failed: boolean;
}
-// Ready VMs to keep per network mode. A mode's pool fills only after its first
-// dynamic Worker, so an app without dynamic Workers boots no VM.
+// A mode's pool fills only after its first dynamic Worker, so an app without
+// dynamic Workers boots no VM.
export const workerLoaderPoolSizeSetting = "ORANGE_WORKER_LOADER_POOL_SIZE";
+export const workerLoaderPoolMaxSetting = "ORANGE_WORKER_LOADER_POOL_MAX";
const defaultPoolSize = 2;
-const maxPoolSize = 16;
-// The pool refills a moment after a Worker loads, so the prewarm does not slow
-// the Worker's first calls.
-const refillDelayMs = 250;
-
-export function workerLoaderPoolSize(settings: Readonly>): number {
- const value = settings[workerLoaderPoolSizeSetting];
- if (value === undefined || value === "") return defaultPoolSize;
- const size = Number(value);
- if (!Number.isInteger(size) || size < 0 || size > maxPoolSize) {
+const defaultPoolMax = 16;
+const poolSettingLimit = 64;
+// Prewarms in flight across both modes. They share one agentOS sidecar with
+// the running Workers. A load that finds its pool empty never waits for this.
+const maxParallelPrewarms = 4;
+// The longest one Worker holds back the refill while it loads.
+const maxRefillHoldMs = 1_000;
+// How far back the claim rate looks when loads keep the refill held.
+const claimRateWindowMs = 1_000;
+// Enough claim times to show a rate that needs every parallel prewarm.
+const maxRecentClaims = 512;
+// The prewarm time before the first one is measured.
+const initialPrewarmMs = 250;
+const warnRatio = 0.8;
+
+export interface WorkerLoaderPoolLimits {
+ // Ready VMs kept per network mode when no burst is recent.
+ size: number;
+ // The most ready VMs per network mode. 0 boots every VM on demand.
+ max: number;
+}
+
+// Every agentOS VM of dynamic Workers counts: running Workers, prewarmed VMs, VMs that boot,
+// and VMs that are still stopping. A running Worker's slot frees as soon as it stops, before its
+// VM is gone, so without this budget a churn of Workers could hold more VMs than the slots.
+export const workerLoaderMaxVmsSetting = "ORANGE_WORKER_LOADER_MAX_VMS";
+
+// Defaults to the slots plus one pool's most, so the pool has room beside full slots.
+export function workerLoaderMaxVms(
+ settings: Readonly>,
+ maxWorkers: number,
+ pool: WorkerLoaderPoolLimits,
+): number {
+ return positiveIntegerSetting(
+ settings,
+ workerLoaderMaxVmsSetting,
+ maxWorkers + pool.max,
+ Number.MAX_SAFE_INTEGER,
+ maxWorkers,
+ );
+}
+
+function poolSetting(settings: Readonly>, name: string, fallback: number): number {
+ const value = settings[name];
+ if (value === undefined || value === "") return fallback;
+ const count = Number(value);
+ if (!Number.isInteger(count) || count < 0 || count > poolSettingLimit) {
+ throw new RangeError(
+ `${name} must be an integer from 0 to ${poolSettingLimit}, got ${JSON.stringify(value)}. Set ${workerLoaderPoolMaxSetting} to 0 to boot every dynamic Worker's VM on demand.`,
+ );
+ }
+ return count;
+}
+
+export function workerLoaderPoolLimits(settings: Readonly>): WorkerLoaderPoolLimits {
+ const max = poolSetting(settings, workerLoaderPoolMaxSetting, defaultPoolMax);
+ const size = poolSetting(settings, workerLoaderPoolSizeSetting, Math.min(defaultPoolSize, max));
+ if (size > max) {
throw new RangeError(
- `${workerLoaderPoolSizeSetting} must be an integer from 0 to ${maxPoolSize}, got ${JSON.stringify(value)}. Set it to 0 to boot every dynamic Worker's VM on demand.`,
+ `${workerLoaderPoolSizeSetting} (${size}) must not be larger than ${workerLoaderPoolMaxSetting} (${max}).`,
);
}
- return size;
+ return { max, size };
}
export interface AgentOsWorkerLoaderBackendOptions {
@@ -110,111 +180,303 @@ export interface AgentOsWorkerLoaderBackendOptions {
bundlePath(): string;
// Reports a failure off the request path, such as a failed prewarm.
onDiagnostic(message: string): void;
- poolSize: number;
+ onWarning(message: string): void;
+ // At least the slots of the DynamicWorkerPool, so every Worker with a slot gets a VM.
+ maxVms: number;
+ metrics: WorkerLoaderMetrics;
+ pool: WorkerLoaderPoolLimits & {
+ // How long a pool keeps VMs above `size` after its last peak.
+ idleMs: number;
+ };
}
export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend {
readonly #options: AgentOsWorkerLoaderBackendOptions;
+ // Every VM from its boot until its disposal finishes.
readonly #vms = new Set();
+ readonly #stopping = new Set();
+ // Opens that wait for room in the VM budget, first in line first. Each boots its VM in the
+ // callback, so no prewarm takes the room first.
+ readonly #vmWaiters: { boot(): void; fail(error: Error): void }[] = [];
+ #warnedVms = false;
readonly #children = new Set<{ exited: Promise; kill(): void }>();
readonly #pools = new Map();
- // The one prewarm in flight. Prewarms run one at a time, since every VM
- // shares one agentOS sidecar with the dynamic Workers that are running.
- #warming: Promise | undefined;
+ readonly #warming = new Set>();
+ // Dynamic Workers that hold back the refill, since a prewarm that runs
+ // beside them slows them about 3x. A Worker holds it while it loads. A
+ // Worker that loads beside another holds it until its first calls finish
+ // too, so a burst's first calls never share the sidecar with its refill.
+ readonly #holds = new Set<{ beside: boolean }>();
+ // A moving average of how long a prewarm takes.
+ #prewarmMs = initialPrewarmMs;
+ // Claim times across both modes, oldest first, within one prewarm time plus the rate window.
+ readonly #recentClaims: number[] = [];
+ // VMs that boot for a load that found no prewarmed VM.
+ #bootingOnDemand = 0;
#closed = false;
constructor(options: AgentOsWorkerLoaderBackendOptions) {
this.#options = options;
+ options.metrics.vmGauges = () => this.#gauges();
// Each VM owns a sidecar process, so stop every child and dispose the
// VMs when the runtime stops.
onRuntimeClose(() => this.#close());
}
+ #gauges(): VmGauges {
+ let pooled = 0;
+ const poolReady: Record = {};
+ for (const pool of this.#pools.values()) {
+ pooled += pool.ready.length;
+ poolReady[pool.mode] = pool.ready.length;
+ }
+ const booting = this.#warming.size + this.#bootingOnDemand;
+ const stopping = this.#stopping.size;
+ return {
+ maxVms: this.#options.maxVms,
+ poolReady,
+ waiting: this.#vmWaiters.length,
+ vms: { booting, live: Math.max(0, this.#vms.size - pooled - booting - stopping), pooled, stopping },
+ };
+ }
+
async open(network: boolean): Promise {
if (this.#closed) throw new Error("The Worker Loader has stopped.");
const mode: NetworkMode = network ? "network" : "deny";
let pool = this.#pools.get(mode);
if (pool === undefined) {
- pool = { mode, ready: [], waiting: undefined, warming: false };
+ pool = { claims: [], failed: false, mode, peak: 0, ready: [], shrink: undefined, waiting: [], warming: 0 };
this.#pools.set(mode, pool);
}
+ const hold = { beside: this.#holds.size > 0 };
+ for (const other of this.#holds) other.beside = true;
+ this.#holds.add(hold);
+ const release = (): void => {
+ if (!this.#holds.delete(hold)) return;
+ clearTimeout(timer);
+ this.#refill();
+ };
+ // A Worker whose load hangs must not stop the refill for good.
+ const timer = setTimeout(release, maxRefillHoldMs);
+ timer.unref();
+ const hooks: LoadHooks = {
+ loaded: () => {
+ if (!hold.beside) release();
+ },
+ settled: release,
+ };
+ try {
+ return await this.#openVm(pool, hooks);
+ } catch (error: unknown) {
+ release();
+ throw error;
+ }
+ }
+
+ async #openVm(pool: VmPool, hooks: LoadHooks): Promise {
// A ready VM serves exactly one dynamic Worker and is disposed with it.
- const ready = await this.#claim(pool);
+ const claim = this.#claim(pool);
+ this.#refill();
+ const ready = await claim;
+ this.#options.metrics.claim(
+ pool.mode,
+ ready === undefined ? "miss" : claim instanceof Promise ? "warming" : "hit",
+ );
if (this.#closed) {
if (ready !== undefined) await this.#discard(ready);
throw new Error("The Worker Loader has stopped.");
}
- const loaded = (): void => {
- setTimeout(() => this.#refill(), refillDelayMs).unref();
- };
if (ready !== undefined) {
const child = Promise.resolve(ready.child);
// The child is running even if the Worker fails before it is used.
- return this.#host(() => this.#discard(ready), ready.booted, ready.root, () => child, loaded);
+ return this.#host(() => this.#discard(ready), ready.booted, ready.root, () => child, hooks);
}
- const owned = this.#boot(mode);
+ let owned: OwnedVm | undefined;
let booted: BootedVm;
try {
- booted = await owned.booted;
+ owned = await this.#bootWithinBudget(pool.mode);
+ this.#bootingOnDemand += 1;
+ try {
+ booted = await owned.booted;
+ } finally {
+ this.#bootingOnDemand -= 1;
+ }
} catch (error: unknown) {
- await owned.dispose();
+ await owned?.dispose();
throw bootError(error);
}
+ const bootedVm = owned;
const root = posix.join(workersRoot, randomUUID());
- return this.#host(() => owned.dispose(), booted, root, () => this.#spawn(booted.vm, root, owned), loaded);
+ return this.#host(() => bootedVm.dispose(), booted, root, () => this.#spawn(booted.vm, root, bootedVm), hooks);
}
- // A ready VM, or the prewarm in flight if no other open waits for it, since
- // it finishes sooner than a new boot. Otherwise the open boots its own VM.
+ // Boots a VM for a load once the VM budget has room. At the budget, prewarmed VMs stop to
+ // make room, and the load waits for a VM that stops.
+ #bootWithinBudget(mode: NetworkMode): Promise {
+ if (this.#closed) return Promise.reject(new Error("The Worker Loader has stopped."));
+ if (this.#vmWaiters.length === 0 && this.#vms.size < this.#options.maxVms) {
+ return Promise.resolve(this.#boot(mode));
+ }
+ return new Promise((resolveBoot, rejectBoot) => {
+ this.#vmWaiters.push({ boot: () => resolveBoot(this.#boot(mode)), fail: rejectBoot });
+ this.#makeRoom();
+ });
+ }
+
+ // Stops prewarmed VMs, from the fullest pool first, while more loads wait for room than the
+ // VMs that already stop will free.
+ #makeRoom(): void {
+ let short = this.#vmWaiters.length - this.#stopping.size;
+ while (short > 0) {
+ let fullest: VmPool | undefined;
+ for (const pool of this.#pools.values()) {
+ if (pool.ready.length > (fullest?.ready.length ?? 0)) fullest = pool;
+ }
+ const ready = fullest?.ready.shift();
+ if (ready === undefined) return;
+ void this.#discard(ready);
+ short -= 1;
+ }
+ }
+
+ // A VM finished stopping, so a load that waits for room may boot.
+ #vmFreed(): void {
+ if (this.#closed) {
+ for (const waiter of this.#vmWaiters.splice(0)) waiter.fail(new Error("The Worker Loader has stopped."));
+ }
+ while (this.#vmWaiters.length > 0 && this.#vms.size < this.#options.maxVms) {
+ this.#vmWaiters.shift()?.boot();
+ }
+ if (this.#vms.size < Math.ceil(this.#options.maxVms * warnRatio)) this.#warnedVms = false;
+ this.#refill();
+ }
+
+ // A ready VM, or a prewarm in flight that no other open waits for, since it
+ // finishes sooner than a new boot. Otherwise the open boots its own VM at
+ // once, beside any other boot.
#claim(pool: VmPool): ReadyVm | Promise | undefined {
+ this.#recordClaim(pool);
const ready = pool.ready.shift();
- if (ready !== undefined || !pool.warming || pool.waiting !== undefined) return ready;
- return new Promise((resolveReady) => {
- pool.waiting = resolveReady;
- });
+ if (ready !== undefined || pool.warming <= pool.waiting.length) return ready;
+ return new Promise((resolveReady) => pool.waiting.push(resolveReady));
+ }
+
+ // Raises the pool's target to the claims seen within one prewarm time, so
+ // the refill keeps up with a steady stream of loads and the next burst of
+ // the same size finds its VMs ready.
+ #recordClaim(pool: VmPool): void {
+ const { idleMs, max } = this.#options.pool;
+ pool.failed = false;
+ if (max === 0) return;
+ const now = performance.now();
+ this.#recentClaims.push(now);
+ const oldest = now - this.#prewarmMs - claimRateWindowMs;
+ while (this.#recentClaims.length > maxRecentClaims || (this.#recentClaims[0] ?? now) < oldest) {
+ this.#recentClaims.shift();
+ }
+ pool.claims.push(now);
+ if (pool.claims.length > max) pool.claims.shift();
+ const recent = pool.claims.filter((at) => now - at <= this.#prewarmMs).length;
+ if (recent < pool.peak) return;
+ pool.peak = recent;
+ clearTimeout(pool.shrink);
+ pool.shrink = setTimeout(() => {
+ pool.peak = 0;
+ pool.shrink = undefined;
+ const excess = pool.ready.splice(0, Math.max(0, pool.ready.length - this.#target(pool)));
+ for (const ready of excess) void this.#discard(ready);
+ }, idleMs).unref();
+ }
+
+ #target(pool: VmPool): number {
+ const { max, size } = this.#options.pool;
+ return Math.min(max, Math.max(size, pool.peak));
}
- // Prewarms one VM for the first pool below its size, then the next. A
- // failed prewarm is reported and stops the refill until the next Worker
- // loads, so a VM that cannot boot never loops in the background.
+ // Ready VMs, plus prewarms in flight that no open waits for.
+ #supply(pool: VmPool): number {
+ return pool.ready.length + pool.warming - pool.waiting.length;
+ }
+
+ // How many prewarms may run now. With no dynamic Worker loading, `maxParallelPrewarms`. While
+ // Workers load, a prewarm beside them slows them, so only a steady stream of claims earns
+ // prewarms: as many as keep up with its rate. A steady stream claims VMs both within the last
+ // prewarm time and in the second before it; a burst does only once, so it never refills
+ // beside its own loads.
+ #refillLimit(): number {
+ if (this.#holds.size === 0) return maxParallelPrewarms;
+ const now = performance.now();
+ let recent = 0;
+ let earlier = 0;
+ for (const at of this.#recentClaims) {
+ const age = now - at;
+ if (age <= this.#prewarmMs) recent += 1;
+ else if (age <= this.#prewarmMs + claimRateWindowMs) earlier += 1;
+ }
+ const ratePerMs = Math.min(recent / this.#prewarmMs, earlier / claimRateWindowMs);
+ return Math.min(maxParallelPrewarms, Math.ceil(ratePerMs * this.#prewarmMs));
+ }
+
+ // Starts prewarms for the pools furthest below their target, up to the refill limit at once,
+ // while the VM budget has room and no load waits for it.
#refill(): void {
- if (this.#closed || this.#warming !== undefined) return;
- const pool = [...this.#pools.values()].find((entry) => entry.ready.length < this.#options.poolSize);
- if (pool === undefined) return;
- pool.warming = true;
- this.#warming = this.#prewarm(pool.mode).then(
- (ready) => {
- this.#warming = undefined;
- pool.warming = false;
- this.#offer(pool, ready);
- this.#refill();
- },
- (error: unknown) => {
- this.#warming = undefined;
- pool.warming = false;
- this.#takeWaiting(pool)?.(undefined);
- // Closing disposes a booting VM, which fails its prewarm on purpose.
- if (this.#closed) return;
- this.#options.onDiagnostic(
- `OpenOrange cannot prewarm an agentOS VM for dynamic Workers (${pool.mode}): ${errorMessage(error)}. Dynamic Workers boot their VM on demand instead.`,
- );
- },
- );
+ while (
+ !this.#closed &&
+ this.#warming.size < this.#refillLimit() &&
+ this.#vmWaiters.length === 0 &&
+ this.#vms.size < this.#options.maxVms
+ ) {
+ let neediest: VmPool | undefined;
+ let missing = 0;
+ for (const pool of this.#pools.values()) {
+ const gap = this.#target(pool) - this.#supply(pool);
+ if (!pool.failed && gap > missing) {
+ neediest = pool;
+ missing = gap;
+ }
+ }
+ if (neediest === undefined) return;
+ this.#startPrewarm(neediest);
+ }
}
- #takeWaiting(pool: VmPool): ((ready: ReadyVm | undefined) => void) | undefined {
- const waiting = pool.waiting;
- pool.waiting = undefined;
- return waiting;
+ #startPrewarm(pool: VmPool): void {
+ pool.warming += 1;
+ const started = performance.now();
+ const warming = this.#prewarm(pool.mode)
+ .then(
+ (ready) => {
+ pool.warming -= 1;
+ this.#prewarmMs += (performance.now() - started - this.#prewarmMs) / 4;
+ this.#offer(pool, ready);
+ },
+ (error: unknown) => {
+ pool.warming -= 1;
+ // An open that waited for this prewarm boots its own VM.
+ if (pool.waiting.length > pool.warming) pool.waiting.pop()?.(undefined);
+ // Closing disposes a booting VM, which fails its prewarm on purpose.
+ if (this.#closed || pool.failed) return;
+ pool.failed = true;
+ this.#options.onDiagnostic(
+ `OpenOrange cannot prewarm an agentOS VM for dynamic Workers (${pool.mode}): ${errorMessage(error)}. Dynamic Workers boot their VM on demand instead.`,
+ );
+ },
+ )
+ .finally(() => {
+ this.#warming.delete(warming);
+ this.#refill();
+ });
+ this.#warming.add(warming);
}
#offer(pool: VmPool, ready: ReadyVm): void {
- const waiting = this.#takeWaiting(pool);
+ const waiting = pool.waiting.shift();
if (waiting !== undefined) {
waiting(ready);
return;
}
- if (this.#closed) {
+ // The target may have shrunk while the VM booted, or a load may wait for its room.
+ if (this.#closed || pool.ready.length >= this.#target(pool) || this.#vmWaiters.length > this.#stopping.size) {
void this.#discard(ready);
return;
}
@@ -238,11 +500,11 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend {
booted: BootedVm,
root: string,
spawn: () => Promise,
- loaded: () => void,
+ hooks: LoadHooks,
): WorkerHost {
const { moduleRewrites, vm } = booted;
return {
- loaded,
+ ...hooks,
moduleRewrites,
remove,
root,
@@ -261,10 +523,18 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend {
}
async #spawn(vm: AgentOs, root: string, owned: OwnedVm): Promise {
- const child = await spawnVmChild(vm, root, {
+ const spawned = await spawnVmChild(vm, root, {
dispose: () => owned.dispose(),
report: (message) => this.#options.onDiagnostic(message),
});
+ // A VM counts as stopping from the kill on, so room that it will free is not made twice.
+ const child: VmChild = {
+ ...spawned,
+ kill: () => {
+ this.#stopping.add(owned);
+ spawned.kill();
+ },
+ };
const entry = { exited: child.exited, kill: () => child.kill() };
this.#children.add(entry);
void child.exited.then(() => this.#children.delete(entry));
@@ -295,13 +565,18 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend {
}
#boot(mode: NetworkMode): OwnedVm {
+ const startedAt = performance.now();
const booted = this.#create(mode);
+ void booted.then(
+ () => this.#options.metrics.vmBoot.observe((performance.now() - startedAt) / 1_000),
+ () => undefined,
+ );
let disposed: Promise | undefined;
const owned: OwnedVm = {
booted,
dispose: () => {
disposed ??= (async () => {
- this.#vms.delete(owned);
+ this.#stopping.add(owned);
// A VM that failed to boot has nothing to dispose, and its boot error reaches the
// caller that booted it.
const vm = await booted.then(({ vm: value }) => value).catch(() => undefined);
@@ -310,11 +585,21 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend {
`OpenOrange cannot dispose the agentOS VM of a dynamic Worker: ${errorMessage(error)}`,
);
});
+ this.#stopping.delete(owned);
+ this.#vms.delete(owned);
+ this.#vmFreed();
})();
return disposed;
},
};
this.#vms.add(owned);
+ const { maxVms } = this.#options;
+ if (!this.#warnedVms && this.#vms.size >= Math.ceil(maxVms * warnRatio)) {
+ this.#warnedVms = true;
+ this.#options.onWarning(
+ `${this.#vms.size} of ${maxVms} agentOS VMs for dynamic Workers are in use, counting running Workers, prewarmed VMs, and VMs that boot or stop. At the limit, prewarmed VMs stop first, and a new Worker waits for a VM to stop. Raise ${workerLoaderMaxVmsSetting} to allow more.`,
+ );
+ }
return owned;
}
@@ -330,12 +615,18 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend {
return [specifier, posix.join(runtimeRoot, file)];
}),
);
- const { AgentOs, hostDirMount } = await import("@rivet-dev/agentos-core");
+ const { AgentOs, agentOsOptionFieldSchemas, hostDirMount } = await import("@rivet-dev/agentos-core");
+ // agentOS boots each child from a snapshot of the runtime that the
+ // entry imports, built once per sidecar. agentOS 0.4.8 lacks the option
+ // and rejects options it does not know, so it is set only when present.
+ const snapshot =
+ "snapshotEntrypoints" in agentOsOptionFieldSchemas ? { snapshotEntrypoints: [childEntry] } : {};
// The guest reads Orange's runtime from the host, read-only, so a VM
// boots without a copy of it. The mount shows only the bundle directory.
const vm = await AgentOs.create({
mounts: [hostDirMount(runtimeRoot, bundlePath, { readOnly: true })],
permissions: { network: mode === "network" ? "allow" : "deny" },
+ ...snapshot,
});
return { moduleRewrites, vm };
}
@@ -345,10 +636,14 @@ export class AgentOsWorkerLoaderBackend implements WorkerLoaderBackend {
// A prewarm that is booting finishes or fails first, so none starts a
// child after the children below are stopped.
await Promise.race([
- this.#warming,
+ Promise.all(this.#warming),
new Promise((resolveWait) => setTimeout(resolveWait, exitWaitMs).unref()),
]);
- for (const pool of this.#pools.values()) this.#takeWaiting(pool)?.(undefined);
+ for (const pool of this.#pools.values()) {
+ clearTimeout(pool.shrink);
+ for (const waiting of pool.waiting.splice(0)) waiting(undefined);
+ }
+ for (const waiter of this.#vmWaiters.splice(0)) waiter.fail(new Error("The Worker Loader has stopped."));
// Disposing a VM with a running process fails to observe its exit, so
// stop the children first.
const children = [...this.#children];
@@ -373,7 +668,7 @@ async function spawnVmChild(vm: AgentOs, root: string, options: VmChildOptions):
let exitReason: string | undefined;
let killing = false;
let forceExit: (reason: string) => void = () => undefined;
- const descriptor = await vm.process.spawn("node", [posix.join(runtimeRoot, "worker-loader-child.mjs")], {
+ const descriptor = await vm.process.spawn("node", [childEntry], {
cwd: root,
env: { HOME: "/home/user", PATH: "/usr/local/bin:/usr/bin:/bin" },
onStderr: (chunk) => process.stderr.write(chunk),
@@ -409,8 +704,33 @@ async function spawnVmChild(vm: AgentOs, root: string, options: VmChildOptions):
stdout.end();
for (const listener of exitListeners.splice(0)) listener(exitReason ?? "unknown");
});
- // The VM takes stdin writes in order only if each waits for the last.
- let writes = Promise.resolve();
+ // The VM takes stdin writes in order only if each waits for the last. Frames written in one
+ // turn of the event loop, or while the last write is in flight, go in one write of at most
+ // `maxStdinWriteBytes`.
+ let unsent: string[] = [];
+ let writing = false;
+ const writeUnsent = async (): Promise => {
+ while (unsent.length > 0 && exitReason === undefined) {
+ let bytes = 0;
+ let count = 0;
+ for (const frame of unsent) {
+ const frameBytes = Buffer.byteLength(frame);
+ if (count > 0 && bytes + frameBytes > maxStdinWriteBytes) break;
+ bytes += frameBytes;
+ count += 1;
+ }
+ try {
+ await vm.process.writeStdin(pid, unsent.splice(0, count).join(""));
+ } catch (error: unknown) {
+ // Writes fail once the process exits, and its exit fails its calls.
+ if (exitReason === undefined && !killing) {
+ options.report(`OpenOrange cannot write to dynamic Worker process ${pid}: ${errorMessage(error)}`);
+ }
+ }
+ }
+ unsent = [];
+ writing = false;
+ };
return {
exited: done,
kill(): void {
@@ -436,14 +756,10 @@ async function spawnVmChild(vm: AgentOs, root: string, options: VmChildOptions):
stdout,
write(text): void {
if (exitReason !== undefined) return;
- writes = writes
- .then(() => vm.process.writeStdin(pid, text))
- .catch((error: unknown) => {
- // Writes fail once the process exits, and its exit fails its calls.
- if (exitReason === undefined && !killing) {
- options.report(`OpenOrange cannot write to dynamic Worker process ${pid}: ${errorMessage(error)}`);
- }
- });
+ unsent.push(text);
+ if (writing) return;
+ writing = true;
+ setImmediate(() => void writeUnsent());
},
};
}
diff --git a/packages/runtime-node/src/bindings/worker-loader/child.ts b/packages/runtime-node/src/bindings/worker-loader/child.ts
index 439ae92..66082b3 100644
--- a/packages/runtime-node/src/bindings/worker-loader/child.ts
+++ b/packages/runtime-node/src/bindings/worker-loader/child.ts
@@ -3,7 +3,7 @@
// env, modules, and outbound access that the parent passed.
import { writeSync } from "node:fs";
-import { fileURLToPath, pathToFileURL } from "node:url";
+import { pathToFileURL } from "node:url";
import { DurableObjectRpcTarget } from "../durable-objects/rpc/target.js";
import type { Request, Response } from "../../globals/http.js";
@@ -92,11 +92,20 @@ function startupError(error: unknown, start: DynamicWorkerStart | undefined): un
class DynamicWorkerHost extends DurableObjectRpcTarget {
#worker: LoadedWorkerEntrypoint | undefined;
+ // The first `start`. The parent pipelines its entrypoint calls behind it.
+ #started: Promise | undefined;
- async start(envelope: unknown): Promise {
+ start(envelope: unknown): Promise {
+ if (this.#started !== undefined) {
+ return Promise.resolve(encodeLoaderError(new Error("Dynamic Worker already started")));
+ }
+ this.#started = this.#load(envelope);
+ return this.#started;
+ }
+
+ async #load(envelope: unknown): Promise {
let start: DynamicWorkerStart | undefined;
try {
- if (this.#worker !== undefined) throw new Error("Dynamic Worker already started");
start = parseDynamicWorkerStart(decodeLoaderValue(envelope, true));
const { bundleRoot, mainFile, moduleRoot, outbound } = start;
// node:fs reads `/bundle` from the dynamic Worker's own modules.
@@ -122,8 +131,11 @@ class DynamicWorkerHost extends DurableObjectRpcTarget {
}
}
- entrypoint(name: unknown, props: unknown): LoaderCapabilityTarget {
- if (this.#worker === undefined) throw new Error("Dynamic Worker is not started");
+ async entrypoint(name: unknown, props: unknown): Promise {
+ if (this.#started === undefined) throw new Error("Dynamic Worker is not started");
+ await this.#started;
+ // The parent fails this call with the startup error.
+ if (this.#worker === undefined) throw new Error("Dynamic Worker failed to start");
if (name !== undefined && typeof name !== "string") {
throw new TypeError("Entrypoint name must be a string");
}
@@ -132,13 +144,34 @@ class DynamicWorkerHost extends DurableObjectRpcTarget {
}
}
+interface FrameWriter {
+ send(message: object): void;
+ // Returns the frames not yet written and forgets them.
+ take(): string;
+}
+
// Stdout carries the frames to the parent, so the Worker's own stdout writes
-// travel as frames too. The parent writes them to its stdout. Returns the
-// writer for every other frame.
-function frameStdoutWrites(): (message: object) => void {
+// travel as frames too. The parent writes them to its stdout. Frames sent in
+// one microtask checkpoint leave in one write, so the messages of a call reach
+// the parent together. Returns the writer for every other frame.
+function frameStdoutWrites(): FrameWriter {
const stdout = process.stdout;
- const frameStream = { write: stdout.write.bind(stdout) };
- const sendFrame = (message: object): void => writeFrame(frameStream, message);
+ const write = stdout.write.bind(stdout);
+ let pending = "";
+ const buffer = {
+ write(text: string): boolean {
+ if (pending === "") {
+ queueMicrotask(() => {
+ const frames = pending;
+ pending = "";
+ if (frames !== "") write(frames);
+ });
+ }
+ pending += text;
+ return true;
+ },
+ };
+ const sendFrame = (message: object): void => writeFrame(buffer, message);
stdout.write = (
chunk: string | Uint8Array,
encoding?: BufferEncoding | ((error?: Error | null) => void),
@@ -153,11 +186,23 @@ function frameStdoutWrites(): (message: object) => void {
if (done !== undefined) process.nextTick(done);
return true;
};
- return sendFrame;
+ return {
+ send: sendFrame,
+ take(): string {
+ const frames = pending;
+ pending = "";
+ return frames;
+ },
+ };
}
-if (process.argv[1] === fileURLToPath(import.meta.url)) {
- const sendFrame = frameStdoutWrites();
+// Runs the child. The thin entry `worker-loader-child-main.mjs` calls it, so
+// agentOS can boot the child from a snapshot that holds this module and the
+// whole runtime, evaluated once. Read per-process state such as `process.env`
+// in here or later, never at module top level.
+export function startChild(): void {
+ const frames = frameStdoutWrites();
+ const sendFrame = frames.send;
const report = (error: unknown): void => {
sendFrame({
openOrangeWorkerLoaderDiagnostic:
@@ -171,10 +216,15 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
// The dynamic Worker never outlives the runtime process that loaded it.
process.stdin.once("end", () => process.exit(0));
process.stdout.on("error", () => process.exit(0));
+ // Frames sent just before an exit still reach the parent.
+ process.on("exit", () => {
+ const unsent = frames.take();
+ if (unsent !== "") writeSync(1, unsent);
+ });
// abortIsolate() reports its reason and ends this process at once, so nothing else from the
// aborted isolate reaches the parent. The frame is written synchronously for that reason.
Reflect.set(globalThis, abortIsolateHook, (reason: string): never => {
- writeSync(1, `${JSON.stringify({ openOrangeWorkerLoaderAbort: reason })}\n`);
+ writeSync(1, `${frames.take()}${JSON.stringify({ openOrangeWorkerLoaderAbort: reason })}\n`);
process.exit(1);
});
const transport = new FramedRpcTransport(readFrames(process.stdin, () => undefined), sendFrame);
diff --git a/packages/runtime-node/src/bindings/worker-loader/loader.ts b/packages/runtime-node/src/bindings/worker-loader/loader.ts
index 99bbce1..6d99c11 100644
--- a/packages/runtime-node/src/bindings/worker-loader/loader.ts
+++ b/packages/runtime-node/src/bindings/worker-loader/loader.ts
@@ -22,6 +22,7 @@ import {
} from "./session.js";
import type { CapnWebStub } from "../durable-objects/rpc/capnweb.js";
import { errorMessage, isRecord } from "../unknown-values.js";
+import { WorkerLoaderMetrics } from "./metrics.js";
export interface WorkerFile {
content: string | Uint8Array;
@@ -47,6 +48,9 @@ export interface WorkerHost {
spawn(): Promise;
// Runs once the dynamic Worker has loaded or failed to load.
loaded(): void;
+ // Runs once the loaded dynamic Worker first has no call in flight, or when
+ // it fails to load.
+ settled(): void;
// Disposes the VM. Runs when the process exits or fails to start.
remove(): Promise;
}
@@ -214,97 +218,242 @@ interface LoaderSession {
export const maxWorkersSetting = "ORANGE_WORKER_LOADER_MAX_WORKERS";
export const idleTimeoutSetting = "ORANGE_WORKER_LOADER_IDLE_TIMEOUT_MS";
+export const queueTimeoutSetting = "ORANGE_WORKER_LOADER_QUEUE_TIMEOUT_MS";
+export const maxQueuedSetting = "ORANGE_WORKER_LOADER_MAX_QUEUED";
const defaultMaxWorkers = 32;
const defaultIdleTimeoutMs = 30_000;
+const defaultQueueTimeoutMs = 5_000;
+const defaultMaxQueued = 256;
const warnRatio = 0.8;
+const retryAfterSeconds = 1;
-// A new dynamic Worker needs a slot, and every running one holds a slot until it stops.
+export type WorkerLoaderLimitCode = "queue_full" | "queue_timeout";
+
+// A new dynamic Worker needs a slot, and every running one holds a slot until it stops. With every
+// slot busy, a new one waits in line; this error refuses it when the line is full or its wait
+// passes the queue timeout. Like the overloaded error of a Cloudflare Durable Object, it carries
+// `overloaded` and `retryable`, and the runtime answers it with 503 and Retry-After when the
+// Worker does not catch it.
+// https://developers.cloudflare.com/durable-objects/best-practices/error-handling/
export class WorkerLoaderLimitError extends RangeError {
- constructor(message: string) {
+ readonly code: WorkerLoaderLimitCode;
+ readonly overloaded = true;
+ readonly retryable = true;
+ readonly retryAfterSeconds = retryAfterSeconds;
+
+ constructor(code: WorkerLoaderLimitCode, message: string) {
super(message);
this.name = "WorkerLoaderLimitError";
+ this.code = code;
}
}
// Node.js runs a setTimeout() delay above this after 1 ms.
const maxTimerDelayMs = 2_147_483_647;
-function positiveIntegerSetting(
+export function positiveIntegerSetting(
settings: Readonly>,
name: string,
fallback: number,
max: number,
+ min = 1,
): number {
const raw = settings[name];
if (raw === undefined || raw.trim() === "") return fallback;
const value = Number(raw);
- if (!Number.isSafeInteger(value) || value < 1 || value > max) {
- throw new RangeError(`${name} must be an integer from 1 to ${max}. Got: ${JSON.stringify(raw)}`);
+ if (!Number.isSafeInteger(value) || value < min || value > max) {
+ throw new RangeError(`${name} must be an integer from ${min} to ${max}. Got: ${JSON.stringify(raw)}`);
}
return value;
}
+export function workerLoaderIdleTimeoutMs(settings: Readonly>): number {
+ return positiveIntegerSetting(settings, idleTimeoutSetting, defaultIdleTimeoutMs, maxTimerDelayMs);
+}
+
+export function workerLoaderMaxWorkers(settings: Readonly>): number {
+ return positiveIntegerSetting(settings, maxWorkersSetting, defaultMaxWorkers, Number.MAX_SAFE_INTEGER);
+}
+
+interface SlotWaiter {
+ grant(): void;
+ fail(error: Error): void;
+}
+
+export interface DynamicWorkerPoolLimits {
+ idleTimeoutMs: number;
+ maxQueued: number;
+ maxWorkers: number;
+ queueTimeoutMs: number;
+}
+
// Bounds the dynamic Workers that run at once across every Worker Loader binding of the runtime.
-// Each one is an agentOS VM and a Node.js process, about 54 MiB. At the limit, a new Worker stops
-// the least recently used idle one. A Worker with a call in flight is never stopped.
+// Each one is an agentOS VM and a Node.js process. At the limit, a new Worker stops the least
+// recently used idle one. A Worker with a call in flight is never stopped. With none idle, new
+// Workers wait in line, first come first served, until a slot frees or the queue timeout passes.
export class DynamicWorkerPool {
readonly idleTimeoutMs: number;
+ readonly maxQueued: number;
readonly maxWorkers: number;
+ readonly queueTimeoutMs: number;
+ readonly metrics: WorkerLoaderMetrics;
// Least recently used first.
readonly #live = new Set();
+ // First in line first.
+ readonly #queue = new Map();
readonly #warn: (message: string) => void;
- #warned = false;
+ #pumping = false;
+ #warnedLive = false;
+ #warnedQueue = false;
- constructor(limits: { idleTimeoutMs: number; maxWorkers: number }, warn: (message: string) => void) {
+ constructor(limits: DynamicWorkerPoolLimits, warn: (message: string) => void, metrics = new WorkerLoaderMetrics()) {
this.idleTimeoutMs = limits.idleTimeoutMs;
+ this.maxQueued = limits.maxQueued;
this.maxWorkers = limits.maxWorkers;
+ this.queueTimeoutMs = limits.queueTimeoutMs;
+ this.metrics = metrics;
+ metrics.slotGauges = () => ({ live: this.#live.size, maxWorkers: this.maxWorkers, queued: this.#queue.size });
this.#warn = warn;
}
static fromSettings(
settings: Readonly>,
warn: (message: string) => void,
+ metrics?: WorkerLoaderMetrics,
): DynamicWorkerPool {
return new DynamicWorkerPool(
{
- idleTimeoutMs: positiveIntegerSetting(settings, idleTimeoutSetting, defaultIdleTimeoutMs, maxTimerDelayMs),
- maxWorkers: positiveIntegerSetting(settings, maxWorkersSetting, defaultMaxWorkers, Number.MAX_SAFE_INTEGER),
+ idleTimeoutMs: workerLoaderIdleTimeoutMs(settings),
+ maxQueued: positiveIntegerSetting(settings, maxQueuedSetting, defaultMaxQueued, Number.MAX_SAFE_INTEGER, 0),
+ maxWorkers: workerLoaderMaxWorkers(settings),
+ queueTimeoutMs: positiveIntegerSetting(settings, queueTimeoutSetting, defaultQueueTimeoutMs, maxTimerDelayMs),
},
warn,
+ metrics,
);
}
- admit(worker: DynamicWorker): void {
- if (this.#live.size >= this.maxWorkers) {
- let evicted = false;
- for (const candidate of this.#live) {
- if (!candidate.idle) continue;
- candidate.evict();
- evicted = true;
- break;
+ get queued(): number {
+ return this.#queue.size;
+ }
+
+ // Resolves once `worker` holds a slot.
+ acquire(worker: DynamicWorker): Promise {
+ const queuedAt = performance.now();
+ return new Promise((resolveSlot, rejectSlot) => {
+ let timer: NodeJS.Timeout | undefined;
+ this.#queue.set(worker, {
+ fail: (error) => {
+ clearTimeout(timer);
+ rejectSlot(error);
+ },
+ grant: () => {
+ clearTimeout(timer);
+ this.metrics.queueWait.observe((performance.now() - queuedAt) / 1_000);
+ resolveSlot();
+ },
+ });
+ this.#pump();
+ if (!this.#queue.has(worker)) return;
+ // Only the Workers behind those that got a slot count toward the queue limit.
+ if (this.#queue.size > this.maxQueued) {
+ this.#queue.delete(worker);
+ rejectSlot(
+ this.#refuse(
+ "queue_full",
+ `OpenOrange cannot start another dynamic Worker: all ${this.maxWorkers} slots are busy and ${this.#queue.size} Workers already wait for one, the most ${maxQueuedSetting} allows. Retry later, or raise ${maxWorkersSetting} or ${maxQueuedSetting}.`,
+ ),
+ );
+ return;
}
- if (!evicted) {
- throw new WorkerLoaderLimitError(
- `OpenOrange cannot start another dynamic Worker: ${this.#live.size} are running, none is idle, and the limit is ${this.maxWorkers}. Raise ${maxWorkersSetting} or let calls finish first.`,
+ if (!this.#warnedQueue && this.#queue.size >= Math.ceil(this.maxQueued * warnRatio)) {
+ this.#warnedQueue = true;
+ this.#warn(
+ `${this.#queue.size} dynamic Workers wait for a slot, and past ${this.maxQueued} new ones fail with WorkerLoaderLimitError. Raise ${maxWorkersSetting} or ${maxQueuedSetting}, or add runners.`,
);
}
- }
- this.#live.add(worker);
- if (!this.#warned && this.#live.size >= Math.ceil(this.maxWorkers * warnRatio)) {
- this.#warned = true;
- this.#warn(
- `${this.#live.size} of ${this.maxWorkers} dynamic Workers are running. At the limit, a new one stops the least recently used idle one, or fails with WorkerLoaderLimitError if none is idle. Raise ${maxWorkersSetting} to allow more.`,
- );
- }
+ timer = setTimeout(() => {
+ if (!this.#queue.delete(worker)) return;
+ this.#queueShrank();
+ rejectSlot(
+ this.#refuse(
+ "queue_timeout",
+ `OpenOrange cannot start another dynamic Worker: it waited ${this.queueTimeoutMs} ms for one of ${this.maxWorkers} busy slots. Retry later, or raise ${maxWorkersSetting} or ${queueTimeoutSetting}.`,
+ ),
+ );
+ }, this.queueTimeoutMs);
+ timer.unref();
+ });
+ }
+
+ // Takes a Worker that stops before it gets a slot out of line.
+ cancel(worker: DynamicWorker, error: Error): void {
+ const waiter = this.#queue.get(worker);
+ if (waiter === undefined) return;
+ this.#queue.delete(worker);
+ this.#queueShrank();
+ waiter.fail(error);
}
touch(worker: DynamicWorker): void {
if (this.#live.delete(worker)) this.#live.add(worker);
}
+ // A Worker may have become idle, so a Worker in line may take its slot.
+ idle(): void {
+ if (this.#queue.size > 0) this.#pump();
+ }
+
remove(worker: DynamicWorker): void {
this.#live.delete(worker);
- if (this.#live.size < Math.ceil(this.maxWorkers * warnRatio)) this.#warned = false;
+ if (this.#live.size < Math.ceil(this.maxWorkers * warnRatio)) this.#warnedLive = false;
+ this.#pump();
+ }
+
+ #refuse(code: WorkerLoaderLimitCode, message: string): WorkerLoaderLimitError {
+ this.metrics.reject(code);
+ return new WorkerLoaderLimitError(code, message);
+ }
+
+ #queueShrank(): void {
+ if (this.#queue.size < Math.ceil(this.maxQueued * warnRatio)) this.#warnedQueue = false;
+ }
+
+ // Hands free slots to Workers in line, in order. Stopping an idle Worker frees its slot, and
+ // its removal must not start a second pass.
+ #pump(): void {
+ if (this.#pumping) return;
+ this.#pumping = true;
+ try {
+ for (const [worker, waiter] of this.#queue) {
+ if (!this.#freeSlot()) break;
+ this.#queue.delete(worker);
+ this.#live.add(worker);
+ this.#warnLive();
+ waiter.grant();
+ }
+ this.#queueShrank();
+ } finally {
+ this.#pumping = false;
+ }
+ }
+
+ #freeSlot(): boolean {
+ if (this.#live.size < this.maxWorkers) return true;
+ for (const candidate of this.#live) {
+ if (!candidate.idle) continue;
+ candidate.evict();
+ return this.#live.size < this.maxWorkers;
+ }
+ return false;
+ }
+
+ #warnLive(): void {
+ if (this.#warnedLive || this.#live.size < Math.ceil(this.maxWorkers * warnRatio)) return;
+ this.#warnedLive = true;
+ this.#warn(
+ `${this.#live.size} of ${this.maxWorkers} dynamic Workers are running. At the limit, a new one stops the least recently used idle one, or waits up to ${this.queueTimeoutMs} ms for a slot and then fails with WorkerLoaderLimitError. Raise ${maxWorkersSetting} to allow more.`,
+ );
}
}
@@ -333,10 +482,18 @@ class DynamicWorker {
#baseExports = 0;
#evicted = false;
#exited = false;
+ #idleCheck = false;
#idleTimer: NodeJS.Timeout | undefined;
#inFlight = 0;
+ // The host of a Worker that has loaded and whose first calls are in flight.
+ #settling: WorkerHost | undefined;
#lastUsed = 0;
#orphaned = false;
+ // The session's main stub once `start` is sent. Entrypoint calls pipeline behind `start`, so a
+ // new Worker's first call reaches its process together with `start` instead of after its reply.
+ readonly #sent: Promise;
+ #markSent: (main: CapnWebStub) => void = () => undefined;
+ #failSent: (error: unknown) => void = () => undefined;
#session: LoaderSession | undefined;
#terminated = false;
@@ -350,6 +507,11 @@ class DynamicWorker {
this.#label = label;
this.#onExit = onExit;
this.#options = options;
+ this.#sent = new Promise((resolveSent, rejectSent) => {
+ this.#markSent = resolveSent;
+ this.#failSent = rejectSent;
+ });
+ this.#sent.catch(() => undefined);
}
start(): void {
@@ -362,7 +524,7 @@ class DynamicWorker {
return this.#running ?? Promise.resolve();
}
- // Stopped by the pool, so a held stub may load the Worker again.
+ // Stopped by the pool or refused a slot, so a held stub may load the Worker again.
get evicted(): boolean {
return this.#evicted;
}
@@ -376,7 +538,16 @@ class DynamicWorker {
async #start(): Promise {
let host: WorkerHost | undefined;
+ const { pool } = this.#options;
try {
+ try {
+ await pool.acquire(this);
+ } catch (error: unknown) {
+ // A stub that is still held asks for a slot again on its next call.
+ if (error instanceof WorkerLoaderLimitError) this.#evicted = true;
+ throw error;
+ }
+ const startedAt = performance.now();
const prepared = await this.#getCode();
host = await this.#options.backend.open(prepared.outbound === "network");
const bundleRoot = posix.join(host.root, "bundle");
@@ -404,6 +575,7 @@ class DynamicWorker {
if (this.#terminated) child.kill();
const frames = readFrames(child.stdout, (line) => process.stdout.write(`${line}\n`));
frames.on("message", (message: unknown) => {
+ this.#checkIdle();
if (typeof message !== "object" || message === null) return;
const stdout: unknown = Reflect.get(message, "openOrangeWorkerLoaderStdout");
if (typeof stdout === "string") process.stdout.write(stdout);
@@ -415,6 +587,7 @@ class DynamicWorker {
}
});
const transport = new FramedRpcTransport(frames, (message) => {
+ this.#checkIdle();
child.write(`${JSON.stringify(message)}\n`);
});
const session = openLoaderSession(transport);
@@ -427,14 +600,23 @@ class DynamicWorker {
this.#onExit();
void removeHost.remove();
});
- const reply = await session.main.start(envelope);
- if (reply !== undefined && reply !== null) decodeLoaderReply(reply);
+ const started = session.main.start(envelope);
+ // The envelope's capabilities are exported once `start` is sent, before any other call.
this.#baseExports = session.stats().exports;
+ this.#markSent(session.main);
+ const reply = await started;
+ if (reply !== undefined && reply !== null) decodeLoaderReply(reply);
this.#session = session;
this.#lastUsed = performance.now();
- this.#armIdleTimer(this.#options.pool.idleTimeoutMs);
+ pool.metrics.workerStart.observe((this.#lastUsed - startedAt) / 1_000);
+ this.#armIdleTimer(pool.idleTimeoutMs);
+ this.#checkIdle();
+ if (this.#inFlight === 0) host.settled();
+ else this.#settling = host;
return session;
} catch (error: unknown) {
+ this.#failSent(error);
+ host?.settled();
this.terminate();
// A running child frees the slot when it exits. Without one, nothing else frees it.
if (this.#child === undefined) {
@@ -453,6 +635,18 @@ class DynamicWorker {
}
}
+ // A message in either direction can end the last call or stream, which leaves the Worker
+ // idle. Only Workers in line for a slot need to know at once, so the check runs only then,
+ // at most once per turn of the event loop.
+ #checkIdle(): void {
+ if (this.#idleCheck || this.#options.pool.queued === 0) return;
+ this.#idleCheck = true;
+ setImmediate(() => {
+ this.#idleCheck = false;
+ if (this.idle) this.#options.pool.idle();
+ });
+ }
+
// Like workerd, an isolate that abortIsolate() ended fails its calls with an internal error,
// and the reason goes only to the log under the same reference.
#exitError(reason: string): Error {
@@ -486,15 +680,24 @@ class DynamicWorker {
const settle = (): void => {
this.#inFlight -= 1;
this.#lastUsed = performance.now();
+ if (this.#inFlight === 0) {
+ this.#settling?.settled();
+ this.#settling = undefined;
+ }
if (this.#orphaned && this.idle) this.evict();
+ else this.#checkIdle();
};
void call.then(settle, settle);
return call;
}
- // No stub references this unnamed Worker any more, so it stops once idle.
+ // No stub references this unnamed Worker any more, so it stops once idle. A Worker in line for
+ // a slot leaves the line, unless a call waits on it: the call holds no stub.
release(): void {
this.#orphaned = true;
+ if (this.#inFlight === 0) {
+ this.#options.pool.cancel(this, new Error(`Dynamic Worker ${this.#label} has no stub left.`));
+ }
if (this.idle) this.evict();
}
@@ -509,15 +712,16 @@ class DynamicWorker {
// A capability for `getEntrypoint(name, { props })`.
entrypoint(name: string | undefined, props: unknown): Promise {
- const running = this.#running ?? Promise.reject(new Error("Dynamic Worker is not started"));
+ const running = this.#running;
+ if (running === undefined) return Promise.reject(new Error("Dynamic Worker is not started"));
const propsEnvelope = props === undefined ? undefined : encodeLoaderValue(props);
- const remote = running.then(async ({ main }) => {
- const stub = await main.entrypoint(name, propsEnvelope);
- if ((typeof stub !== "object" || stub === null) && typeof stub !== "function") {
- throw new Error("Dynamic Worker returned an invalid entrypoint");
- }
+ const remote = this.#sent.then((main) => {
+ // A promise for the entrypoint, so calls on it pipeline too. The process resolves it once
+ // the Worker has loaded. A call settles only after `start` does, so a Worker that fails to
+ // load fails its calls with the startup error.
+ const stub: object = main.entrypoint(name, propsEnvelope);
this.#entrypointImports += 1;
- return new RemoteCapability(stub);
+ return new RemoteCapability(stub, running);
});
remote.catch(() => undefined);
return remote;
@@ -530,6 +734,7 @@ class DynamicWorker {
capability.dispose();
this.#entrypointImports -= 1;
if (this.#orphaned && this.idle) this.evict();
+ else this.#checkIdle();
},
() => undefined,
);
@@ -537,6 +742,7 @@ class DynamicWorker {
terminate(): void {
this.#terminated = true;
+ this.#options.pool.cancel(this, new Error("The Worker Loader has stopped."));
if (this.#child !== undefined && !this.#exited) this.#child.kill();
}
}
@@ -649,7 +855,8 @@ export class WorkerStub {
constructor(handle: WorkerHandle) {
this.#handle = handle;
- // The Worker starts loading at once, and a full pool throws here.
+ // The Worker starts loading at once. With every slot busy, it waits in line, and its calls
+ // wait with it.
handle.acquire();
handle.hold(this);
}
@@ -759,7 +966,6 @@ export class WorkerLoader {
onExit();
},
);
- this.#options.pool.admit(worker);
this.#all.add(worker);
worker.start();
return worker;
diff --git a/packages/runtime-node/src/bindings/worker-loader/metrics.ts b/packages/runtime-node/src/bindings/worker-loader/metrics.ts
new file mode 100644
index 0000000..2ccb059
--- /dev/null
+++ b/packages/runtime-node/src/bindings/worker-loader/metrics.ts
@@ -0,0 +1,175 @@
+// Prometheus metrics for dynamic Workers. The runtime appends them to the runner's
+// metrics endpoint, so an autoscaler can scale on slot use and queue wait.
+
+import { readdirSync, readFileSync } from "node:fs";
+
+// Seconds. Covers a warm claim (about 10 ms) up to the default queue timeout.
+const secondsBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30];
+
+class Histogram {
+ readonly #counts = secondsBuckets.map(() => 0);
+ #count = 0;
+ #sum = 0;
+
+ observe(seconds: number): void {
+ this.#count += 1;
+ this.#sum += seconds;
+ for (const [index, bound] of secondsBuckets.entries()) {
+ if (seconds <= bound) this.#counts[index] = (this.#counts[index] ?? 0) + 1;
+ }
+ }
+
+ render(name: string, labels = ""): string[] {
+ const join = labels === "" ? "" : `${labels},`;
+ const lines = secondsBuckets.map((bound, index) => `${name}_bucket{${join}le="${bound}"} ${this.#counts[index] ?? 0}`);
+ lines.push(`${name}_bucket{${join}le="+Inf"} ${this.#count}`);
+ const suffix = labels === "" ? "" : `{${labels}}`;
+ lines.push(`${name}_sum${suffix} ${this.#sum}`, `${name}_count${suffix} ${this.#count}`);
+ return lines;
+ }
+}
+
+export interface SlotGauges {
+ live: number;
+ maxWorkers: number;
+ queued: number;
+}
+
+export interface VmGauges {
+ vms: { booting: number; live: number; pooled: number; stopping: number };
+ maxVms: number;
+ poolReady: Record;
+ // Loads that wait for room in the VM budget.
+ waiting: number;
+}
+
+// One instance per runtime. The slot pool and the agentOS backend record into it.
+export class WorkerLoaderMetrics {
+ readonly queueWait = new Histogram();
+ readonly workerStart = new Histogram();
+ readonly vmBoot = new Histogram();
+ readonly #claims = new Map();
+ readonly #rejections = new Map();
+ slotGauges: (() => SlotGauges) | undefined;
+ vmGauges: (() => VmGauges) | undefined;
+
+ // `warming` waited for a prewarm in flight, and `miss` booted a VM on demand.
+ claim(mode: string, result: "hit" | "miss" | "warming"): void {
+ const key = `mode="${mode}",result="${result}"`;
+ this.#claims.set(key, (this.#claims.get(key) ?? 0) + 1);
+ }
+
+ reject(code: string): void {
+ this.#rejections.set(code, (this.#rejections.get(code) ?? 0) + 1);
+ }
+
+ render(): string {
+ const out: string[] = [];
+ const metric = (name: string, type: string, help: string): void => {
+ out.push(`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`);
+ };
+ const slots = this.slotGauges?.();
+ if (slots !== undefined) {
+ metric("orange_worker_loader_workers", "gauge", "Dynamic Workers that hold a slot.");
+ out.push(`orange_worker_loader_workers ${slots.live}`);
+ metric("orange_worker_loader_workers_max", "gauge", "Slots: ORANGE_WORKER_LOADER_MAX_WORKERS.");
+ out.push(`orange_worker_loader_workers_max ${slots.maxWorkers}`);
+ metric("orange_worker_loader_slot_utilization", "gauge", "Slots in use divided by slots.");
+ out.push(`orange_worker_loader_slot_utilization ${slots.live / slots.maxWorkers}`);
+ metric("orange_worker_loader_queue_depth", "gauge", "Dynamic Workers that wait for a slot.");
+ out.push(`orange_worker_loader_queue_depth ${slots.queued}`);
+ }
+ const gauges = this.vmGauges?.();
+ if (gauges !== undefined) {
+ metric("orange_worker_loader_vms", "gauge", "agentOS VMs of dynamic Workers, by state.");
+ for (const [state, count] of Object.entries(gauges.vms)) {
+ out.push(`orange_worker_loader_vms{state="${state}"} ${count}`);
+ }
+ metric("orange_worker_loader_vms_max", "gauge", "VM budget: ORANGE_WORKER_LOADER_MAX_VMS.");
+ out.push(`orange_worker_loader_vms_max ${gauges.maxVms}`);
+ metric("orange_worker_loader_vm_queue_depth", "gauge", "Dynamic Workers with a slot that wait for room in the VM budget.");
+ out.push(`orange_worker_loader_vm_queue_depth ${gauges.waiting}`);
+ metric("orange_worker_loader_pool_ready", "gauge", "Prewarmed VMs ready to claim, by network mode.");
+ for (const [mode, count] of Object.entries(gauges.poolReady)) {
+ out.push(`orange_worker_loader_pool_ready{mode="${mode}"} ${count}`);
+ }
+ }
+ metric("orange_worker_loader_queue_wait_seconds", "histogram", "Time a dynamic Worker waited for a slot.");
+ out.push(...this.queueWait.render("orange_worker_loader_queue_wait_seconds"));
+ metric("orange_worker_loader_start_seconds", "histogram", "Time from a slot to a loaded dynamic Worker.");
+ out.push(...this.workerStart.render("orange_worker_loader_start_seconds"));
+ metric("orange_worker_loader_vm_boot_seconds", "histogram", "Time to boot a VM and start its child process.");
+ out.push(...this.vmBoot.render("orange_worker_loader_vm_boot_seconds"));
+ metric("orange_worker_loader_pool_claims_total", "counter", "VM claims, by network mode and pool hit or miss.");
+ for (const [labels, count] of this.#claims) out.push(`orange_worker_loader_pool_claims_total{${labels}} ${count}`);
+ metric("orange_worker_loader_rejections_total", "counter", "Dynamic Workers refused with WorkerLoaderLimitError, by code.");
+ for (const [code, count] of this.#rejections) out.push(`orange_worker_loader_rejections_total{code="${code}"} ${count}`);
+ metric("orange_runtime_resident_memory_bytes", "gauge", "Resident memory of the runner and of its child processes, such as the agentOS sidecar.");
+ const memory = residentMemory();
+ out.push(`orange_runtime_resident_memory_bytes{process="runner"} ${memory.runner}`);
+ if (memory.children !== undefined) {
+ out.push(`orange_runtime_resident_memory_bytes{process="children"} ${memory.children}`);
+ }
+ const limit = memoryLimitBytes();
+ if (limit !== undefined) {
+ metric("orange_runtime_memory_limit_bytes", "gauge", "The cgroup memory limit of the runner.");
+ out.push(`orange_runtime_memory_limit_bytes ${limit}`);
+ }
+ return `${out.join("\n")}\n`;
+ }
+}
+
+// Linux only: the children's RSS comes from /proc, which other systems lack.
+function residentMemory(): { children: number | undefined; runner: number } {
+ const runner = process.memoryUsage.rss();
+ if (process.platform !== "linux") return { children: undefined, runner };
+ let children = 0;
+ const pending = [process.pid];
+ const seen = new Set();
+ while (pending.length > 0) {
+ const pid = pending.pop();
+ if (pid === undefined || seen.has(pid)) continue;
+ seen.add(pid);
+ if (pid !== process.pid) children += rssOf(pid);
+ for (const child of childrenOf(pid)) pending.push(child);
+ }
+ return { children, runner };
+}
+
+// A process can exit while it is read, so a missing file counts as nothing.
+function readProc(path: string): string {
+ try {
+ return readFileSync(path, "utf8");
+ } catch {
+ return "";
+ }
+}
+
+function childrenOf(pid: number): number[] {
+ let tasks: string[];
+ try {
+ tasks = readdirSync(`/proc/${pid}/task`);
+ } catch {
+ return [];
+ }
+ return tasks.flatMap((task) =>
+ readProc(`/proc/${pid}/task/${task}/children`)
+ .split(" ")
+ .filter((value) => value !== "")
+ .map(Number),
+ );
+}
+
+function rssOf(pid: number): number {
+ const match = /^VmRSS:\s+(\d+) kB/m.exec(readProc(`/proc/${pid}/status`));
+ return match?.[1] === undefined ? 0 : Number(match[1]) * 1024;
+}
+
+function memoryLimitBytes(): number | undefined {
+ if (process.platform !== "linux") return undefined;
+ const cgroup = /^0::(.*)$/m.exec(readProc("/proc/self/cgroup"))?.[1];
+ if (cgroup === undefined) return undefined;
+ const raw = readProc(`/sys/fs/cgroup${cgroup}/memory.max`).trim();
+ const value = Number(raw);
+ return raw === "" || raw === "max" || !Number.isFinite(value) ? undefined : value;
+}
diff --git a/packages/runtime-node/src/bindings/worker-loader/session.ts b/packages/runtime-node/src/bindings/worker-loader/session.ts
index 75dd584..e5ca276 100644
--- a/packages/runtime-node/src/bindings/worker-loader/session.ts
+++ b/packages/runtime-node/src/bindings/worker-loader/session.ts
@@ -340,10 +340,14 @@ export class LoaderCapabilityTarget extends DurableObjectRpcTarget {
// process's LoaderCapabilityTarget.
export class RemoteCapability implements Capability {
readonly #stub: object;
+ // Settles before each call's result does, and its rejection replaces the call's. See
+ // `DynamicWorker.entrypoint()`.
+ readonly #ready: Promise | undefined;
#released = false;
- constructor(stub: object) {
+ constructor(stub: object, ready?: Promise) {
this.#stub = stub;
+ this.#ready = ready;
}
call(path: readonly string[], args: readonly unknown[] | undefined): CallOutcome {
@@ -353,7 +357,14 @@ export class RemoteCapability implements Capability {
const envelope = args === undefined ? undefined : encodeLoaderValue([...args]);
const invoke: unknown = Reflect.get(this.#stub, "invoke");
if (typeof invoke !== "function") throw new Error("Dynamic Worker capability is invalid");
- const reply: unknown = await Reflect.apply(invoke, this.#stub, [[...path], envelope]);
+ let reply: unknown;
+ try {
+ reply = await Reflect.apply(invoke, this.#stub, [[...path], envelope]);
+ } catch (error: unknown) {
+ await this.#ready;
+ throw error;
+ }
+ await this.#ready;
return decodeLoaderReply(reply);
})();
value.catch(() => undefined);
@@ -363,7 +374,7 @@ export class RemoteCapability implements Capability {
}
dup(): Capability {
- return new RemoteCapability(duplicate(this.#stub));
+ return new RemoteCapability(duplicate(this.#stub), this.#ready);
}
dispose(): void {
diff --git a/packages/runtime-node/src/globals/implementations.ts b/packages/runtime-node/src/globals/implementations.ts
index 9b17f11..215b606 100644
--- a/packages/runtime-node/src/globals/implementations.ts
+++ b/packages/runtime-node/src/globals/implementations.ts
@@ -37,7 +37,7 @@ import {
import { Navigator, navigator } from "./navigator.js";
import { nodeWebGlobals } from "./node-web.js";
import { scheduler } from "./scheduler.js";
-import { WebAssembly } from "./webassembly.js";
+import { nativeWebAssembly } from "./webassembly.js";
import {
WebSocket,
WebSocketPair,
@@ -96,7 +96,10 @@ const scopeGlobals = {
URL,
URLPattern,
URLSearchParams,
- WebAssembly,
+ // A getter, so loading this module never reads WebAssembly. See nativeWebAssembly().
+ get WebAssembly() {
+ return nativeWebAssembly();
+ },
WebSocket,
WebSocketPair,
WebSocketRequestResponsePair,
@@ -142,10 +145,13 @@ type ScopeGlobals = typeof scopeGlobals;
// assertion. The global-scope differential fixture reads these globals from self.
const self = globalThis as unknown as GlobalScope;
-export const globalImplementations = {
- ...scopeGlobals,
- self,
-};
+// Copies the descriptors, so the WebAssembly getter is not called here.
+// SAFETY: defineProperties adds every scopeGlobals property to the object,
+// but TypeScript types its result as the first argument.
+export const globalImplementations = Object.defineProperties(
+ { self },
+ Object.getOwnPropertyDescriptors(scopeGlobals),
+) as ScopeGlobals & { self: GlobalScope };
export const globalImplementationStatuses: Record<
keyof typeof globalImplementations,
diff --git a/packages/runtime-node/src/globals/webassembly.ts b/packages/runtime-node/src/globals/webassembly.ts
index 4dc264c..77dbfe4 100644
--- a/packages/runtime-node/src/globals/webassembly.ts
+++ b/packages/runtime-node/src/globals/webassembly.ts
@@ -141,10 +141,13 @@ function isWebAssemblyNamespace(value: unknown): value is WebAssemblyNamespace {
}
// Preserve Node's full namespace (including host-only helpers used by Undici),
-// while exposing only the Cloudflare-visible contract to Orange code.
-const nativeWebAssembly: unknown = Reflect.get(globalThis, "WebAssembly");
-if (!isWebAssemblyNamespace(nativeWebAssembly)) {
- throw new Error("Node.js does not provide the required WebAssembly global");
+// while exposing only the Cloudflare-visible contract to Orange code. Read on
+// use, not at module load: V8 has no WebAssembly global while agentOS builds
+// a module snapshot of the dynamic Worker child.
+export function nativeWebAssembly(): WebAssemblyNamespace {
+ const value: unknown = Reflect.get(globalThis, "WebAssembly");
+ if (!isWebAssemblyNamespace(value)) {
+ throw new Error("Node.js does not provide the required WebAssembly global");
+ }
+ return value;
}
-
-export const WebAssembly = nativeWebAssembly;
diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts
index a8a07e8..cca917c 100644
--- a/packages/runtime-node/src/index.ts
+++ b/packages/runtime-node/src/index.ts
@@ -78,7 +78,7 @@ export {
WritableStream,
} from "./globals/native.js";
export { Event } from "./globals/event.js";
-export { WebAssembly } from "./globals/webassembly.js";
+export { nativeWebAssembly } from "./globals/webassembly.js";
export { WebSocket, WebSocketPair } from "./globals/websocket.js";
export {
clearInterval,
diff --git a/packages/runtime-node/src/runner.ts b/packages/runtime-node/src/runner.ts
index dfc1bc3..ec8b91e 100644
--- a/packages/runtime-node/src/runner.ts
+++ b/packages/runtime-node/src/runner.ts
@@ -55,13 +55,21 @@ import {
type WorkflowEngineObject,
workflowEngineClassName,
} from "./bindings/workflows/engine-object.js";
-import { AgentOsWorkerLoaderBackend, workerLoaderPoolSize } from "./bindings/worker-loader/agentos.js";
+import {
+ AgentOsWorkerLoaderBackend,
+ workerLoaderMaxVms,
+ workerLoaderPoolLimits,
+} from "./bindings/worker-loader/agentos.js";
import {
DynamicWorkerPool,
WorkerLoader,
+ WorkerLoaderLimitError,
+ workerLoaderIdleTimeoutMs,
+ workerLoaderMaxWorkers,
workerLoaderVmBundlePath,
type WorkerLoaderBackend,
} from "./bindings/worker-loader/loader.js";
+import { WorkerLoaderMetrics } from "./bindings/worker-loader/metrics.js";
import { createDispatchNamespace } from "./bindings/dispatch-namespace.js";
import type { EnvironmentBinding } from "./bindings/create-env.js";
import type { ConfiguredDurableObjectClass } from "./bindings/durable-objects/classes.js";
@@ -218,15 +226,24 @@ function requestBody(
}
// Dynamic Workers run only in agentOS VMs. All Worker Loader bindings share one
-// backend.
+// backend, one VM budget, and one set of metrics.
+const workerLoaderMetrics = new WorkerLoaderMetrics();
let sharedWorkerLoaderBackend: WorkerLoaderBackend | undefined;
function workerLoaderBackend(): WorkerLoaderBackend {
+ const settings = runtimeSettings();
+ const pool = workerLoaderPoolLimits(settings);
sharedWorkerLoaderBackend ??= new AgentOsWorkerLoaderBackend({
bundlePath: () => workerLoaderVmBundlePath(import.meta.url),
+ maxVms: workerLoaderMaxVms(settings, workerLoaderMaxWorkers(settings), pool),
+ metrics: workerLoaderMetrics,
onDiagnostic(message): void {
send({ message, type: "background-error" });
},
- poolSize: workerLoaderPoolSize(runtimeSettings()),
+ onWarning(message): void {
+ send({ message, type: "warning" });
+ },
+ // The pool keeps VMs above its idle size as long as a Worker stays idle before it stops.
+ pool: { ...pool, idleMs: workerLoaderIdleTimeoutMs(settings) },
});
return sharedWorkerLoaderBackend;
}
@@ -234,10 +251,17 @@ function workerLoaderBackend(): WorkerLoaderBackend {
// One pool bounds the dynamic Workers of every Worker Loader binding.
let sharedWorkerLoaderPool: DynamicWorkerPool | undefined;
function workerLoaderPool(warn: (message: string) => void): DynamicWorkerPool {
- sharedWorkerLoaderPool ??= DynamicWorkerPool.fromSettings(runtimeSettings(), warn);
+ sharedWorkerLoaderPool ??= DynamicWorkerPool.fromSettings(runtimeSettings(), warn, workerLoaderMetrics);
return sharedWorkerLoaderPool;
}
+// A Worker that does not catch a WorkerLoaderLimitError is out of dynamic Worker capacity, not
+// broken, so the client gets 503 and when to retry. Cloudflare answers a Worker that cannot run
+// for platform limits with 503 too.
+function capacityFailure(error: unknown): { retryAfterSeconds: number } | undefined {
+ return error instanceof WorkerLoaderLimitError ? { retryAfterSeconds: error.retryAfterSeconds } : undefined;
+}
+
function requestHeaders(request: IncomingMessage): Headers {
const headers = new Headers();
for (const [name, value] of Object.entries(request.headers)) {
@@ -1205,14 +1229,19 @@ export async function startRuntime(
reportWaitUntil(dispatch.waitUntil);
} catch (error: unknown) {
send({ ...errorDetails(error), type: "request-error" });
- if (!outgoing.headersSent) {
- outgoing.statusCode = 500;
- outgoing.setHeader("content-type", "text/plain; charset=utf-8");
- }
if (outgoing.headersSent) {
outgoing.destroy(error instanceof Error ? error : undefined);
- } else {
+ return;
+ }
+ outgoing.setHeader("content-type", "text/plain; charset=utf-8");
+ const capacity = capacityFailure(error);
+ if (capacity === undefined) {
+ outgoing.statusCode = 500;
outgoing.end("Internal Server Error");
+ } else {
+ outgoing.statusCode = 503;
+ outgoing.setHeader("retry-after", String(capacity.retryAfterSeconds));
+ outgoing.end("Service Unavailable");
}
}
})();
@@ -1257,8 +1286,20 @@ export async function startRuntime(
} catch (error: unknown) {
if (workerSocket !== null) closeUncoupledWorkerWebSocket(workerSocket);
send({ ...errorDetails(error), type: "request-error" });
- writeUpgradeFailure(socket);
- reportRequest(incoming, 500, startedAt);
+ const capacity = capacityFailure(error);
+ if (capacity === undefined) {
+ writeUpgradeFailure(socket);
+ reportRequest(incoming, 500, startedAt);
+ } else {
+ await writeRejectedUpgrade(
+ socket,
+ new Response("Service Unavailable", {
+ headers: { "retry-after": String(capacity.retryAfterSeconds) },
+ status: 503,
+ }),
+ );
+ reportRequest(incoming, 503, startedAt);
+ }
}
})();
});
@@ -1274,16 +1315,26 @@ export async function startRuntime(
await closeRegistry();
throw error;
}
- // A Rivet cluster scrapes the runner's metrics. A local Engine has no scraper.
- const metricsPort = externalEngine ? metricsPortFromEnvironment(process.env) : undefined;
+ // A Rivet cluster scrapes the runner's metrics. `orange dev` serves them too when the port is set.
+ const metricsPort = metricsPortFromEnvironment(process.env);
const metricsServer =
metricsPort === undefined
? undefined
: await startMetricsServer(metricsPort, async () => {
+ const loaderMetrics = runtimeBindings.workerLoaders.length > 0 ? workerLoaderMetrics.render() : "";
if (durableObjectRegistry === undefined) {
+ if (loaderMetrics !== "") {
+ return new Response(loaderMetrics, { headers: { "content-type": "text/plain; version=0.0.4" } });
+ }
return new Response("registry not started\n", { status: 503 });
}
- return await durableObjectRegistry.prometheusMetrics();
+ const rivet = await durableObjectRegistry.prometheusMetrics();
+ if (loaderMetrics === "" || !rivet.ok) return rivet;
+ // Only the content type carries over: the body is longer than RivetKit's.
+ return new Response(`${await rivet.text()}${loaderMetrics}`, {
+ headers: { "content-type": rivet.headers.get("content-type") ?? "text/plain; version=0.0.4" },
+ status: rivet.status,
+ });
});
// The cron actor runs in this process.
const cronSync = new AbortController();
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 6c85703..a3c82f7 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -12,8 +12,8 @@ importers:
specifier: 3.1153.0
version: 3.1153.0
'@rivet-dev/agentos-core':
- specifier: 0.4.8
- version: 0.4.8
+ specifier: 0.4.9-rc.1
+ version: 0.4.9-rc.1
'@rivet-dev/durable-object-migrator':
specifier: 1.0.3
version: 1.0.3
@@ -172,8 +172,8 @@ importers:
specifier: 3.1153.0
version: 3.1153.0
'@rivet-dev/agentos-core':
- specifier: 0.4.8
- version: 0.4.8
+ specifier: 0.4.9-rc.1
+ version: 0.4.9-rc.1
'@rivet-dev/durable-object-migrator':
specifier: 1.0.3
version: 1.0.3
@@ -1424,40 +1424,40 @@ packages:
engines: {node: '>=18'}
hasBin: true
- '@rivet-dev/agentos-core@0.4.8':
- resolution: {integrity: sha512-EekiSBrBGAR6VI41BlzQyaIDRXGxHillrojST5t8XnZ30jfzqpymWkPC997G+MT5NPyPcrv/q3duAjX06imzXQ==}
+ '@rivet-dev/agentos-core@0.4.9-rc.1':
+ resolution: {integrity: sha512-pqE3TSCSB5hXImPOHeQ5edXR6IBI2FhCT2lHUeLZMcEBY5rAAcplJFF3JdeGkBvaC0YSTBr3P5mmfyvUpUdmVw==}
- '@rivet-dev/agentos-runtime-core@0.4.8':
- resolution: {integrity: sha512-uCNXNPJXisctZrRonSvjzPxazwQErPZaOrtogibnONd11D6KxDT2u/p89BqihUiK3KCQs8NLkVr3WaYgwZ+0bA==}
+ '@rivet-dev/agentos-runtime-core@0.4.9-rc.1':
+ resolution: {integrity: sha512-yr2SUC6bNdv+R5rkJqq/4FPipP0OiE1kFJV6T+lAFjpoalsUn9X04ZaIvMHgvAgdL6Gug31dK6kLd+D/nhOx6A==}
- '@rivet-dev/agentos-runtime-sidecar-darwin-arm64@0.4.8':
- resolution: {integrity: sha512-ZaL+MEHqtbP1WRF575m2LQFP6tBcIPb6p4vKnZGYuUad2o9qqE1qeOhG3YhFdymrE8QUMHwFKDvL/+7Ep1k/7Q==}
+ '@rivet-dev/agentos-runtime-sidecar-darwin-arm64@0.4.9-rc.1':
+ resolution: {integrity: sha512-ofVhX5pWAP5OEdiv/MKoC/rzbGegIzBj9EIK8hP8KzRdVMq3uWPtTF+4QjPHmLUhOBT1qyOpmirenhkyafFsZg==}
engines: {node: '>=20'}
cpu: [arm64]
os: [darwin]
- '@rivet-dev/agentos-runtime-sidecar-darwin-x64@0.4.8':
- resolution: {integrity: sha512-chpzdQjs4SdSGrlTj6MSAk14/E91kyb6zcrbw2CJCn2tE4CJi0BUqgFAdWHKfZznwbnWXNB7vU9oBxYLmHSQQw==}
+ '@rivet-dev/agentos-runtime-sidecar-darwin-x64@0.4.9-rc.1':
+ resolution: {integrity: sha512-/gTYCcB/AEvaEgdLU+bPdfRX0vqaKtmgTWdwAm77jENcdSuiFNuAuxYmhqXB15Pm9Bylo57tqhX/Dl5OePKVmw==}
engines: {node: '>=20'}
cpu: [x64]
os: [darwin]
- '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu@0.4.8':
- resolution: {integrity: sha512-ysmQ8kfBNAhRGodTkSJUftb7PowrxBhhDil66mSKVOQwO6jN21/K6M+K8y0DGTS5d1azqf+fZe2UzewMb3dUbQ==}
+ '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu@0.4.9-rc.1':
+ resolution: {integrity: sha512-HSicsWKJRrFsCcjEW54ZTT+G07axN7cku5r0jVtGkx6DSSstSqMU8QSWj5mDO6dT6pVsTAHDXORqizbO0RcRWw==}
engines: {node: '>=20'}
cpu: [arm64]
os: [linux]
libc: [glibc]
- '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu@0.4.8':
- resolution: {integrity: sha512-DEJNL3VcxWqRwIDcOSWHQ3APTNFZRvv6wokWP4dV4dwHsNc8dCKMLf8B2qYKw3Qljb7loinppeOu6WRwDaJgqw==}
+ '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu@0.4.9-rc.1':
+ resolution: {integrity: sha512-1WS+G0Wj3pA+2qKME2R92Qjz0wFweF8R0Lss0BAtRrWmUmmHvb5mzjmcgDTkyJVwvC/MHKO0sNPmARRN9NapTA==}
engines: {node: '>=20'}
cpu: [x64]
os: [linux]
libc: [glibc]
- '@rivet-dev/agentos-runtime-sidecar@0.4.8':
- resolution: {integrity: sha512-SUeyGdmSKl6NyLhFoLo5KJa48b84Cg40Dcw80CxY+62u4tNIiops3XpZEp2Xzeh2USD7GaYfvVBfzYba9votmw==}
+ '@rivet-dev/agentos-runtime-sidecar@0.4.9-rc.1':
+ resolution: {integrity: sha512-Ag0uUmdZIrlajssHJ9F7RcGzkZCGL4c407e5nPv0fNV3FvBeqq43+RXdF+W4t0FhaTtkbLqh+04OIkpMUSw59A==}
engines: {node: '>=20'}
'@rivet-dev/durable-object-migrator@1.0.3':
@@ -4263,40 +4263,40 @@ snapshots:
- react-native-b4a
- supports-color
- '@rivet-dev/agentos-core@0.4.8':
+ '@rivet-dev/agentos-core@0.4.9-rc.1':
dependencies:
- '@rivet-dev/agentos-runtime-core': 0.4.8
- '@rivet-dev/agentos-runtime-sidecar': 0.4.8
+ '@rivet-dev/agentos-runtime-core': 0.4.9-rc.1
+ '@rivet-dev/agentos-runtime-sidecar': 0.4.9-rc.1
'@xterm/headless': 6.0.0
croner: 10.0.1
long-timeout: 0.1.1
zod: 4.5.4
- '@rivet-dev/agentos-runtime-core@0.4.8':
+ '@rivet-dev/agentos-runtime-core@0.4.9-rc.1':
dependencies:
'@bare-ts/lib': 0.4.1
- '@rivet-dev/agentos-runtime-sidecar': 0.4.8
+ '@rivet-dev/agentos-runtime-sidecar': 0.4.9-rc.1
zod: 4.5.4
zod-to-json-schema: 3.25.2(zod@4.5.4)
- '@rivet-dev/agentos-runtime-sidecar-darwin-arm64@0.4.8':
+ '@rivet-dev/agentos-runtime-sidecar-darwin-arm64@0.4.9-rc.1':
optional: true
- '@rivet-dev/agentos-runtime-sidecar-darwin-x64@0.4.8':
+ '@rivet-dev/agentos-runtime-sidecar-darwin-x64@0.4.9-rc.1':
optional: true
- '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu@0.4.8':
+ '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu@0.4.9-rc.1':
optional: true
- '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu@0.4.8':
+ '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu@0.4.9-rc.1':
optional: true
- '@rivet-dev/agentos-runtime-sidecar@0.4.8':
+ '@rivet-dev/agentos-runtime-sidecar@0.4.9-rc.1':
optionalDependencies:
- '@rivet-dev/agentos-runtime-sidecar-darwin-arm64': 0.4.8
- '@rivet-dev/agentos-runtime-sidecar-darwin-x64': 0.4.8
- '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu': 0.4.8
- '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu': 0.4.8
+ '@rivet-dev/agentos-runtime-sidecar-darwin-arm64': 0.4.9-rc.1
+ '@rivet-dev/agentos-runtime-sidecar-darwin-x64': 0.4.9-rc.1
+ '@rivet-dev/agentos-runtime-sidecar-linux-arm64-gnu': 0.4.9-rc.1
+ '@rivet-dev/agentos-runtime-sidecar-linux-x64-gnu': 0.4.9-rc.1
'@rivet-dev/durable-object-migrator@1.0.3':
dependencies:
diff --git a/src/build/build-package.ts b/src/build/build-package.ts
index b3b9f7d..126e3dc 100644
--- a/src/build/build-package.ts
+++ b/src/build/build-package.ts
@@ -74,6 +74,14 @@ export async function buildWorkerLoaderVmBundle(
join(outputDirectory, "manifest.json"),
`${JSON.stringify(manifest, null, "\t")}\n`,
);
+ // The process entry agentOS runs for each dynamic Worker (`childEntry` in
+ // worker-loader/agentos.ts). It stays this small so agentOS can boot it
+ // from a snapshot of everything it imports, and only these two lines run
+ // in each process.
+ await writeFile(
+ join(outputDirectory, "worker-loader-child-main.mjs"),
+ 'import { startChild } from "./worker-loader-child.mjs";\nstartChild();\n',
+ );
}
export async function buildPackage(
diff --git a/src/commands/d1.ts b/src/commands/d1.ts
index ac738b7..b4be930 100644
--- a/src/commands/d1.ts
+++ b/src/commands/d1.ts
@@ -194,6 +194,8 @@ async function openDatabase(
deploymentIdentity,
host: "127.0.0.1",
normalizedConfiguration: "{}",
+ // A one-shot D1 command has nothing to scrape.
+ serveMetrics: false,
port: 0,
projectRoot: config.projectRoot,
...(endpoint === undefined
diff --git a/src/commands/dev.ts b/src/commands/dev.ts
index 667889b..4466035 100644
--- a/src/commands/dev.ts
+++ b/src/commands/dev.ts
@@ -432,6 +432,8 @@ async function executeDevCommand(
...(index === 0 ? { onRequest: (request) => reporter.request(request) } : {}),
port: index === 0 ? parsed.port : 0,
projectRoot: config.projectRoot,
+ // One port can have one metrics server.
+ serveMetrics: index === 0,
queueConsumers: config.queueConsumers,
tailConsumers: config.tailConsumers,
...(config.workerName === undefined ? {} : { workerName: config.workerName }),
diff --git a/tests/bindings/worker-loader.test.ts b/tests/bindings/worker-loader.test.ts
index ee2cf93..065c804 100644
--- a/tests/bindings/worker-loader.test.ts
+++ b/tests/bindings/worker-loader.test.ts
@@ -2,12 +2,13 @@ import assert from "node:assert/strict";
import { glob, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { builtinModules } from "node:module";
+import { createServer, type AddressInfo } from "node:net";
import { dirname, join } from "node:path";
import test from "node:test";
import { build, type Plugin } from "esbuild";
-import { copyProject, isRecord, repositoryRoot, startOrangeDev, withOrangeDev } from "../harness/orange-dev.js";
+import { copyProject, isRecord, repositoryRoot, startOrangeDev, waitFor, withOrangeDev } from "../harness/orange-dev.js";
const fixtureRoot = join(repositoryRoot, "tests", "fixtures", "worker-loader");
@@ -158,6 +159,10 @@ test("loads dynamic Workers with workerd Worker Loader semantics in agentOS VMs"
// name or no name loads a fresh Worker and calls the code callback.
assert.deepEqual(await get("cache"), { counts: [1, 2, 3, 1, 1], loads: 3 });
+ // Calls made while a Worker loads reach it in order and each gets its own result, even
+ // beside a 3 MiB request body.
+ assert.deepEqual(await get("pipelined"), ["/a:3:one", `/large:${3 * 1024 * 1024}:end`, 42, "/b:3:two"]);
+
// A syntax error in a module rejects the call with V8's SyntaxError.
assert.deepEqual(await get("syntax"), {
error: { message: "Unexpected token ';'", name: "SyntaxError" },
@@ -254,10 +259,7 @@ test("stops idle dynamic Workers without interrupting calls in flight", async ()
ORANGE_WORKER_LOADER_MAX_WORKERS: "2",
};
await withOrangeDev("worker-loader", async (dev) => {
- const result = await dev.json("/eviction");
- assert.ok(isRecord(result), JSON.stringify(result));
- const { full, ...rest } = result;
- assert.deepEqual(rest, {
+ assert.deepEqual(await dev.json("/eviction"), {
// After the idle timeout, get() with the same name calls getCode again and
// the Worker starts with fresh module state.
named: [1, 2, 1],
@@ -266,20 +268,17 @@ test("stops idle dynamic Workers without interrupting calls in flight", async ()
// call loads the same code again.
heldCounts: [1, 1],
// Calls that outlast the idle timeout finish, and a full pool of busy
- // Workers does not stop them.
+ // Workers does not stop them. A Worker loaded meanwhile waits for a slot.
slept: [{ value: 3_000 }, { value: 3_000 }],
+ full: { value: 1 },
// Idle Workers give up their slots, so more load() calls than the limit
// all succeed.
sequential: [1, 1, 1, 1, 1, 1],
- // With both Workers handling a call made in the same tick, load() throws
- // and both calls finish.
- sameTick: "WorkerLoaderLimitError",
+ // With both Workers handling a call made in the same tick, a third Worker
+ // waits for one of them to finish, and both calls finish.
+ sameTick: { value: 1 },
sameTickCalls: [{ value: 2 }, { value: 2 }],
});
- // With every slot busy, load() throws the typed error that names the limit.
- assert.ok(isRecord(full) && isRecord(full.error), JSON.stringify(full));
- assert.equal(full.error.name, "WorkerLoaderLimitError");
- assert.match(String(full.error.message), /the limit is 2\. Raise ORANGE_WORKER_LOADER_MAX_WORKERS/);
}, { environment });
});
@@ -312,6 +311,135 @@ test("keeps a dynamic Worker running until every body it streams ends, and frees
}, { environment });
});
+async function freePort(): Promise {
+ const server = createServer();
+ await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen));
+ const { port } = server.address() as AddressInfo;
+ await new Promise((resolveClose) => server.close(resolveClose));
+ return port;
+}
+
+// Reads the runner's Prometheus metrics into `name{labels}` -> value.
+async function scrape(port: number): Promise