From 70c8b83a9bd160df9de27c67b5d320b2b6baf76c Mon Sep 17 00:00:00 2001 From: Kacper Wojciechowski Date: Mon, 24 Aug 2026 20:26:04 +0000 Subject: [PATCH] feat(runtime): make the guest-execution admission ceiling operator-tunable The process-wide V8 executor slot cap was derived from available_parallelism() and had no override path in any shipped build: every runtime.* config value came from RuntimeConfig::default(). Its own limit error told operators to "raise runtime.executor.maxActiveVms", a knob nothing could set. The cap admits concurrently running guest executions, not VMs. Every live guest process holds one slot for its whole lifetime, so a shell and the command it waits on need two, and a fleet of parallel agents exhausts it well before any per-VM cap. CPU count is the wrong unit: the slot costs one OS thread and one thread-affine V8 isolate, and an agent parked on a network read burns no CPU while holding one. - Read AGENTOS_MAX_ACTIVE_GUEST_EXECUTIONS at sidecar startup, before the process topology is fixed. Missing, non-numeric, zero, or above-ceiling values fail startup with a typed error instead of being clamped. - Default to a fixed 64 rather than the host core count, so admitted concurrency no longer varies per machine, with a hard ceiling of 1024. - Rename the knob, config field, and limit error to say guest executions, and point the error at a variable an operator can actually set. - Log the effective ceiling at startup so the admitted value is observable before an execution is rejected for exceeding it. Kept process-scoped rather than exposed as a client wire field: one sidecar process is shared by every VM and connection it hosts, so no single tenant may rewrite the ceiling for its neighbours. --- crates/native-sidecar/src/stdio.rs | 14 +- .../tests/fixtures/limits-inventory.json | 13 ++ crates/runtime/src/lib.rs | 167 ++++++++++++++++-- crates/v8-runtime/src/embedded_runtime.rs | 6 +- crates/v8-runtime/src/session.rs | 30 ++-- .../tests/embedded_runtime_session.rs | 4 +- docs/content/docs/debugging.mdx | 21 +++ docs/content/docs/resource-limits.mdx | 57 ++++++ 8 files changed, 284 insertions(+), 28 deletions(-) diff --git a/crates/native-sidecar/src/stdio.rs b/crates/native-sidecar/src/stdio.rs index 913b2eb022..65dfeaaf42 100644 --- a/crates/native-sidecar/src/stdio.rs +++ b/crates/native-sidecar/src/stdio.rs @@ -747,10 +747,22 @@ fn run_with_optional_control( extensions: Vec>, control_fd: Option, ) -> Result<(), Box> { - let config = NativeSidecarConfig { + let mut config = NativeSidecarConfig { compile_cache_root: Some(default_compile_cache_root()), ..NativeSidecarConfig::default() }; + // Operator overrides must land before `SidecarRuntime::process` fixes the + // process topology: the first caller's config is the one the whole process + // keeps, and a later differing config is a typed error, not a re-configure. + config.runtime.apply_env_overrides()?; + // The admitted ceiling is fixed for the life of the process and shared by + // every VM it hosts, so make the effective value observable at startup + // rather than only when an execution is rejected for exceeding it. + tracing::info!( + max_active_guest_executions = config.runtime.max_active_guest_executions, + env = agentos_runtime::MAX_ACTIVE_GUEST_EXECUTIONS_ENV, + "guest execution admission ceiling" + ); let runtime = agentos_runtime::SidecarRuntime::process(&config.runtime)?; let runtime_context = runtime.context(); // Initialize the embedded V8 runtime + platform now, on the long-lived main diff --git a/crates/native-sidecar/tests/fixtures/limits-inventory.json b/crates/native-sidecar/tests/fixtures/limits-inventory.json index f9fca61aa7..e19f23e874 100644 --- a/crates/native-sidecar/tests/fixtures/limits-inventory.json +++ b/crates/native-sidecar/tests/fixtures/limits-inventory.json @@ -1658,6 +1658,19 @@ "rationale": "Default for the configured process-wide protocol queue bound.", "wired": "RuntimeConfig.protocol.max_process_events" }, + { + "name": "DEFAULT_MAX_ACTIVE_GUEST_EXECUTIONS", + "path": "crates/runtime/src/lib.rs", + "class": "policy", + "rationale": "Default process-wide cap on concurrently running guest executions; each slot owns one OS thread and one V8 isolate.", + "wired": "RuntimeConfig.max_active_guest_executions" + }, + { + "name": "MAX_ACTIVE_GUEST_EXECUTIONS_CEILING", + "path": "crates/runtime/src/lib.rs", + "class": "invariant", + "rationale": "Hard ceiling for the AGENTOS_MAX_ACTIVE_GUEST_EXECUTIONS operator override; admission stays bounded whatever an operator requests." + }, { "name": "DEFAULT_VM_EXECUTOR_TEARDOWN_TIMEOUT_MS", "path": "crates/runtime/src/lib.rs", diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 71392d0879..5b995ef4c6 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -69,6 +69,38 @@ const DEFAULT_MAX_PROCESS_HTTP2_EVENT_BYTES: usize = 512 * 1024 * 1024; const DEFAULT_TASK_POLL_WATCHDOG_MS: u64 = 100; const DEFAULT_MAX_TERMINAL_TASK_REPORTS: usize = 4_096; const DEFAULT_VM_EXECUTOR_TEARDOWN_TIMEOUT_MS: u64 = 5_000; + +/// Process-wide ceiling on concurrently running guest executions. +/// +/// Each admitted execution owns one OS thread and one V8 isolate (thread-affine, +/// so it can never be multiplexed onto a shared pool) capped at +/// `DEFAULT_HEAP_LIMIT_MB`. The binding constraint is therefore threads and +/// memory, NOT CPU: an agent parked on a network read, or a shell blocked in +/// `waitpid`, burns no CPU and still holds its slot for the whole life of the +/// guest process. Deriving this from `available_parallelism()` made the ceiling +/// depend on the host's core count and silently rejected ordinary workloads — +/// a shell plus the command it waits on already needs two slots. +/// +/// A fixed default keeps the admitted concurrency identical on every host. +/// Raise it with `AGENTOS_MAX_ACTIVE_GUEST_EXECUTIONS`. +pub const DEFAULT_MAX_ACTIVE_GUEST_EXECUTIONS: usize = 64; + +/// Hard ceiling for `AGENTOS_MAX_ACTIVE_GUEST_EXECUTIONS`. +/// +/// Admission stays bounded regardless of what an operator asks for: at this +/// ceiling the process still reserves 1024 OS threads and isolates, which is +/// past the point where a host is thread- and memory-bound. Requests above it +/// are a typed configuration error, never a silent clamp. +pub const MAX_ACTIVE_GUEST_EXECUTIONS_CEILING: usize = 1_024; + +/// Operator override for [`DEFAULT_MAX_ACTIVE_GUEST_EXECUTIONS`]. +/// +/// Read once, by the process entrypoint, before any VM exists. The value is +/// process topology (see [`SidecarRuntime::process`]) and is deliberately not a +/// client wire field: one sidecar process is shared by every VM and connection, +/// so no single tenant may rewrite it for its neighbours. +pub const MAX_ACTIVE_GUEST_EXECUTIONS_ENV: &str = "AGENTOS_MAX_ACTIVE_GUEST_EXECUTIONS"; + pub const DEFAULT_PROTOCOL_MAX_INGRESS_FRAMES: usize = 128; pub const DEFAULT_PROTOCOL_MAX_INGRESS_BYTES: usize = 64 * 1024 * 1024; pub const DEFAULT_PROTOCOL_MAX_CONTROL_FRAMES: usize = 1_024; @@ -417,7 +449,10 @@ impl RuntimeResourceConfig { #[derive(Clone, Debug, PartialEq, Eq)] pub struct RuntimeConfig { pub worker_threads: usize, - pub max_active_vm_executors: usize, + /// Process-wide cap on concurrently running guest executions (JavaScript, + /// TypeScript, Python, and WASM alike — every live guest process holds one). + /// See [`DEFAULT_MAX_ACTIVE_GUEST_EXECUTIONS`]. + pub max_active_guest_executions: usize, pub vm_executor_teardown_timeout_ms: u64, pub blocking_worker_threads: usize, pub max_blocking_jobs: usize, @@ -438,7 +473,7 @@ impl Default for RuntimeConfig { .unwrap_or(1); Self { worker_threads: available.clamp(1, 4), - max_active_vm_executors: available.max(1), + max_active_guest_executions: DEFAULT_MAX_ACTIVE_GUEST_EXECUTIONS, vm_executor_teardown_timeout_ms: DEFAULT_VM_EXECUTOR_TEARDOWN_TIMEOUT_MS, blocking_worker_threads: available.clamp(1, 4), max_blocking_jobs: DEFAULT_MAX_BLOCKING_JOBS, @@ -455,12 +490,46 @@ impl Default for RuntimeConfig { } impl RuntimeConfig { + /// Apply operator overrides from the process environment. + /// + /// Call this from the process entrypoint, before [`SidecarRuntime::process`] + /// fixes the topology. A present-but-unusable value is a hard, typed error + /// naming the variable and its bounds: an operator who asked for a specific + /// admission ceiling must never silently get a different one. + pub fn apply_env_overrides(&mut self) -> Result<(), RuntimeBuildError> { + self.apply_env_overrides_from(|key| std::env::var(key).ok()) + } + + /// Testable core of [`apply_env_overrides`]. `read` resolves a variable + /// name to its value, mirroring `std::env::var(..).ok()`. + pub fn apply_env_overrides_from( + &mut self, + read: impl Fn(&str) -> Option, + ) -> Result<(), RuntimeBuildError> { + let Some(raw) = read(MAX_ACTIVE_GUEST_EXECUTIONS_ENV) else { + return Ok(()); + }; + let value = raw.trim(); + let parsed: usize = value.parse().map_err(|_| { + RuntimeBuildError(format!( + "ERR_AGENTOS_RUNTIME_CONFIG: {MAX_ACTIVE_GUEST_EXECUTIONS_ENV} must be an integer between 1 and {MAX_ACTIVE_GUEST_EXECUTIONS_CEILING}, got {value:?}" + )) + })?; + if parsed == 0 || parsed > MAX_ACTIVE_GUEST_EXECUTIONS_CEILING { + return Err(RuntimeBuildError(format!( + "ERR_AGENTOS_RUNTIME_CONFIG: {MAX_ACTIVE_GUEST_EXECUTIONS_ENV} must be between 1 and {MAX_ACTIVE_GUEST_EXECUTIONS_CEILING}, got {parsed}" + ))); + } + self.max_active_guest_executions = parsed; + Ok(()) + } + pub fn validate(&self) -> Result<(), RuntimeBuildError> { for (field, value) in [ ("runtime.workerThreads", self.worker_threads), ( - "runtime.executor.maxActiveVms", - self.max_active_vm_executors, + "runtime.executor.maxActiveGuestExecutions", + self.max_active_guest_executions, ), ( "runtime.blocking.workerThreads", @@ -1114,7 +1183,7 @@ pub struct RuntimeContext { fairness: FairWorkBroker, terminal_failure: Arc>>, task_poll_watchdog: Duration, - max_active_vm_executors: usize, + max_active_guest_executions: usize, vm_executor_teardown_timeout: Duration, blocking_job_timeout: Duration, admission_open: Arc, @@ -1145,8 +1214,8 @@ impl RuntimeContext { &self.metrics } - pub fn max_active_vm_executors(&self) -> usize { - self.max_active_vm_executors + pub fn max_active_guest_executions(&self) -> usize { + self.max_active_guest_executions } pub fn vm_executor_teardown_timeout(&self) -> Duration { @@ -1257,7 +1326,7 @@ impl RuntimeContext { fairness: self.fairness.clone(), terminal_failure: Arc::new(Mutex::new(None)), task_poll_watchdog: self.task_poll_watchdog, - max_active_vm_executors: self.max_active_vm_executors, + max_active_guest_executions: self.max_active_guest_executions, vm_executor_teardown_timeout: self.vm_executor_teardown_timeout, blocking_job_timeout: self.blocking_job_timeout, admission_open, @@ -1535,7 +1604,7 @@ impl SidecarRuntime { fairness, terminal_failure: Arc::new(Mutex::new(None)), task_poll_watchdog: Duration::from_millis(config.task_poll_watchdog_ms), - max_active_vm_executors: config.max_active_vm_executors, + max_active_guest_executions: config.max_active_guest_executions, vm_executor_teardown_timeout: Duration::from_millis( config.vm_executor_teardown_timeout_ms, ), @@ -1598,6 +1667,78 @@ impl SidecarRuntime { mod tests { use super::*; + fn env_override(value: Option<&str>) -> Result { + let mut config = RuntimeConfig::default(); + let value = value.map(str::to_owned); + config.apply_env_overrides_from(|key| { + (key == MAX_ACTIVE_GUEST_EXECUTIONS_ENV) + .then(|| value.clone()) + .flatten() + })?; + Ok(config) + } + + #[test] + fn default_guest_execution_ceiling_does_not_depend_on_host_cpu_count() { + // A CPU-derived ceiling rejected ordinary workloads on small hosts and + // made admitted concurrency differ per machine. Guest executions are + // bounded by threads and memory, not by cores. + assert_eq!( + RuntimeConfig::default().max_active_guest_executions, + DEFAULT_MAX_ACTIVE_GUEST_EXECUTIONS + ); + assert!( + DEFAULT_MAX_ACTIVE_GUEST_EXECUTIONS >= 2, + "a shell and the command it waits on already need two slots" + ); + } + + #[test] + fn absent_guest_execution_override_keeps_the_default() { + let config = env_override(None).expect("absent override is not an error"); + assert_eq!( + config.max_active_guest_executions, + DEFAULT_MAX_ACTIVE_GUEST_EXECUTIONS + ); + } + + #[test] + fn guest_execution_override_applies_and_stays_valid() { + let config = env_override(Some(" 128 ")).expect("surrounding whitespace is accepted"); + assert_eq!(config.max_active_guest_executions, 128); + config.validate().expect("override must stay valid"); + } + + #[test] + fn unusable_guest_execution_override_is_a_typed_error() { + // An operator who asked for a specific ceiling must never silently get a + // different one: no clamping, no falling back to the default. + for value in ["", "many", "0", "-1", "1.5"] { + let error = env_override(Some(value)).expect_err("unusable override must be rejected"); + assert!( + error.to_string().contains(MAX_ACTIVE_GUEST_EXECUTIONS_ENV), + "error must name the variable: {error}" + ); + } + + let above_ceiling = MAX_ACTIVE_GUEST_EXECUTIONS_CEILING + 1; + let error = env_override(Some(&above_ceiling.to_string())) + .expect_err("a request above the hard ceiling must be rejected"); + assert!( + error + .to_string() + .contains(&MAX_ACTIVE_GUEST_EXECUTIONS_CEILING.to_string()), + "error must name the ceiling: {error}" + ); + + let at_ceiling = env_override(Some(&MAX_ACTIVE_GUEST_EXECUTIONS_CEILING.to_string())) + .expect("the ceiling itself is admissible"); + assert_eq!( + at_ceiling.max_active_guest_executions, + MAX_ACTIVE_GUEST_EXECUTIONS_CEILING + ); + } + #[test] fn process_runtime_bounds_every_resource_class_by_default() { let runtime = SidecarRuntime::build(RuntimeConfig::default()).expect("build runtime"); @@ -1650,12 +1791,14 @@ mod tests { .contains("runtime.tasks.maxTerminalReports")); let error = RuntimeConfig { - max_active_vm_executors: 0, + max_active_guest_executions: 0, ..RuntimeConfig::default() } .validate() - .expect_err("zero VM executor capacity must be rejected"); - assert!(error.to_string().contains("runtime.executor.maxActiveVms")); + .expect_err("zero guest-execution capacity must be rejected"); + assert!(error + .to_string() + .contains("runtime.executor.maxActiveGuestExecutions")); let error = RuntimeConfig { vm_executor_teardown_timeout_ms: 0, diff --git a/crates/v8-runtime/src/embedded_runtime.rs b/crates/v8-runtime/src/embedded_runtime.rs index aa83a3de23..93c3a88438 100644 --- a/crates/v8-runtime/src/embedded_runtime.rs +++ b/crates/v8-runtime/src/embedded_runtime.rs @@ -63,7 +63,7 @@ impl EmbeddedV8Runtime { // without immediately evicting each other. let snapshot_cache = Arc::new(SnapshotCache::new(8)); let call_id_router: CallIdRouter = Arc::new(BridgeCallRegistry::with_default_limit()); - let configured_max_concurrency = runtime.max_active_vm_executors(); + let configured_max_concurrency = runtime.max_active_guest_executions(); let executor_teardown_timeout = runtime.vm_executor_teardown_timeout(); let session_mgr = Arc::new(Mutex::new(SessionManager::new( max_concurrency.unwrap_or(configured_max_concurrency), @@ -682,7 +682,7 @@ pub fn spawn_embedded_runtime_ipc( let shutdown_stream = host_stream.try_clone()?; let alive = Arc::new(AtomicBool::new(true)); let alive_for_thread = Arc::clone(&alive); - let max_concurrency = max_concurrency.unwrap_or_else(|| runtime.max_active_vm_executors()); + let max_concurrency = max_concurrency.unwrap_or_else(|| runtime.max_active_guest_executions()); // AGENTOS_THREAD_SITE: embedded-v8-dispatch let join_handle = thread::Builder::new() @@ -1175,7 +1175,7 @@ mod tests { .lock() .expect("embedded runtime codec test lock poisoned"); let mut config = agentos_runtime::RuntimeConfig { - max_active_vm_executors: 2, + max_active_guest_executions: 2, vm_executor_teardown_timeout_ms: 31, ..agentos_runtime::RuntimeConfig::default() }; diff --git a/crates/v8-runtime/src/session.rs b/crates/v8-runtime/src/session.rs index c5139ba709..31c4a8c1e5 100644 --- a/crates/v8-runtime/src/session.rs +++ b/crates/v8-runtime/src/session.rs @@ -1062,12 +1062,15 @@ impl SessionSlotPermit { metrics: RuntimeMetrics, ) -> Result { let (lock, _) = &**control; - let mut active = lock - .lock() - .map_err(|_| String::from("ERR_AGENTOS_VM_EXECUTOR_POISONED: slot lock poisoned"))?; + let mut active = lock.lock().map_err(|_| { + String::from("ERR_AGENTOS_GUEST_EXECUTION_POISONED: slot lock poisoned") + })?; if *active >= maximum { return Err(format!( - "ERR_AGENTOS_VM_EXECUTOR_LIMIT: active V8 executors reached limit of {maximum}; raise runtime.executor.maxActiveVms" + "ERR_AGENTOS_GUEST_EXECUTION_LIMIT: concurrently running guest executions reached the process limit of {maximum}; \ +every live guest process (JavaScript, TypeScript, Python, or WASM command) holds one slot for its whole lifetime, \ +so a parent and the child it waits on need two. Raise runtime.executor.maxActiveGuestExecutions by setting {} on the sidecar process.", + agentos_runtime::MAX_ACTIVE_GUEST_EXECUTIONS_ENV )); } *active += 1; @@ -1090,10 +1093,12 @@ impl Drop for SessionSlotPermit { cvar.notify_all(); } Ok(_) => eprintln!( - "ERR_AGENTOS_VM_EXECUTOR_ACCOUNTING_UNDERFLOW: executor permit released at zero" + "ERR_AGENTOS_GUEST_EXECUTION_ACCOUNTING_UNDERFLOW: executor permit released at zero" ), Err(_) => { - eprintln!("ERR_AGENTOS_VM_EXECUTOR_POISONED: executor permit could not be released") + eprintln!( + "ERR_AGENTOS_GUEST_EXECUTION_POISONED: executor permit could not be released" + ) } } } @@ -4005,7 +4010,7 @@ mod tests { return; } let mut config = agentos_runtime::RuntimeConfig { - max_active_vm_executors: 3, + max_active_guest_executions: 3, vm_executor_teardown_timeout_ms: 23, ..agentos_runtime::RuntimeConfig::default() }; @@ -4016,7 +4021,7 @@ mod tests { let (event_tx, _event_rx) = crossbeam_channel::unbounded(); let router: CallIdRouter = Arc::new(BridgeCallRegistry::with_default_limit()); let mut manager = SessionManager::new( - runtime.max_active_vm_executors(), + runtime.max_active_guest_executions(), event_tx, router, Arc::new(SnapshotCache::new(1)), @@ -4169,7 +4174,12 @@ mod tests { let error = mgr .create_session("s3".into(), None, None, None) .expect_err("third executor must be rejected before thread creation"); - assert!(error.contains("ERR_AGENTOS_VM_EXECUTOR_LIMIT")); + assert!(error.contains("ERR_AGENTOS_GUEST_EXECUTION_LIMIT")); + // The limit error must name a knob an operator can actually set. + assert!( + error.contains(agentos_runtime::MAX_ACTIVE_GUEST_EXECUTIONS_ENV), + "limit error must say how to raise it: {error}" + ); // Allow threads to acquire slots std::thread::sleep(std::time::Duration::from_millis(300)); @@ -4251,7 +4261,7 @@ mod tests { let error = mgr .create_session("two-phase".into(), None, None, None) .expect_err("old generation must retain its executor permit"); - assert!(error.contains("ERR_AGENTOS_VM_EXECUTOR_LIMIT")); + assert!(error.contains("ERR_AGENTOS_GUEST_EXECUTION_LIMIT")); first_shutdown.finish(); mgr.create_session("two-phase".into(), None, None, None) diff --git a/crates/v8-runtime/tests/embedded_runtime_session.rs b/crates/v8-runtime/tests/embedded_runtime_session.rs index 47552d90ee..f04b72b667 100644 --- a/crates/v8-runtime/tests/embedded_runtime_session.rs +++ b/crates/v8-runtime/tests/embedded_runtime_session.rs @@ -489,7 +489,7 @@ fn assert_overload_rejects_before_thread_and_recovers_after_release() -> io::Res assert!( overload .to_string() - .contains("ERR_AGENTOS_VM_EXECUTOR_LIMIT"), + .contains("ERR_AGENTOS_GUEST_EXECUTION_LIMIT"), "unexpected executor overload error: {overload}" ); runtime.unregister_session(&session_b); @@ -592,7 +592,7 @@ fn assert_shared_runtime_handles_share_concurrency_quota() -> io::Result<()> { assert!( overload .to_string() - .contains("ERR_AGENTOS_VM_EXECUTOR_LIMIT"), + .contains("ERR_AGENTOS_GUEST_EXECUTION_LIMIT"), "unexpected shared executor overload error: {overload}" ); clients[3].unregister_session(&session_ids[3]); diff --git a/docs/content/docs/debugging.mdx b/docs/content/docs/debugging.mdx index 8179120a19..6c78af37c3 100644 --- a/docs/content/docs/debugging.mdx +++ b/docs/content/docs/debugging.mdx @@ -32,6 +32,27 @@ const agentOs = await AgentOs.create({ The durable session and committed SQLite history remain available. The crash *reason* is on the adapter's stderr; the exit event reports that the live runtime disappeared. A later explicit prompt performs restoration. See [Sessions](/agentos/docs/sessions). +## Guest executions fail to start + +```text +failed to start guest JavaScript runtime: ERR_AGENTOS_GUEST_EXECUTION_LIMIT: +concurrently running guest executions reached the process limit of 64 +``` + +The sidecar admits a bounded number of concurrently running guest processes +across **all** the VMs it hosts, and every live guest process holds one slot for +its whole lifetime. Fleets of parallel agents hit this well before they exhaust +any per-VM cap. + +Raise it on the host process rather than running more agentOS hosts: + +```bash +AGENTOS_MAX_ACTIVE_GUEST_EXECUTIONS=128 node app.mjs +``` + +See [Process-wide guest-execution cap](/agentos/docs/resource-limits#process-wide-guest-execution-cap) +for how to size it. + ## Runtime logs (sidecar) The agentOS sidecar emits structured **logfmt** logs for request handling, networking, and lifecycle. Configure them with environment variables on the **host process** (the sidecar inherits the host environment): diff --git a/docs/content/docs/resource-limits.mdx b/docs/content/docs/resource-limits.mdx index 98f53eb568..645c19d6f7 100644 --- a/docs/content/docs/resource-limits.mdx +++ b/docs/content/docs/resource-limits.mdx @@ -76,6 +76,63 @@ hints, which Linux is also permitted to ignore. Advice that would discard data or change fork, core-dump, or host VM mapping policy returns `ENOTSUP` because the runtime cannot apply it. +## Process-wide guest-execution cap + +Separate from the per-VM caps above, one sidecar process admits a bounded +number of **concurrently running guest executions** across every VM it hosts. + +Each running guest process — JavaScript, TypeScript, Python, or a WASM command — +owns one OS thread and one V8 isolate. Isolates are thread-affine, so they +cannot be multiplexed onto a shared pool. A slot is held for the **whole +lifetime** of the guest process, not just while it uses CPU: an agent parked on +a network read still holds one, and a shell blocked waiting on a child holds one +while the child holds another. + +| Setting | Scope | Default | +|---|---|---| +| `AGENTOS_MAX_ACTIVE_GUEST_EXECUTIONS` | The whole sidecar process, shared by every VM | `64` | + +Set it in the environment of the process that spawns the sidecar; agentOS +clients pass their environment to the sidecar they spawn. + +```bash +AGENTOS_MAX_ACTIVE_GUEST_EXECUTIONS=128 node ./my-agent-host.js +``` + +Valid values are `1` to `1024`. A value that is missing, non-numeric, zero, or +above the ceiling fails sidecar startup with a typed error rather than being +clamped — an operator who asks for a specific ceiling never silently gets a +different one. + +The sidecar logs the effective ceiling at startup, so you can confirm what a +process actually admitted: + +```text +INFO guest execution admission ceiling max_active_guest_executions=128 +``` + +This is **process topology, not a per-VM limit**, so it is deliberately not a +field on the `limits` config: one sidecar process is shared by every VM and +connection it hosts, and no single tenant may rewrite the ceiling for its +neighbours. Read it once, before any VM exists. + +### Sizing it + +Size to **peak concurrent guest processes**, not to VM or agent count. A useful +rule for agent fleets is two to three times the number of agents you run in +parallel: each agent holds one slot for its whole session, and every tool call +that shells out holds another for as long as it runs. + +At the limit, the guest execution fails immediately — there is no queue — with +`ERR_AGENTOS_GUEST_EXECUTION_LIMIT` naming the current ceiling and this +variable. Raising the ceiling costs one OS thread and one isolate per added +slot; the isolate's heap stays bounded by `limits.jsRuntime.v8HeapLimitMb`. + +Running several agentOS hosts to work around this cap is usually the wrong +trade: each extra sidecar process duplicates the fixed V8 background pool, +runtime workers, warm-isolate pool, and snapshot cache, and splits warm-start +caching across processes. Prefer one host with a higher ceiling. + ## Sidecar liveness Separate from the guest caps above, the host detects a dead or wedged sidecar