From f93b24a501ad8982ff67f54d2f710d385a529149 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 00:43:43 -0500 Subject: [PATCH 01/22] feat(game): zones watch world cells instead of polling them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit World cells are the write model; each zone keeps a pushed read model of the cells it declares in CelldZone::watch. A watching read leaves an in-memory lease, the cell pushes {topic, version, view} after commits and alarms that change its view, the zone keeps the highest version per topic and reconciles every 45 ± 15 s. Content rollout and dev settings ride the same path, so the 1 s polls are gone; Mob Pit's hood, borough and city reads become typed watches. A view unchanged apart from its version is not pushed, so N rooms waking (room_awake) do not cost N² pushes. [[zone-world-views-pushed]] [[world-scale-game-services]] --- distributed_game/Cargo.toml | 2 +- distributed_game/README.md | 41 +- distributed_game/src/celld_zone/cells.rs | 270 ++++++++++++- distributed_game/src/celld_zone/mod.rs | 25 +- distributed_game/src/celld_zone/object.rs | 311 +++++++++----- distributed_game/src/world/mod.rs | 5 + distributed_game/src/world/watch.rs | 469 ++++++++++++++++++++++ examples/game/cells/src/lib.rs | 64 ++- examples/game/server/rooms/src/celld.rs | 97 ++--- examples/game/server/rooms/src/zone.rs | 72 +++- examples/game/server/tiers/src/lib.rs | 54 +++ 11 files changed, 1216 insertions(+), 194 deletions(-) create mode 100644 distributed_game/src/world/watch.rs diff --git a/distributed_game/Cargo.toml b/distributed_game/Cargo.toml index 898796225..6b305fc18 100644 --- a/distributed_game/Cargo.toml +++ b/distributed_game/Cargo.toml @@ -28,7 +28,7 @@ celld = ["cell", "command-host", "dep:reqwest", "dep:uuid"] workers-rs = ["cell", "dep:worker", "distributed/workers-rs"] # Zones as celld Durable Objects (wasm32): `celld_zone::ZoneObject`, # `CelldObjectHost` (facts object to object), `celld_zone_object!`. -celld-zone = ["cell", "ticket", "workers-rs", "dep:js-sys", "dep:wasm-bindgen", "dep:wasm-bindgen-futures"] +celld-zone = ["cell", "ticket", "workers-rs", "dep:js-sys", "dep:wasm-bindgen", "dep:wasm-bindgen-futures", "dep:futures-util"] # Per-zone Prometheus metrics (dependency-free text exposition). metrics = [] # Load-test bots (library + `game-bots` binary). diff --git a/distributed_game/README.md b/distributed_game/README.md index c7ea7e625..5b2c6ed7a 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -831,6 +831,42 @@ them exactly once across crashes), and the zone's lifecycle reports [`zone_activity`](src/world/zone_activity.rs) cell, which the `zone_status` read model projects for minimaps. +**The world around: watched cells (CQRS).** World cells (a hood, a +borough, the content rollout, the dev settings) are the write model; each +zone keeps a small read model of the ones it cares about. The zone +declares them; each cell's view decodes into a plain struct and becomes a +normal zone message: + +```rust,ignore +impl CelldZone for MobPit { + fn watch(zone: &str) -> Watches { + Watches::new() + .cell(HOOD_KIND, &hood_of(zone), |read: HoodRead| read.state.map(PitMessage::hood)) + .cell(CITY_KIND, CITY_SHARD, |read: CityRead| read.state.map(PitMessage::city)) + } +} + +// the cell: one field and two calls (reads leave a lease, commits push) +self.watchers.serve(&self.env, &self.cell, req, read_hood, sessions).await +self.watchers.alarm(&self.env, &self.cell, read_hood).await +``` + +The zone reads each cell on wake (`GET …?watch=PIT_ROOM/pit.0.0`, which +leaves a lease in the cell's memory — no write); after every commit or +alarm that moves the aggregate's version the cell pushes `{topic, +version, view}` to the live watchers ([`CellWatchers`](src/celld_zone/cells.rs)); +the zone applies only newer versions and re-reads every 45 ± 15 s, renewing +its lease and repairing a lost push. A command that moves the aggregate but +not its view (a heartbeat such as Mob Pit's `room_awake`) pushes nothing, +so N rooms waking do not cost N² pushes. A push never wakes a sleeping zone +(it hydrates on wake). Load is O(view changes × watchers + zones / 45 s), +not every zone reading every cell every second: on one laptop node the +refusals of saturated world cells (celld's 64-in-flight cap) fell from 85 +to 25 with 128 one-player Mob Pit rooms and from 12 to 3 with 64 fighting +ones (numbers in the `mobpit-zone-load-test` task). Read models are for +display and world flavour; rewards stay on the fact → command path. +`CelldZone::refresh` remains for other async reads. + **Limits.** A live zone never moves (only idle objects are rebalanced), so load spreads by first touch and node caps. Objects of one class share V8 isolates: a slow zone delays its neighbours. A node crash takes about the @@ -1181,8 +1217,8 @@ principals commit concurrently; settlements arrive in completion order. (`celld-zone`) apply the same simulation to each socket when the worker sets `ZONE_DEV_SETTINGS=1`; the setting lives in the worker's dev settings object (`celld_zone::ZoneDevSettings`, `PUT /zone-dev/netsim` - `{latencyMs, jitterMs, loss}` with the internal secret) and every zone - object reads it within a second. + `{latencyMs, jitterMs, loss}` with the internal secret), pushed to every + zone object watching it (`world::watch`). ## World structure @@ -1194,6 +1230,7 @@ principals commit concurrently; settlements arrive in completion order. | `ZoneOwnership` (`LocalOwnership`, `FencedLeaseOwnership`, `celld::CellZoneOwnership`) | every activation acquires a fenced claim (`host#activation`) and renews it; a failed renewal disposes the zone **without** saving. Two hosts never run the same zone. `CellZoneOwnership` keeps each claim in a celld `ZoneLease:{zone}` cell (`world::zone_lease`, mount with `mount_zone_lease`): acquire and release publish `zone.activated` / `zone.released`, renewals are internal (no outward events), and the lease's expiry schedule (`ctx.schedule`; the routes require `Capability::Scheduling` and `Clock`) publishes `zone.lease_expired` when a holder stops renewing, so services see a crashed zone server; each event carries the lease's `ZoneLeaseStatus { zone_id, holder, token, status }`. `world::ownership_conformance` is the behaviour every implementation passes. | | `ZoneStateStore` (`celld::CellZoneStateStore`; `MemoryZoneStateStore` for unit tests) | hibernation snapshots (async `load` / `save` / `delete`), written under the claim's fencing token (stale writers get `StateError::Stale`, counted in `distributed_game_hibernation_saves_total{outcome="stale"}` and logged to stderr, never silently dropped). On celld each zone's snapshot is its own `ZoneState:{zone}` cell (`world::zone_state`): inline up to 64 KiB, larger ones in the worker's R2 bucket with the digest in the cell; saves and deletes publish `zone_state.saved` / `zone_state.deleted` with a `ZoneSnapshotStatus { zone_id, token, stored, content_version, bytes }` (no bytes). `LocalOwnership` seeds its tokens from the Unix time (µs) at creation, so a restarted host still outranks the snapshots its previous run saved. | | `world::zone_status` (feature `cell`) | the read model of those events: `zone_leases` and `zone_snapshots` rows (`projection!` upserts of the state bodies), `projectors()` to mount on any consumer service and read-model store, and `ZoneStatus::of` → `Active { holder }` / `Hibernated { bytes }` / `Cold`. For views (world maps, minimaps): it answers "which zones are active or asleep?" without opening a cell per zone (on celld a read of a never-used cell creates it). Eventually consistent; ownership and restore always go to the cells. | +| `world::watch` (`Watches`, `Watchers`, `TopicVersions`) | watched world views: a zone's typed list of the cells it watches; a cell's in-memory watcher leases; the zone's per-topic version guard (a late push or read never rolls back). On celld: `CelldZone::watch` + `celld_zone::CellWatchers`. | | `ZoneTopology` (`DottedTopology` default: `eldoria.port.docks` → `eldoria.port` → `eldoria`) | parent links for tier-up events. | **Per-zone host settings.** `GameServer::with_host_config(|zone_id, host| diff --git a/distributed_game/src/celld_zone/cells.rs b/distributed_game/src/celld_zone/cells.rs index 34ef151d0..33943e65d 100644 --- a/distributed_game/src/celld_zone/cells.rs +++ b/distributed_game/src/celld_zone/cells.rs @@ -1,5 +1,8 @@ //! Serving fact cells next to celld zones (wasm32): the internal wait-path -//! route every fact cell of a game worker needs, and the zone-activity cell. +//! route every fact cell of a game worker needs, watched world cells +//! ([`CellWatchers`]), and the zone-activity cell. + +use std::cell::RefCell; use distributed::cell_host::{ AggregateCell, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, CelldOutbox, @@ -9,10 +12,13 @@ use distributed::cell_host::{ use distributed::microsvc::Session; use distributed::Aggregate; use serde_json::{json, Value}; -use worker::{Env, Method, Request, Response, Result, State}; +use worker::{Env, Headers, Method, Request, RequestInit, Response, Result, State}; use crate::cell::fact_session_from_headers; use crate::world::content_rollout::{mount_content_rollout, ContentRollout}; +use crate::world::watch::{ + parse_watcher_address, topic, watch_query, WatchUpdate, Watchers, PUSH_CONCURRENCY, PUSH_ROUTE, +}; use crate::world::zone_activity::{mount_zone_activity, ZoneActivity}; /// Builds the handler session from trusted request headers (after the @@ -137,6 +143,192 @@ where } } +/// A watched world cell's watchers ([`crate::world::watch`]): zones that +/// read the cell with `?watch=` get its view pushed after every commit (and +/// every alarm) that moves its version. Memory only. +/// +/// ```ignore +/// pub struct HoodCell { cell: AggregateCell, env: Env, watchers: CellWatchers } +/// // fetch: self.watchers.serve(&self.env, &self.cell, req, read_hood, sessions).await +/// // alarm: self.watchers.alarm(&self.env, &self.cell, read_hood).await +/// ``` +pub struct CellWatchers { + topic: String, + watchers: RefCell, +} + +impl CellWatchers { + /// The watchers of cell `kind` whose Durable Object is `state` (its + /// name is the shard). + pub fn new(kind: &str, state: &State) -> Self { + Self::of(kind, &state.id().name().unwrap_or_default()) + } + + /// The watchers of cell `kind/shard`. + pub fn of(kind: &str, shard: &str) -> Self { + Self { + topic: topic(kind, shard), + watchers: RefCell::new(Watchers::new()), + } + } + + /// [`serve_cell`], plus: a read with `?watch=` leaves a lease and its + /// body carries the cell's `version`; a command that changed the view + /// is pushed to the live watchers. + pub async fn serve( + &self, + env: &Env, + cell: &AggregateCell, + req: Request, + read: F, + sessions: Sessions, + ) -> Result + where + A: Aggregate + Send + Sync + 'static, + F: AsyncFn(&AggregateCell) -> std::result::Result, + { + if req.method() == Method::Get { + if let Err(error) = authenticate(&req, env) { + return error_response(error); + } + let url = req.url()?; + if let Some((watcher, lease)) = url.query().and_then(watch_query) { + self.watchers.borrow_mut().watch(&watcher, now_ms(), lease); + } + } + let is_command = req.method() == Method::Post; + let response = if req.method() == Method::Get { + match versioned(cell, &read).await { + Ok(body) => json_status(body, 200), + Err(error) => error_response(CellDispatchError::Internal(error)), + } + } else { + serve_cell(env, cell, req, &read, sessions).await + }; + if is_command { + self.publish(env, cell, &read).await; + } + response + } + + /// [`cell_alarm`], then a push if a schedule moved the version. + pub async fn alarm(&self, env: &Env, cell: &AggregateCell, read: F) -> Result + where + A: Aggregate + Send + Sync + 'static, + F: AsyncFn(&AggregateCell) -> std::result::Result, + { + let response = cell_alarm(env, cell).await; + self.publish(env, cell, &read).await; + response + } + + /// Push the cell's view to its live watchers when it changed (best + /// effort, off the request; a lost push is repaired by the + /// watcher's next reconcile read). + async fn publish(&self, env: &Env, cell: &AggregateCell, read: &F) + where + A: Aggregate + Send + Sync + 'static, + F: AsyncFn(&AggregateCell) -> std::result::Result, + { + let live = self.watchers.borrow_mut().live(now_ms()); + if live.is_empty() { + return; + } + let Ok(view) = versioned(cell, read).await else { + return; + }; + let version = crate::world::watch::view_version(&view); + if !self.watchers.borrow_mut().publish(version, &view) { + return; + } + let update = WatchUpdate { + topic: self.topic.clone(), + version, + view, + }; + push_all(env.clone(), live, update); + } +} + +/// `read(cell)` with the aggregate's version in its `version` field. +async fn versioned(cell: &AggregateCell, read: &F) -> std::result::Result +where + A: Aggregate + Send + Sync + 'static, + F: AsyncFn(&AggregateCell) -> std::result::Result, +{ + let version = cell + .load() + .await + .map_err(|error| error.to_string())? + .map_or(0, |aggregate| aggregate.entity().version()); + let mut body = read(cell).await?; + if let Value::Object(fields) = &mut body { + fields.insert("version".into(), json!(version)); + } + Ok(body) +} + +/// Push `update` to `watchers`, [`PUSH_CONCURRENCY`] at a time, off the +/// caller. +pub(crate) fn push_all(env: Env, watchers: Vec, update: WatchUpdate) { + wasm_bindgen_futures::spawn_local(async move { + let Some(secret) = internal_secret(&env) else { + return; + }; + let Ok(body) = serde_json::to_string(&update) else { + return; + }; + for chunk in watchers.chunks(PUSH_CONCURRENCY) { + let pushes = chunk + .iter() + .map(|watcher| push(&env, watcher, &secret, &body)); + futures_util::future::join_all(pushes).await; + } + }); +} + +async fn push(env: &Env, watcher: &str, secret: &str, body: &str) { + let Some((binding, name)) = parse_watcher_address(watcher) else { + return; + }; + let result: Result<()> = async { + let stub = env + .durable_object(binding)? + .id_from_name(name)? + .get_stub()?; + let headers = Headers::new(); + headers.set(CELL_INTERNAL_SECRET_HEADER, secret)?; + headers.set(super::ZONE_HEADER, name)?; + headers.set("content-type", "application/json")?; + let mut init = RequestInit::new(); + init.with_method(Method::Post) + .with_headers(headers) + .with_body(Some(body.into())); + let url = format!("https://zone/{name}/{PUSH_ROUTE}"); + stub.fetch_with_request(Request::new_with_init(&url, &init)?) + .await?; + Ok(()) + } + .await; + if let Err(error) = result { + worker::console_warn!("world push to {watcher}: {error}"); + } +} + +fn internal_secret(env: &Env) -> Option { + env.secret(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + .or_else(|_| { + env.var(CELL_INTERNAL_SECRET_ENV) + .map(|value| value.to_string()) + }) + .ok() +} + +fn now_ms() -> u64 { + js_sys::Date::now() as u64 +} + /// A cell's alarm: due schedules, then the outbox drain. pub async fn cell_alarm(env: &Env, cell: &AggregateCell) -> Result where @@ -194,16 +386,25 @@ async fn read_activity(cell: &AggregateCell) -> std::result::Resul /// network simulation under shard [`NETSIM_SHARD`](super::NETSIM_SHARD). /// `GET /zone-dev/{shard}` returns the stored JSON (`{}` when unset), `PUT` /// (or `POST`) replaces it. Every request needs the internal secret. Zone -/// objects read it only when `ZONE_DEV_SETTINGS=1`. +/// objects watch it only when `ZONE_DEV_SETTINGS=1` (a `GET` with `?watch=`; +/// each write is pushed to the watchers with the next `version`). pub struct ZoneDevSettings { state: State, env: Env, + watchers: RefCell, + topic: String, } impl ZoneDevSettings { /// The settings object of `state`. pub fn new(state: State, env: Env) -> Self { - Self { state, env } + let topic = topic(super::ZONE_DEV_KIND, &state.id().name().unwrap_or_default()); + Self { + state, + env, + watchers: RefCell::new(Watchers::new()), + topic, + } } /// Durable Object `fetch`. @@ -212,10 +413,26 @@ impl ZoneDevSettings { return error_response(error); } const KEY: &str = "settings"; + const VERSION: &str = "version"; match req.method() { Method::Get => { + if let Some((watcher, lease)) = req.url()?.query().and_then(watch_query) { + self.watchers.borrow_mut().watch(&watcher, now_ms(), lease); + } let stored: Option = self.state.storage().get(KEY).await.ok().flatten(); - json_status(stored.unwrap_or_else(|| json!({})), 200) + let version: u64 = self + .state + .storage() + .get(VERSION) + .await + .ok() + .flatten() + .unwrap_or(0); + let mut body = stored.unwrap_or_else(|| json!({})); + if let Value::Object(fields) = &mut body { + fields.insert(VERSION.into(), json!(version)); + } + json_status(body, 200) } Method::Put | Method::Post => { let body: Value = match req.json().await { @@ -226,7 +443,33 @@ impl ZoneDevSettings { )) } }; + let version: u64 = self + .state + .storage() + .get(VERSION) + .await + .ok() + .flatten() + .unwrap_or(0) + + 1; self.state.storage().put(KEY, &body).await?; + self.state.storage().put(VERSION, version).await?; + let live = self.watchers.borrow_mut().live(now_ms()); + let mut view = body.clone(); + if let Value::Object(fields) = &mut view { + fields.insert(VERSION.into(), json!(version)); + } + if !live.is_empty() && self.watchers.borrow_mut().publish(version, &view) { + push_all( + self.env.clone(), + live, + WatchUpdate { + topic: self.topic.clone(), + version, + view, + }, + ); + } json_status(body, 200) } _ => json_status(json!({ "error": "not found" }), 404), @@ -241,16 +484,20 @@ impl ZoneDevSettings { /// `sessions` decides who may start a rollout: it must give the admin's /// session the [`CONTENT_ADMIN_ROLE`](crate::world::content_rollout::CONTENT_ADMIN_ROLE) /// (the cell's own schedules need no session). `GET` returns -/// `{"state": ContentState | null}`. +/// `{"state": ContentState | null, "version": n}`; zones watch it +/// ([`CellWatchers`]). pub struct ContentRolloutCell { cell: AggregateCell, env: Env, sessions: Sessions, + watchers: CellWatchers, } impl ContentRolloutCell { /// Open the cell of `state`, publishing its events to `outbox`. pub fn new(state: State, env: Env, outbox: CelldOutbox, sessions: Sessions) -> Self { + let watchers = + CellWatchers::new(crate::world::content_rollout::CONTENT_ROLLOUT_KIND, &state); let cell = mount_content_rollout( AggregateCell::::from_state_with_snapshots(state, 20) .expect("content rollout cell identity"), @@ -260,17 +507,22 @@ impl ContentRolloutCell { cell, env, sessions, + watchers, } } - /// Durable Object `fetch`. + /// Durable Object `fetch` (zones watch it: every phase is pushed). pub async fn fetch(&self, req: Request) -> Result { - serve_cell(&self.env, &self.cell, req, read_rollout, self.sessions).await + self.watchers + .serve(&self.env, &self.cell, req, read_rollout, self.sessions) + .await } /// Durable Object `alarm` (the rollout's own schedule). pub async fn alarm(&self) -> Result { - cell_alarm(&self.env, &self.cell).await + self.watchers + .alarm(&self.env, &self.cell, read_rollout) + .await } } diff --git a/distributed_game/src/celld_zone/mod.rs b/distributed_game/src/celld_zone/mod.rs index 65b10f5f5..16ad4fb91 100644 --- a/distributed_game/src/celld_zone/mod.rs +++ b/distributed_game/src/celld_zone/mod.rs @@ -42,10 +42,15 @@ //! the object's outbox before it is sent and deleted when settled; the //! next activation or the outbox alarm resends what is left, and the //! cells' command ledger turns a resend into a replay: exactly once. -//! - **The world around.** [`CelldZone::refresh`] runs off the tick on -//! every wake and every [`CelldZone::refresh_every`] while the zone -//! ticks (e.g. reading the cells of the tiers a room belongs to); its -//! message is delivered into the zone. +//! - **The world around.** A zone *watches* the world cells it cares +//! about ([`CelldZone::watch`], [`crate::world::watch`]): each is read +//! on wake, then pushed by its cell ([`CellWatchers`]) after every +//! change, and re-read every ~45 s (renewing the zone's in-memory watch +//! lease, repairing a lost push). Each view decodes into a plain struct +//! that becomes a zone message. The object takes pushes at its class +//! binding (`celld_zone_object!(PitRoom, …)` → `PIT_ROOM`). Custom async +//! reads still go through [`CelldZone::refresh`] / +//! [`CelldZone::refresh_every`]. //! - **Status.** The zone reports its lifecycle as facts into its //! [`zone_activity`](crate::world::zone_activity) cell, projected into the //! [`zone_status`](crate::world::zone_status) read model (minimaps). @@ -86,8 +91,8 @@ //! **Live content updates.** With `ZONE_CONTENT_ROLLOUT=` every //! zone object follows the world's //! [`content_rollout`](crate::world::content_rollout) cell -//! ([`ContentRolloutCell`], binding `CONTENT_ROLLOUT`): it reads it on wake -//! and every second while it ticks and runs +//! ([`ContentRolloutCell`], binding `CONTENT_ROLLOUT`): it watches the cell +//! (each phase is pushed) and runs //! [`zone_step`](crate::world::content_rollout::zone_step) — freeze with //! the change notes, then switch to the new version and resume (a hard //! rollout reloads stale clients at once). A zone that slept through a @@ -100,9 +105,9 @@ //! Dev only: with `ZONE_DEV_SETTINGS=1` every zone object applies the //! network simulation ([`NetSim`](crate::netsim::NetSim): latency, jitter, //! loss per socket and direction) stored in the worker's -//! [`ZoneDevSettings`] object (`zone-dev/netsim`), read on wake and every -//! second while it ticks; a dev route writes it (one setting for every -//! zone, like the native server's `set_net_sim`). +//! [`ZoneDevSettings`] object (`zone-dev/netsim`), watched like the content +//! rollout (each write is pushed); a dev route writes it (one setting for +//! every zone, like the native server's `set_net_sim`). //! //! Debug only: celld does not tell application code which node runs an //! object. With `CELLD_ZONE_NODES` set (operator listeners) the object asks @@ -123,7 +128,7 @@ pub use plan::{ #[cfg(target_arch = "wasm32")] pub use cells::{ - authenticate, cell_alarm, error_response, fact_sessions, json_status, serve_cell, + authenticate, cell_alarm, error_response, fact_sessions, json_status, serve_cell, CellWatchers, ContentRolloutCell, Sessions, ZoneActivityCell, ZoneDevSettings, }; #[cfg(target_arch = "wasm32")] diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index 30e953af3..38ac23da3 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -37,6 +37,10 @@ use crate::protocol::{error, ClientMessage, Notice, Role, ServerMessage, PROTOCO use crate::stamp::FactStamp; use crate::ticket::{NonceCache, TicketKey, VerifiedTicket}; use crate::world::content_rollout::{zone_step, ContentState, ZoneStep, CONTENT_ROLLOUT_KIND}; +use crate::world::watch::{ + next_reconcile_ms, topic, view_version, watch_query_string, watcher_address, TopicVersions, + WatchUpdate, Watches, PUSH_ROUTE, WATCH_LEASE_MS, +}; use crate::world::zone_activity::{ zone_principal, RecordActivity, ACTIVITY_ACTIVE, ACTIVITY_DORMANT, ACTIVITY_HIBERNATED, ZONE_ACTIVITY_KIND, @@ -62,8 +66,9 @@ const FACT_CAPACITY: usize = 512; const OUTBOX_RETRY_MS: f64 = 10_000.0; /// Shortest crash watchdog period (each re-arm is a durable alarm write). const WATCHDOG_MIN_MS: f64 = 60_000.0; -/// Read the dev settings this often while ticking (ms). -const DEV_SETTINGS_EVERY_MS: f64 = 1_000.0; +/// Re-read watched cells this often when the object cannot take pushes +/// (no class binding: built with [`ZoneObject::new`] and no `ZONE_BINDING`). +const POLL_MS: f64 = 1_000.0; /// Server → client, through the socket's simulated network. enum OutItem { @@ -112,8 +117,23 @@ pub trait CelldZone: Zone { None } + /// The world cells the zone watches ([`Watches`]): each is read on + /// wake, pushed by its cell after every change and re-read every ~45 s; + /// its view becomes a message ([`Zone::on_message`]). Eventually + /// consistent: world flavour, never reward settlement. + /// + /// ```ignore + /// fn watch(zone_id: &str) -> Watches { + /// Watches::new().cell(HOOD_KIND, &hood_of(zone_id), |read: HoodRead| read.state.map(PitMessage::Hood)) + /// } + /// ``` + fn watch(_zone_id: &str) -> Watches { + Watches::new() + } + /// How often [`refresh`](Self::refresh) runs while the zone ticks - /// (`None`: never). It also runs once on every wake. + /// (`None`: never). It also runs once on every wake. For cell state, + /// prefer [`watch`](Self::watch) (pushed, no polling). fn refresh_every() -> Option { None } @@ -163,6 +183,18 @@ impl ObjectEnv { /// `GET /{kind}/{shard}` on a cell of this worker (its read route), /// object to object, with the internal secret. pub async fn read_cell(&self, kind: &str, shard: &str) -> std::result::Result { + self.read_cell_watching(kind, shard, None).await + } + + /// [`read_cell`](Self::read_cell) that also watches the cell for + /// `watcher` ([`crate::world::watch`]): the cell pushes its next + /// versions until the lease ends. + pub async fn read_cell_watching( + &self, + kind: &str, + shard: &str, + watcher: Option<&str>, + ) -> std::result::Result { let secret = self .var(CELL_INTERNAL_SECRET_ENV) .ok_or_else(|| format!("{CELL_INTERNAL_SECRET_ENV} is required"))?; @@ -172,7 +204,11 @@ impl ObjectEnv { .map_err(|e| e.to_string())?; let mut init = RequestInit::new(); init.with_method(Method::Get).with_headers(headers); - let url = format!("https://cell/{kind}/{}", encode_segment(shard)); + let mut url = format!("https://cell/{kind}/{}", encode_segment(shard)); + if let Some(watcher) = watcher { + url.push('?'); + url.push_str(&watch_query_string(watcher, WATCH_LEASE_MS)); + } let request = Request::new_with_init(&url, &init).map_err(|e| e.to_string())?; let stub = stub_for(&self.env, kind, shard).map_err(|e| e.to_string())?; let mut response = stub @@ -447,9 +483,13 @@ struct Inner { drain_at: Option, /// Spawns a drain task waking at the given time (set by the object). drainer: Option>, - last_dev_read_ms: f64, - /// Last read of the world's content rollout cell. - last_content_read_ms: f64, + /// Watched world cells ([`CelldZone::watch`] plus the dev settings and + /// content rollout), built on the first wake. + watches: Option>>, + /// Highest version applied per watched topic (this activation). + world_versions: TopicVersions, + /// When each watched topic is read next (ms). + world_due: HashMap, /// [`CelldZone::refresh`] calls of this activation, and the last one. refresh_round: u64, last_refresh_ms: f64, @@ -706,10 +746,24 @@ pub struct ZoneObject { state: Rc, env: ObjectEnv, inner: Rc>>, + /// Namespace binding of the object's class (`PIT_ROOM`): where watched + /// cells push. Empty: no pushes, watched cells are polled. + binding: Rc, } impl ZoneObject { - /// The object for `state` (constructor of the Durable Object). + /// The object of Durable Object class `class` (bound, by convention, as + /// [`class_binding`](crate::world::watch::class_binding)`(class)`, e.g. + /// `PitRoom` → `PIT_ROOM`); [`crate::celld_zone_object!`] calls it. + pub fn of_class(state: State, env: Env, class: &str) -> Self { + let mut object = Self::new(state, env); + object.binding = crate::world::watch::class_binding(class).into(); + object + } + + /// The object for `state` (constructor of the Durable Object). Watched + /// cells push to it only if `ZONE_BINDING` names its class binding; + /// prefer [`of_class`](Self::of_class). pub fn new(state: State, env: Env) -> Self { let zone_id = state.id().name().unwrap_or_default(); let env = ObjectEnv::new(env); @@ -758,8 +812,9 @@ impl ZoneObject { in_lines: HashMap::new(), drain_at: None, drainer: None, - last_dev_read_ms: 0.0, - last_content_read_ms: 0.0, + watches: None, + world_versions: TopicVersions::new(), + world_due: HashMap::new(), refresh_round: 0, last_refresh_ms: 0.0, }; @@ -768,10 +823,12 @@ impl ZoneObject { inner.borrow_mut().drainer = Some(Rc::new(move |due: f64| { wasm_bindgen_futures::spawn_local(drain_task(weak.clone(), due)); })); + let binding = env.var("ZONE_BINDING").unwrap_or_default().into(); Self { state: Rc::new(state), env, inner, + binding, } } @@ -792,6 +849,9 @@ impl ZoneObject { return Response::error("zone id required", 400); } let path = req.url()?.path().to_string(); + if req.method() == Method::Post && path.ends_with(&format!("/{PUSH_ROUTE}")) { + return self.take_push(req).await; + } if path.ends_with("/info") { // Debug JSON without secrets; pages on other origins may read it. let response = Response::from_json(&self.info())?; @@ -1101,7 +1161,7 @@ impl ZoneObject { host_config.reconnect_grace = options.reconnect_grace; host_config.boot_epoch = (now_ms() * 1000.0) as u64; // The content version the zone slept on (a rollout it missed is - // caught up by `read_content`). + // caught up by its first content rollout read). if let Some(snapshot) = &stored { host_config.content_version = snapshot.version.clone(); } @@ -1174,15 +1234,14 @@ impl ZoneObject { } self.store_status().await; self.find_node(); - // Off the join path: a wake (often a doorway into a sleeping zone, - // with the player's inputs waiting) does not wait for dev settings. - let this = self.handle(); - wasm_bindgen_futures::spawn_local(async move { this.read_dev_settings().await }); - // The snapshot restored the zone's last content version; catch up - // with a rollout it slept through (off the join path too: a Hello - // on another version waits for its own read, see `hello`). - let this = self.handle(); - wasm_bindgen_futures::spawn_local(async move { this.read_content().await }); + // Hydrate every watched cell, off the join path: a wake (often a + // doorway into a sleeping zone, with the player's inputs waiting) + // does not wait for them. The snapshot restored the zone's last + // content version; the rollout read catches up with one it slept + // through (a Hello on another version waits for its own read, see + // `hello`). + self.inner.borrow_mut().world_versions.clear(); + self.reconcile_world(true); self.refresh(); Ok(()) } @@ -1222,68 +1281,148 @@ impl ZoneObject { } } - /// Dev only (`ZONE_DEV_SETTINGS=1`): read the network simulation from - /// the worker's dev settings object. - async fn read_dev_settings(&self) { - let enabled = { - let mut inner = self.inner.borrow_mut(); - inner.last_dev_read_ms = now_ms(); - inner.options.as_ref().is_some_and(|o| o.dev_settings) + /// The world whose content rollout the zone follows, if any. + fn content_world(&self) -> Option { + let inner = self.inner.borrow(); + inner + .options + .as_ref() + .and_then(|o| o.content_rollout.clone()) + } + + /// Every watched topic: the dev settings (`ZONE_DEV_SETTINGS=1`), the + /// content rollout (`ZONE_CONTENT_ROLLOUT`) and the game's + /// [`CelldZone::watch`]. + fn world_topics(&self) -> Vec { + let (dev, zone_id) = { + let inner = self.inner.borrow(); + let dev = inner.options.as_ref().is_some_and(|o| o.dev_settings); + (dev, inner.zone_id.clone()) }; - if !enabled { - return; - } - match self.env.read_cell(ZONE_DEV_KIND, NETSIM_SHARD).await { - Ok(value) => { - let sim = serde_json::from_value::(value) - .unwrap_or_default() - .to_sim(); - self.inner.borrow_mut().net_sim = sim; + let watches = self + .inner + .borrow_mut() + .watches + .get_or_insert_with(|| Rc::new(Z::watch(&zone_id))) + .clone(); + let mut topics = Vec::with_capacity(watches.len() + 2); + if dev { + topics.push(topic(ZONE_DEV_KIND, NETSIM_SHARD)); + } + if let Some(world) = self.content_world() { + topics.push(topic(CONTENT_ROLLOUT_KIND, &world)); + } + topics.extend(watches.iter().map(|w| w.topic())); + topics + } + + /// Read the watched topics that are due (all of them with `all`, on + /// wake) off the tick. With a class binding each read renews the + /// zone's watch and the next comes ~45 s later (pushes carry the + /// changes); without one the cells are polled every [`POLL_MS`]. + fn reconcile_world(&self, all: bool) { + let now = now_ms(); + let pushed = !self.binding.is_empty(); + for topic in self.world_topics() { + let due = { + let mut inner = self.inner.borrow_mut(); + let next = inner.world_due.get(&topic).copied().unwrap_or(0.0); + if !all && now < next { + continue; + } + let after = if pushed { + next_reconcile_ms(now as u64, js_sys::Math::random()) as f64 + } else { + now + POLL_MS + }; + inner.world_due.insert(topic.clone(), after); + true + }; + if due { + let this = self.handle(); + wasm_bindgen_futures::spawn_local(async move { this.reconcile(&topic).await }); } - Err(e) => worker::console_warn!("zone dev settings: {e}"), } } - /// [`read_dev_settings`](Self::read_dev_settings) off the tick, at most - /// every [`DEV_SETTINGS_EVERY_MS`]. - fn refresh_dev_settings(&self) { - let due = { - let inner = self.inner.borrow(); - inner.options.as_ref().is_some_and(|o| o.dev_settings) - && now_ms() - inner.last_dev_read_ms >= DEV_SETTINGS_EVERY_MS + /// Read watched `topic` (`kind/shard`), watching it when the object + /// can take pushes, and apply its view. + async fn reconcile(&self, topic: &str) { + let Some((kind, shard)) = topic.split_once('/') else { + return; }; - if due { - self.inner.borrow_mut().last_dev_read_ms = now_ms(); - let this = self.handle(); - wasm_bindgen_futures::spawn_local(async move { this.read_dev_settings().await }); + let watcher = (!self.binding.is_empty()).then(|| { + let zone_id = self.inner.borrow().zone_id.clone(); + watcher_address(&self.binding, &zone_id) + }); + match self + .env + .read_cell_watching(kind, shard, watcher.as_deref()) + .await + { + Ok(view) => self.apply_view(topic, view), + Err(e) => worker::console_warn!("world view {topic}: {e}"), } } - /// Read the world's content rollout cell and follow it - /// ([`zone_step`]: freeze, apply + resume). - async fn read_content(&self) { - let world = { - let mut inner = self.inner.borrow_mut(); - inner.last_content_read_ms = now_ms(); - inner - .options - .as_ref() - .and_then(|o| o.content_rollout.clone()) + /// A pushed update (`POST …/__world`, internal secret): applied when + /// the zone is in memory and watches the topic; a sleeping zone + /// hydrates on wake instead (a push never wakes it). + async fn take_push(&self, mut req: Request) -> Result { + if let Err(error) = super::cells::authenticate(&req, self.env.env()) { + return super::cells::error_response(error); + } + let Ok(update) = req.json::().await else { + return Response::error("a world update is required", 400); }; - let Some(world) = world else { + let awake = self.inner.borrow().host.is_some(); + if awake && self.world_topics().contains(&update.topic) { + self.apply_view(&update.topic, update.view); + } + Response::empty().map(|r| r.with_status(204)) + } + + /// Apply `view` of `topic` if it is newer than what the zone has. + fn apply_view(&self, topic: &str, view: Value) { + let version = view_version(&view); + if !self + .inner + .borrow_mut() + .world_versions + .accept(topic, version) + { return; - }; - match self.env.read_cell(CONTENT_ROLLOUT_KIND, &world).await { - Ok(value) => { - let state = value - .get("state") - .cloned() - .and_then(|state| serde_json::from_value::(state).ok()); - if let Some(state) = state { - self.apply_content(&state); - } + } + let (kind, shard) = topic.split_once('/').unwrap_or((topic, "")); + if kind == ZONE_DEV_KIND && shard == NETSIM_SHARD { + let sim = serde_json::from_value::(view) + .unwrap_or_default() + .to_sim(); + self.inner.borrow_mut().net_sim = sim; + return; + } + if kind == CONTENT_ROLLOUT_KIND && self.content_world().as_deref() == Some(shard) { + let state = view + .get("state") + .cloned() + .and_then(|state| serde_json::from_value::(state).ok()); + if let Some(state) = state { + self.apply_content(&state); } - Err(e) => worker::console_warn!("content rollout: {e}"), + return; + } + let watches = self.inner.borrow().watches.clone(); + let Some(message) = watches + .as_ref() + .and_then(|w| w.get(topic)) + .and_then(|watch| watch.message(view)) + else { + return; + }; + let mut inner = self.inner.borrow_mut(); + if let Some(host) = inner.host.as_mut() { + host.deliver(ZoneInbound::Message(message)); + inner.flush(); } } @@ -1320,24 +1459,6 @@ impl ZoneObject { inner.flush(); } - /// [`read_content`](Self::read_content) off the tick, at most every - /// [`DEV_SETTINGS_EVERY_MS`]. - fn refresh_content(&self) { - let due = { - let inner = self.inner.borrow(); - inner - .options - .as_ref() - .is_some_and(|o| o.content_rollout.is_some()) - && now_ms() - inner.last_content_read_ms >= DEV_SETTINGS_EVERY_MS - }; - if due { - self.inner.borrow_mut().last_content_read_ms = now_ms(); - let this = self.handle(); - wasm_bindgen_futures::spawn_local(async move { this.read_content().await }); - } - } - async fn hello(&self, ws: &WebSocket, hello: crate::protocol::Hello) -> Result<()> { if hello.protocol_version != PROTOCOL_VERSION { reject(ws, error::VERSION_MISMATCH, "protocol version"); @@ -1433,7 +1554,9 @@ impl ZoneObject { .is_some_and(|h| h.config().content_version != hello.content_version) }; if behind { - self.read_content().await; + if let Some(world) = self.content_world() { + self.reconcile(&topic(CONTENT_ROLLOUT_KIND, &world)).await; + } } let mut inner = self.inner.borrow_mut(); let Some(host) = inner.host.as_mut() else { @@ -1575,6 +1698,7 @@ impl ZoneObject { state: self.state.clone(), env: self.env.clone(), inner: self.inner.clone(), + binding: self.binding.clone(), } } @@ -1641,8 +1765,7 @@ impl ZoneObject { // Storage stays off the loop: the simulation keeps its pace // while a write waits on the bucket. self.spool_later(); - self.refresh_dev_settings(); - self.refresh_content(); + self.reconcile_world(false); self.refresh_if_due(); if stop { self.persist().await; @@ -1988,7 +2111,11 @@ macro_rules! celld_zone_object { impl ::worker::DurableObject for $name { fn new(state: ::worker::State, env: ::worker::Env) -> Self { Self { - zone: $crate::celld_zone::ZoneObject::new(state, env), + zone: $crate::celld_zone::ZoneObject::of_class( + state, + env, + ::core::stringify!($name), + ), } } diff --git a/distributed_game/src/world/mod.rs b/distributed_game/src/world/mod.rs index 7f1b6787b..4491de223 100644 --- a/distributed_game/src/world/mod.rs +++ b/distributed_game/src/world/mod.rs @@ -22,6 +22,10 @@ //! / hibernated / cold without opening a cell). //! - `content_rollout` (feature `cell`): a world's live content rollout //! (freeze → apply → resume across every celld-hosted zone) as a cell. +//! - [`watch`]: watched world views — a zone keeps pushed read models of +//! the world cells it cares about ([`watch::Watches`]); cells keep +//! in-memory watcher leases ([`watch::Watchers`]) and push each new +//! version; a slow reconcile read repairs lost pushes. //! - [`barrier`]: content-version policy types shared by the host and the //! world barrier ([`StalePolicy`], [`BarrierMode`]). @@ -34,6 +38,7 @@ pub mod directory; pub mod ownership; pub mod state; pub mod topology; +pub mod watch; #[cfg(feature = "cell")] #[allow(missing_docs)] pub mod zone_activity; diff --git a/distributed_game/src/world/watch.rs b/distributed_game/src/world/watch.rs new file mode 100644 index 000000000..cc77fdf46 --- /dev/null +++ b/distributed_game/src/world/watch.rs @@ -0,0 +1,469 @@ +//! Watched world views: zones keep pushed read models of the world cells +//! they care about instead of polling them. +//! +//! A world cell (a hood, the content rollout, …) is the write model. A zone +//! that cares about it **watches** it: the zone's read of the cell (`GET +//! /{kind}/{shard}?watch=&lease=`) returns the cell's view with +//! its `version` and leaves a lease, and after every commit that moves the +//! version the cell pushes [`WatchUpdate`] `{topic, version, view}` to the +//! zones whose lease is live. The zone keeps the highest version per topic +//! ([`TopicVersions`]), so a late push or read never rolls it back, and +//! re-reads each topic every [`RECONCILE_MS`] ± [`RECONCILE_JITTER_MS`] +//! (renewing its lease; repairing a lost push). Eventually consistent by +//! design: for display and world flavour, never for settling rewards. +//! +//! Cost: O(changes × watchers + zones / [`RECONCILE_MS`]) instead of every +//! zone reading every cell every second or two. +//! +//! Leases live in the cell's memory only ([`Watchers`]): a watch writes +//! nothing, and a cell that restarted relearns its watchers within one +//! reconcile. +//! +//! A zone declares what it watches with [`Watches`] (typed: each view +//! decodes into a plain serde struct and becomes one of the zone's +//! messages); the celld zone host does the reads, leases and pushes. + +use std::collections::BTreeMap; + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// How long a watch lasts without a renewing read (ms). +pub const WATCH_LEASE_MS: u64 = 120_000; +/// How often a zone re-reads each watched topic (ms), on average. +pub const RECONCILE_MS: u64 = 45_000; +/// Spread of [`RECONCILE_MS`] (± ms), so zones that woke together do not +/// read together. +pub const RECONCILE_JITTER_MS: u64 = 15_000; +/// Watchers pushed to at once by one cell. +pub const PUSH_CONCURRENCY: usize = 16; +/// Query parameter naming the watcher (`/`). +pub const WATCH_PARAM: &str = "watch"; +/// Query parameter with the lease (ms). +pub const LEASE_PARAM: &str = "lease"; +/// Path segment of the zone route that takes pushes (`POST …/__world`). +pub const PUSH_ROUTE: &str = "__world"; + +/// The topic of cell `kind/shard`. +pub fn topic(kind: &str, shard: &str) -> String { + format!("{kind}/{shard}") +} + +/// A watcher's address: the Durable Object namespace binding of its class +/// and its name (`PIT_ROOM/pit.0.0`). +pub fn watcher_address(binding: &str, name: &str) -> String { + format!("{binding}/{name}") +} + +/// `(binding, name)` of a [`watcher_address`]. +pub fn parse_watcher_address(address: &str) -> Option<(&str, &str)> { + address + .split_once('/') + .filter(|(binding, name)| !binding.is_empty() && !name.is_empty()) +} + +/// The class binding of a Durable Object class name, by convention +/// (`PitRoom` → `PIT_ROOM`). +pub fn class_binding(class: &str) -> String { + let mut binding = String::with_capacity(class.len() + 4); + for (i, c) in class.chars().enumerate() { + if c.is_uppercase() && i > 0 { + binding.push('_'); + } + binding.push(c.to_ascii_uppercase()); + } + binding +} + +/// What a cell pushes to its watchers (`POST …/__world`). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WatchUpdate { + /// [`topic`] of the cell. + pub topic: String, + /// The cell's version (its aggregate version). + pub version: u64, + /// The cell's view (what its read route returns). + pub view: Value, +} + +/// A cell's watchers: leases by [`watcher_address`] (memory only). +#[derive(Debug, Default)] +pub struct Watchers { + leases: BTreeMap, + published: Option, + last_view: Option, +} + +impl Watchers { + /// No watchers. + pub fn new() -> Self { + Self::default() + } + + /// `watcher` watches until `now_ms + lease_ms` (capped at + /// [`WATCH_LEASE_MS`]); a zero lease ends its watch. + pub fn watch(&mut self, watcher: &str, now_ms: u64, lease_ms: u64) { + if lease_ms == 0 { + self.leases.remove(watcher); + return; + } + let until = now_ms.saturating_add(lease_ms.min(WATCH_LEASE_MS)); + self.leases.insert(watcher.to_string(), until); + } + + /// The watchers with a live lease at `now_ms` (expired ones dropped). + pub fn live(&mut self, now_ms: u64) -> Vec { + self.leases.retain(|_, until| *until > now_ms); + self.leases.keys().cloned().collect() + } + + /// Whether `view` at `version` is news for the watchers: newer than + /// the last one published and different apart from its `version` + /// (a command that moves the aggregate but not its view, like a + /// heartbeat, pushes nothing). Records it when it is. + pub fn publish(&mut self, version: u64, view: &Value) -> bool { + if self.published.is_some_and(|last| version <= last) { + return false; + } + self.published = Some(version); + let content = match view { + Value::Object(fields) => { + let mut fields = fields.clone(); + fields.remove("version"); + Value::Object(fields).to_string() + } + other => other.to_string(), + }; + if self.last_view.as_ref() == Some(&content) { + return false; + } + self.last_view = Some(content); + true + } +} + +/// `(watcher, lease ms)` from a read's query string, if it watches. +pub fn watch_query(query: &str) -> Option<(String, u64)> { + let mut watcher = None; + let mut lease = WATCH_LEASE_MS; + for pair in query.split('&') { + let (key, value) = pair.split_once('=').unwrap_or((pair, "")); + match key { + WATCH_PARAM => watcher = Some(decode_component(value)), + LEASE_PARAM => lease = value.parse().unwrap_or(WATCH_LEASE_MS), + _ => {} + } + } + watcher + .filter(|w| parse_watcher_address(w).is_some()) + .map(|w| (w, lease)) +} + +/// The query string a watching read sends. +pub fn watch_query_string(watcher: &str, lease_ms: u64) -> String { + format!( + "{WATCH_PARAM}={}&{LEASE_PARAM}={lease_ms}", + encode_component(watcher) + ) +} + +/// The view's version: its `version` field (0 when the cell does not +/// version its view: every read then applies). +pub fn view_version(view: &Value) -> u64 { + view.get("version").and_then(Value::as_u64).unwrap_or(0) +} + +/// A zone's highest version seen per topic. +#[derive(Debug, Default, Clone)] +pub struct TopicVersions { + seen: BTreeMap, +} + +impl TopicVersions { + /// Nothing seen. + pub fn new() -> Self { + Self::default() + } + + /// Whether `version` of `topic` should be applied: newer than any + /// seen, or unversioned (0) views, which always apply. + pub fn accept(&mut self, topic: &str, version: u64) -> bool { + if version == 0 { + return true; + } + match self.seen.get(topic) { + Some(&seen) if version <= seen => false, + _ => { + self.seen.insert(topic.to_string(), version); + true + } + } + } + + /// Forget everything (a new activation re-hydrates). + pub fn clear(&mut self) { + self.seen.clear(); + } +} + +/// The next reconcile of a topic: [`RECONCILE_MS`] ± [`RECONCILE_JITTER_MS`] +/// after `now_ms`, `unit` in `[0, 1)` choosing where in the spread. +pub fn next_reconcile_ms(now_ms: u64, unit: f64) -> u64 { + let spread = (unit.clamp(0.0, 1.0) * 2.0 - 1.0) * RECONCILE_JITTER_MS as f64; + now_ms + (RECONCILE_MS as f64 + spread).max(1_000.0) as u64 +} + +type Decode = Box Option>; + +/// One watched cell: its topic and how its view becomes a message. +pub struct Watch { + /// Cell kind. + pub kind: String, + /// Cell shard. + pub shard: String, + decode: Decode, +} + +impl Watch { + /// [`topic`] of the cell. + pub fn topic(&self) -> String { + topic(&self.kind, &self.shard) + } + + /// The message for `view` (`None`: nothing to deliver, e.g. the cell + /// has no state yet or the view does not decode). + pub fn message(&self, view: Value) -> Option { + (self.decode)(view) + } +} + +/// What a zone watches, each cell's view decoding into a plain struct that +/// becomes a zone message: +/// +/// ``` +/// # use distributed_game::world::watch::Watches; +/// # #[derive(serde::Deserialize)] struct HoodRead { state: Option } +/// # enum Msg { Hood(u32) } +/// let watches: Watches = Watches::new() +/// .cell("hood", "0.0", |read: HoodRead| read.state.map(Msg::Hood)); +/// assert_eq!(watches.len(), 1); +/// ``` +pub struct Watches { + entries: Vec>, +} + +impl Default for Watches { + fn default() -> Self { + Self { + entries: Vec::new(), + } + } +} + +impl Watches { + /// Nothing watched. + pub fn new() -> Self { + Self::default() + } + + /// Watch cell `kind/shard`; its view decodes into `V` and `to` makes + /// the zone's message of it. + pub fn cell(mut self, kind: &str, shard: &str, to: F) -> Self + where + V: DeserializeOwned, + F: Fn(V) -> Option + 'static, + { + self.entries.push(Watch { + kind: kind.to_string(), + shard: shard.to_string(), + decode: Box::new(move |view| serde_json::from_value(view).ok().and_then(&to)), + }); + self + } + + /// Watched cells. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Nothing watched. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// The watched cells. + pub fn iter(&self) -> impl Iterator> { + self.entries.iter() + } + + /// The watch of `topic`. + pub fn get(&self, topic: &str) -> Option<&Watch> { + self.entries.iter().find(|w| w.topic() == topic) + } +} + +fn encode_component(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + encoded.push(byte as char) + } + _ => encoded.push_str(&format!("%{byte:02X}")), + } + } + encoded +} + +fn decode_component(value: &str) -> String { + let bytes = value.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'%' if i + 2 < bytes.len() => match u8::from_str_radix(&value[i + 1..i + 3], 16) { + Ok(byte) => { + out.push(byte); + i += 3; + } + Err(_) => { + out.push(b'%'); + i += 1; + } + }, + b'+' => { + out.push(b' '); + i += 1; + } + byte => { + out.push(byte); + i += 1; + } + } + } + String::from_utf8_lossy(&out).into_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn leases_expire_and_renew() { + let mut watchers = Watchers::new(); + watchers.watch("PIT_ROOM/pit.0.0", 1_000, 10_000); + watchers.watch("PIT_ROOM/pit.1.0", 1_000, 60_000); + assert_eq!(watchers.live(5_000).len(), 2); + assert_eq!(watchers.live(12_000), vec!["PIT_ROOM/pit.1.0".to_string()]); + watchers.watch("PIT_ROOM/pit.1.0", 12_000, 0); + assert!(watchers.live(12_000).is_empty()); + // Leases are capped. + watchers.watch("PIT_ROOM/pit.2.0", 0, u64::MAX); + assert!(watchers.live(WATCH_LEASE_MS).is_empty()); + } + + #[test] + fn a_version_is_published_once() { + let mut watchers = Watchers::new(); + let view = |n: u64| json!({ "version": n, "tier": n }); + assert!(watchers.publish(3, &view(3))); + assert!(!watchers.publish(3, &view(3))); + assert!(!watchers.publish(2, &view(2))); + assert!(watchers.publish(4, &view(4))); + } + + #[test] + fn an_unchanged_view_is_not_pushed() { + let mut watchers = Watchers::new(); + assert!(watchers.publish(1, &json!({ "version": 1, "tier": 2 }))); + // The aggregate moved (a heartbeat) but its view did not. + assert!(!watchers.publish(2, &json!({ "version": 2, "tier": 2 }))); + assert!(watchers.publish(3, &json!({ "version": 3, "tier": 3 }))); + } + + #[test] + fn zones_never_roll_back() { + let mut seen = TopicVersions::new(); + assert!(seen.accept("hood/0.0", 5)); + assert!(!seen.accept("hood/0.0", 5)); + assert!(!seen.accept("hood/0.0", 4), "a late push"); + assert!(seen.accept("hood/0.0", 6)); + assert!(seen.accept("borough/0.0", 1), "topics are independent"); + assert!(seen.accept("zone-dev/netsim", 0) && seen.accept("zone-dev/netsim", 0)); + seen.clear(); + assert!(seen.accept("hood/0.0", 2)); + } + + #[test] + fn watch_queries_round_trip() { + let query = watch_query_string("PIT_ROOM/pit.-1.2", 90_000); + assert_eq!(query, "watch=PIT_ROOM%2Fpit.-1.2&lease=90000"); + assert_eq!( + watch_query(&query), + Some(("PIT_ROOM/pit.-1.2".to_string(), 90_000)) + ); + assert_eq!( + watch_query("watch=PIT_ROOM%2Fpit.0.0"), + Some(("PIT_ROOM/pit.0.0".into(), WATCH_LEASE_MS)) + ); + assert_eq!( + watch_query("watch=A%2Fb%"), + Some(("A/b%".into(), WATCH_LEASE_MS)) + ); + assert_eq!(watch_query(""), None); + assert_eq!(watch_query("watch=nobinding"), None); + assert_eq!(watch_query("lease=5"), None); + } + + #[test] + fn class_bindings_follow_the_convention() { + assert_eq!(class_binding("PitRoom"), "PIT_ROOM"); + assert_eq!(class_binding("Room"), "ROOM"); + assert_eq!( + parse_watcher_address("PIT_ROOM/pit.0.0"), + Some(("PIT_ROOM", "pit.0.0")) + ); + assert_eq!(parse_watcher_address("/x"), None); + } + + #[test] + fn reconciles_spread_around_the_period() { + let low = next_reconcile_ms(0, 0.0); + let mid = next_reconcile_ms(0, 0.5); + let high = next_reconcile_ms(0, 0.999_999); + assert_eq!(low, RECONCILE_MS - RECONCILE_JITTER_MS); + assert_eq!(mid, RECONCILE_MS); + assert!(high <= RECONCILE_MS + RECONCILE_JITTER_MS && high > mid); + assert!(high < WATCH_LEASE_MS, "a zone renews before its lease ends"); + } + + #[test] + fn watches_decode_views_into_messages() { + #[derive(Deserialize)] + struct HoodRead { + state: Option, + } + #[derive(Deserialize, Debug, PartialEq)] + struct Hood { + tier: u32, + } + #[derive(Debug, PartialEq)] + enum Msg { + Hood(u32), + } + let watches: Watches = Watches::new().cell("hood", "1.2", |read: HoodRead| { + read.state.map(|h| Msg::Hood(h.tier)) + }); + let watch = watches.get("hood/1.2").expect("watched"); + assert_eq!( + watch.message(json!({"state": {"tier": 3}, "version": 7})), + Some(Msg::Hood(3)) + ); + assert_eq!(watch.message(json!({"state": null})), None); + assert_eq!(watch.message(json!("garbage")), None); + assert_eq!(view_version(&json!({"version": 7})), 7); + assert_eq!(view_version(&json!({})), 0); + } +} diff --git a/examples/game/cells/src/lib.rs b/examples/game/cells/src/lib.rs index 57e02ab67..3e93a96ad 100644 --- a/examples/game/cells/src/lib.rs +++ b/examples/game/cells/src/lib.rs @@ -17,6 +17,8 @@ //! boss room's zone fact raises its hood; the Queue consumer below runs //! the saga that ripples hood events into the borough and borough events //! into the city (idempotent commands, ids derived from the event ids). +//! Rooms watch them (`CellWatchers`): each change is pushed to the rooms +//! holding a watch lease, which re-read every ~45 s (no polling). //! //! - `content-rollout/world` (`ContentRolloutObject`): the world's live //! content (`distributed_game::world::content_rollout`). The services' @@ -58,7 +60,7 @@ use distributed::microsvc::{ use distributed::{sourced, Aggregate, Entity}; use distributed_game::cell::{fact_session_from_headers, session_stamp}; use distributed_game::celld_zone::{ - forward_to_zone, ContentRolloutCell, ZoneActivityCell, ZoneDevSettings, + forward_to_zone, CellWatchers, ContentRolloutCell, ZoneActivityCell, ZoneDevSettings, }; use distributed_game::lifecycle::{ Lifecycle, CONNECTION_LOST, PLAYER_DISCONNECTED, PLAYER_JOINED, PLAYER_LEFT, PLAYER_RECONNECTED, @@ -70,8 +72,8 @@ use mobpit_crawler::{ Crawler, MobKilledArgs, RecordKill, CRAWLER_KIND, MOB_KILLED_EVENT, PRESENCE_KIND, }; use mobpit_tiers::{ - mount_borough, mount_city, mount_hood, Borough, City, Hood, BOROUGH_KIND, CITY_KIND, HOOD_KIND, - SAGA_PRINCIPAL, + mount_borough, mount_city, mount_hood, Borough, BoroughRead, City, CityRead, Hood, HoodRead, + BOROUGH_KIND, CITY_KIND, HOOD_KIND, SAGA_PRINCIPAL, }; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -456,20 +458,17 @@ fn borough_session(header: &dyn Fn(&str) -> Option) -> Option { async fn read_hood(cell: &AggregateCell) -> std::result::Result { let hood = cell.load().await.map_err(|error| error.to_string())?; - Ok(json!({ "state": hood.map(|h| h.state()) })) + serde_json::to_value(HoodRead::of(hood.as_ref())).map_err(|error| error.to_string()) } async fn read_borough(cell: &AggregateCell) -> std::result::Result { let borough = cell.load().await.map_err(|error| error.to_string())?; - Ok(json!({ - "state": borough.as_ref().map(|b| b.state()), - "hoods_down": borough.map(|b| b.hoods_down()).unwrap_or_default(), - })) + serde_json::to_value(BoroughRead::of(borough.as_ref())).map_err(|error| error.to_string()) } async fn read_city(cell: &AggregateCell) -> std::result::Result { let city = cell.load().await.map_err(|error| error.to_string())?; - Ok(json!({ "state": city.map(|c| c.state()) })) + serde_json::to_value(CityRead::of(city.as_ref())).map_err(|error| error.to_string()) } /// A hood: its boss rooms' facts and its own respawn schedule. @@ -477,25 +476,34 @@ async fn read_city(cell: &AggregateCell) -> std::result::Result, env: Env, + /// Rooms watching it (pushed after every change). + watchers: CellWatchers, } impl DurableObject for HoodCell { fn new(state: State, env: Env) -> Self { console_error_panic_hook::set_once(); + let watchers = CellWatchers::new(HOOD_KIND, &state); let cell = mount_hood( AggregateCell::::from_state_with_snapshots(state, 50) .expect("hood cell identity"), ) .with_celld_outbox(outbox(&env)); - Self { cell, env } + Self { + cell, + env, + watchers, + } } async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_hood, fact_session).await + self.watchers + .serve(&self.env, &self.cell, req, read_hood, fact_session) + .await } async fn alarm(&self) -> Result { - alarm(&self.env, &self.cell).await + self.watchers.alarm(&self.env, &self.cell, read_hood).await } } @@ -504,25 +512,36 @@ impl DurableObject for HoodCell { pub struct BoroughCell { cell: AggregateCell, env: Env, + /// Rooms watching it (pushed after every change). + watchers: CellWatchers, } impl DurableObject for BoroughCell { fn new(state: State, env: Env) -> Self { console_error_panic_hook::set_once(); + let watchers = CellWatchers::new(BOROUGH_KIND, &state); let cell = mount_borough( AggregateCell::::from_state_with_snapshots(state, 50) .expect("borough cell identity"), ) .with_celld_outbox(outbox(&env)); - Self { cell, env } + Self { + cell, + env, + watchers, + } } async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_borough, borough_session).await + self.watchers + .serve(&self.env, &self.cell, req, read_borough, borough_session) + .await } async fn alarm(&self) -> Result { - alarm(&self.env, &self.cell).await + self.watchers + .alarm(&self.env, &self.cell, read_borough) + .await } } @@ -531,25 +550,34 @@ impl DurableObject for BoroughCell { pub struct CityCell { cell: AggregateCell, env: Env, + /// Rooms watching it (pushed after every change). + watchers: CellWatchers, } impl DurableObject for CityCell { fn new(state: State, env: Env) -> Self { console_error_panic_hook::set_once(); + let watchers = CellWatchers::new(CITY_KIND, &state); let cell = mount_city( AggregateCell::::from_state_with_snapshots(state, 50) .expect("city cell identity"), ) .with_celld_outbox(outbox(&env)); - Self { cell, env } + Self { + cell, + env, + watchers, + } } async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_city, saga_session).await + self.watchers + .serve(&self.env, &self.cell, req, read_city, saga_session) + .await } async fn alarm(&self) -> Result { - alarm(&self.env, &self.cell).await + self.watchers.alarm(&self.env, &self.cell, read_city).await } } diff --git a/examples/game/server/rooms/src/celld.rs b/examples/game/server/rooms/src/celld.rs index 580bc7b60..62adb5f5d 100644 --- a/examples/game/server/rooms/src/celld.rs +++ b/examples/game/server/rooms/src/celld.rs @@ -4,13 +4,17 @@ use distributed_game::celld_zone::{CelldZone, LocalFuture, ObjectEnv, ObjectFactSink}; use distributed_game::command_sink::FactRoute; +use distributed_game::world::watch::Watches; use distributed_game::HostConfig; use mobpit_crawler::{RecordKillInput, CRAWLER_KIND, PRESENCE_KIND}; -use crate::zone::{MobPit, PitMessage, RoomConfig, WorldTiers}; +use crate::zone::{MobPit, PitMessage, RoomConfig}; use crate::{DEFAULT_SEED, SEED_VAR}; use game_sim::mobpit::pair_shard; -use mobpit_tiers::{BossDefeatedInput, BOROUGH_KIND, CITY_KIND, CITY_SHARD, HOOD_KIND}; +use mobpit_tiers::{ + BoroughRead, BossDefeatedInput, CityRead, HoodRead, BOROUGH_KIND, CITY_KIND, CITY_SHARD, + HOOD_KIND, +}; /// Players a room admits (the game's cap). pub const MAX_PLAYERS: usize = 32; @@ -64,67 +68,38 @@ impl CelldZone for MobPit { .lifecycle_route(PRESENCE_KIND) } - /// The tiers above the room: its hood and borough every refresh (2 s), - /// the city every 5th. - fn refresh_every() -> Option { - Some(std::time::Duration::from_secs(2)) - } - - fn refresh( - env: &ObjectEnv, - zone_id: &str, - round: u64, - ) -> Option>> { - let room = game_sim::mobpit::parse_room(zone_id)?; + /// The tiers above the room: its hood, borough and the city, pushed by + /// their cells after every change (each view a plain `mobpit_tiers` + /// read struct; the room merges it into its world). + fn watch(zone_id: &str) -> Watches { + let Some(room) = game_sim::mobpit::parse_room(zone_id) else { + return Watches::new(); + }; let hood = game_sim::mobpit::hood_of(room); let borough = game_sim::mobpit::borough_of(hood); - let env = env.clone(); - Some(Box::pin(async move { - let hood_state = env.read_cell(HOOD_KIND, &pair_shard(hood)).await.ok()?; - let borough_state = env - .read_cell(BOROUGH_KIND, &pair_shard(borough)) - .await - .ok()?; - let city = if round % 5 == 0 { - env.read_cell(CITY_KIND, CITY_SHARD).await.ok() - } else { - None - }; - let h = &hood_state["state"]; - let b = &borough_state["state"]; - let mut world = WorldTiers { - hood_tier: h["tier"].as_u64().unwrap_or(0) as u32, - boss_down: h["boss_down"].as_bool().unwrap_or(false), - last_by: h["last_by"].as_str().unwrap_or_default().to_string(), - borough_bosses_down: b["bosses_down"].as_u64().unwrap_or(0) as u32, - weather: b["weather"].as_u64().unwrap_or(0) as u8, - event: b["event"].as_str().unwrap_or_default().to_string(), - event_until_ms: b["event_until_ms"].as_u64().unwrap_or(0), - hoods_down: borough_state["hoods_down"] - .as_array() - .map(|a| { - a.iter() - .filter_map(|v| v.as_str().map(String::from)) - .collect() - }) - .unwrap_or_default(), - ..WorldTiers::default() - }; - match city { - Some(c) => { - world.city_bosses_down = c["state"]["bosses_down"].as_u64().unwrap_or(0) as u32; - world.banner = c["state"]["banner"] - .as_str() - .unwrap_or("The bosses hold the city") - .to_string(); - } - None => { - // Not read this round: keep what the room knows. - return Some(PitMessage::WorldKeepCity(world)); - } - } - Some(PitMessage::World(world)) - })) + Watches::new() + .cell(HOOD_KIND, &pair_shard(hood), |read: HoodRead| { + read.state.map(|h| PitMessage::Hood { + tier: h.tier, + boss_down: h.boss_down, + last_by: h.last_by, + }) + }) + .cell(BOROUGH_KIND, &pair_shard(borough), |read: BoroughRead| { + read.state.map(|b| PitMessage::Borough { + bosses_down: b.bosses_down, + weather: b.weather, + event: b.event, + event_until_ms: b.event_until_ms, + hoods_down: read.hoods_down, + }) + }) + .cell(CITY_KIND, CITY_SHARD, |read: CityRead| { + read.state.map(|c| PitMessage::City { + bosses_down: c.bosses_down, + banner: c.banner, + }) + }) } /// Read the player's Crawler cell (off the tick) for its level and XP. diff --git a/examples/game/server/rooms/src/zone.rs b/examples/game/server/rooms/src/zone.rs index acd0a2863..2302110c7 100644 --- a/examples/game/server/rooms/src/zone.rs +++ b/examples/game/server/rooms/src/zone.rs @@ -441,6 +441,38 @@ pub enum PitMessage { /// The same without the city (not read this round: kept as it was). #[game_event(id = 3)] WorldKeepCity(WorldTiers), + /// The room's hood, pushed by its cell (or re-read). + #[game_event(id = 4)] + Hood { + /// Times its boss fell. + tier: u32, + /// Its boss is down. + boss_down: bool, + /// Who felled it last. + last_by: String, + }, + /// The room's borough, pushed by its cell (or re-read). + #[game_event(id = 5)] + Borough { + /// Bosses down in the borough. + bosses_down: u32, + /// Weather (index into `WEATHER`). + weather: u8, + /// World event (empty when none). + event: String, + /// When it ends (Unix ms). + event_until_ms: u64, + /// Hoods whose boss is down (`hx.hy`). + hoods_down: Vec, + }, + /// The city, pushed by its cell (or re-read). + #[game_event(id = 6)] + City { + /// Bosses down across the city. + bosses_down: u32, + /// The city's banner. + banner: String, + }, } /// Mob Pit's client surface (also used by the leaderboard page). @@ -1854,6 +1886,42 @@ impl Zone for MobPit { } fn on_message(&mut self, ctx: &mut Ctx<'_, Self>, message: PitMessage) { + // One tier pushed: the room's world with that tier replaced. + let message = match message { + PitMessage::Hood { + tier, + boss_down, + last_by, + } => PitMessage::World(WorldTiers { + hood_tier: tier, + boss_down, + last_by, + ..self.world.clone() + }), + PitMessage::Borough { + bosses_down, + weather, + event, + event_until_ms, + hoods_down, + } => PitMessage::World(WorldTiers { + borough_bosses_down: bosses_down, + weather, + event, + event_until_ms, + hoods_down, + ..self.world.clone() + }), + PitMessage::City { + bosses_down, + banner, + } => PitMessage::World(WorldTiers { + city_bosses_down: bosses_down, + banner, + ..self.world.clone() + }), + message => message, + }; match message { PitMessage::Loaded { principal, @@ -1869,11 +1937,13 @@ impl Zone for MobPit { } ctx.send(id, &Self::profile_event(xp, level.max(1), kills, "cell")); } + // Mapped to `World` above. + PitMessage::Hood { .. } | PitMessage::Borough { .. } | PitMessage::City { .. } => {} message @ (PitMessage::World(_) | PitMessage::WorldKeepCity(_)) => { let (world, keep_city) = match message { PitMessage::World(world) => (world, false), PitMessage::WorldKeepCity(world) => (world, true), - PitMessage::Loaded { .. } => unreachable!(), + _ => unreachable!(), }; // Only the hood, borough and city this room belongs to. // A fall reported here may not have reached the hood yet: diff --git a/examples/game/server/tiers/src/lib.rs b/examples/game/server/tiers/src/lib.rs index 33b10d796..55cbe074a 100644 --- a/examples/game/server/tiers/src/lib.rs +++ b/examples/game/server/tiers/src/lib.rs @@ -923,6 +923,60 @@ pub fn saga_step(name: &str, payload: &[u8]) -> Option { }) } +// ------------------------------------------------------------ read views + +/// What a hood cell's read route returns (rooms watch it). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct HoodRead { + /// The hood (`None` until its first event). + pub state: Option, +} + +/// What a borough cell's read route returns (rooms watch it). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct BoroughRead { + /// The borough (`None` until its first event). + pub state: Option, + /// Its hoods whose boss is down (`hx.hy`). + #[serde(default)] + pub hoods_down: Vec, +} + +/// What the city cell's read route returns (rooms watch it). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct CityRead { + /// The city (`None` until its first event). + pub state: Option, +} + +impl HoodRead { + /// The read view of `hood`. + pub fn of(hood: Option<&Hood>) -> Self { + Self { + state: hood.map(Hood::state), + } + } +} + +impl BoroughRead { + /// The read view of `borough`. + pub fn of(borough: Option<&Borough>) -> Self { + Self { + state: borough.map(Borough::state), + hoods_down: borough.map(Borough::hoods_down).unwrap_or_default(), + } + } +} + +impl CityRead { + /// The read view of `city`. + pub fn of(city: Option<&City>) -> Self { + Self { + state: city.map(City::state), + } + } +} + #[cfg(test)] mod tests { use super::*; From f903581ec54acf319402f13c5c3a7bc74d7ce37b Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 00:47:18 -0500 Subject: [PATCH 02/22] feat(game): settle zone facts in batches per lane and cell The fact feed hands a lane's waiting facts to the sink at once (FactSink::commit_batch, max_batch 16, ZONE_FACT_BATCH on celld). CommandFactSink groups consecutive facts for one aggregate; the celld object host sends them as one POST {kind}/{shard}/__batch, which serve_cell runs entry by entry through the idempotent wait path (own command id, principal and stamp) before one outbox drain. The zone deletes settled facts from its outbox in one batched delete. A batch stops at its first retryable failure and is retried from there, attempts counting per fact, so order and exactly-once are unchanged. Game code is unchanged. [[zone-fact-batches]] [[world-scale-game-services]] --- distributed_game/README.md | 29 +- distributed_game/src/celld_zone/cells.rs | 105 ++++++- distributed_game/src/celld_zone/object.rs | 173 ++++++++++- distributed_game/src/celld_zone/plan.rs | 11 + distributed_game/src/command_sink.rs | 202 ++++++++++++- distributed_game/src/facts.rs | 335 ++++++++++++++++++++-- 6 files changed, 794 insertions(+), 61 deletions(-) diff --git a/distributed_game/README.md b/distributed_game/README.md index 5b2c6ed7a..00980704c 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -826,7 +826,9 @@ hibernation (`ZONE_HIBERNATE_AFTER_MS` later, by object alarm, so it also happens after an eviction), restore on the next join, facts to the cells of the same worker object to object (`CelldObjectHost` under the usual `CommandFactSink` routes; a durable outbox in the object's storage makes -them exactly once across crashes), and the zone's lifecycle reports +them exactly once across crashes; a lane's waiting facts for one cell go +as one `POST …/__batch` and leave the outbox in one delete, see +[Durable facts](#durable-facts)), and the zone's lifecycle reports (`active` / `dormant` / `hibernated`) into its [`zone_activity`](src/world/zone_activity.rs) cell, which the `zone_status` read model projects for minimaps. @@ -972,10 +974,13 @@ unlimited rewind. - **Facts, not per-tick persistence.** Movement and combat ticks are never stored, only meaningful outcomes. The tick enqueues and moves on, so storage speed never shows up as lag. -- **Idempotency through keys, not a zone-side outbox.** A fact key plus - the cell's command ledger make retries safe. An outbox on the zone - would add a second write per outcome for little gain, since the zone - state lost in a crash is lost either way. +- **Idempotency through keys; the outbox is a log consumed in batches.** + A fact key plus the cell's command ledger make retries safe. A native + zone keeps facts in its bounded queue only (the zone state lost in a + crash is lost either way). A celld zone also journals them in object + storage, written with the tick's other writes and settled in batches + (one cell request and one delete per batch), so the outbox costs one + batched write per tick, not two writes and a request per outcome. - **Static world from a seed.** Terrain and props are a pure function of seed and content version, generated identically on server and client. Only things that change are replicated. @@ -1138,6 +1143,20 @@ a time (a slow commit delays every later fact). Set without a principal) stay sequential in emission order; different principals commit concurrently; settlements arrive in completion order. +**Batches.** The feed hands its sink up to `max_batch` (default 16, +`FactFeed::with_max_batch`; celld zones: `ZONE_FACT_BATCH`) waiting facts +of a lane at once, `FactSink::commit_batch`. Game code is unchanged +(`ctx.emit`, `#[derive(FactCommand)]`, `#[command]` handlers). The default +commits them one by one; `CommandFactSink` groups consecutive facts for +the same aggregate and calls `FactHost::dispatch_fact_batch`, which the +celld object host sends as one `POST {kind}/{shard}/__batch` (each entry +with its own command id, principal and stamp; `serve_cell` runs them in +order through the same idempotent wait path and drains the cell outbox +once). A batch stops at its first retryable failure and is retried from +there; attempts count per fact, so order, exactly-once and +`RetryPolicy` mean what they mean for single facts. A cell without the +batch route (404) gets the facts one by one. + ## Netcode and operations - **Admission tickets** (`net`): by default join tickets are HS256 JWTs diff --git a/distributed_game/src/celld_zone/cells.rs b/distributed_game/src/celld_zone/cells.rs index 33943e65d..1da7663ec 100644 --- a/distributed_game/src/celld_zone/cells.rs +++ b/distributed_game/src/celld_zone/cells.rs @@ -63,17 +63,28 @@ pub fn json_status(body: Value, status: u16) -> Result { /// The wire form of a dispatch error. pub fn error_response(error: CellDispatchError) -> Result { - json_status( - json!({ "error": error.client_message(), "code": error.code() }), - error.status_code(), - ) + let status = error.status_code(); + json_status(error_body(&error), status) } +fn error_body(error: &CellDispatchError) -> Value { + json!({ "error": error.client_message(), "code": error.code() }) +} + +/// The route of a fact batch: `POST /{kind}/{shard}/__batch` (see +/// [`serve_cell`]). +pub const BATCH_ROUTE: &str = "__batch"; + /// One cell's internal routes: `GET /{kind}/{shard}` → `read(cell)`, `POST /// /{kind}/{shard}/{command}` → the idempotent wait-path dispatch (identity /// from the service-id and principal-partition headers, the session from -/// `sessions`), then a best-effort outbox drain. Every request must carry -/// the internal secret. +/// `sessions`), then a best-effort outbox drain. `POST +/// /{kind}/{shard}/__batch` ([`BATCH_ROUTE`]) runs a zone's consecutive +/// facts for this cell in one request: `{ entries: [{ command, commandId, +/// input, partition, session }] }` → `{ results: [{ status, body }] }`, +/// each entry exactly as its own wait-path command (its own ledger id and +/// session), in order, stopping after the first retryable result, then one +/// outbox drain. Every request must carry the internal secret. pub async fn serve_cell( env: &Env, cell: &AggregateCell, @@ -100,6 +111,9 @@ where Ok(body) => json_status(body, 200), Err(error) => error_response(CellDispatchError::Internal(error)), }, + (Method::Post, Some(route)) if route == BATCH_ROUTE => { + serve_batch(env, cell, req, sessions).await + } (Method::Post, Some(command)) => { let command = command.clone(); let body: Value = match req.json().await { @@ -143,6 +157,85 @@ where } } +async fn serve_batch( + env: &Env, + cell: &AggregateCell, + mut req: Request, + sessions: Sessions, +) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + let Some(service) = req.headers().get(CELL_SERVICE_ID_HEADER).ok().flatten() else { + return error_response(CellDispatchError::Unauthorized); + }; + let body: Value = match req.json().await { + Ok(body) => body, + Err(_) => return error_response(CellDispatchError::BadRequest("invalid JSON".into())), + }; + let Some(entries) = body.get("entries").and_then(Value::as_array) else { + return error_response(CellDispatchError::BadRequest( + "`entries` is required".into(), + )); + }; + let mut results = Vec::with_capacity(entries.len()); + for entry in entries { + let (status, body) = match batch_entry(cell, &service, entry, sessions).await { + Ok(body) => (200, body), + Err(error) => (error.status_code(), error_body(&error)), + }; + let retryable = status >= 500 + || matches!(status, 408 | 429) + || body.get("code").and_then(Value::as_str) == Some("COMMAND_IN_PROGRESS"); + results.push(json!({ "status": status, "body": body })); + if retryable { + break; + } + } + if let Err(error) = cell.drain_outbox(env).await { + worker::console_error!("post-commit drain deferred: {error}"); + } + json_status(json!({ "results": results }), 200) +} + +/// One `__batch` entry as its own wait-path command. +async fn batch_entry( + cell: &AggregateCell, + service: &str, + entry: &Value, + sessions: Sessions, +) -> std::result::Result +where + A: Aggregate + Send + Sync + 'static, +{ + let field = |name: &str| entry.get(name).and_then(Value::as_str); + let (Some(command), Some(partition)) = (field("command"), field("partition")) else { + return Err(CellDispatchError::BadRequest( + "`command` and `partition` are required".into(), + )); + }; + let parsed = CellWaitPathRequest::parse(json!({ + "commandId": entry.get("commandId"), + "input": entry.get("input"), + })) + .map_err(CellDispatchError::BadRequest)?; + let identity = CellCommandIdentity::new( + service.to_string(), + partition.to_string(), + &parsed.command_id, + )?; + let vars = entry.get("session").and_then(Value::as_object); + let header = |name: &str| { + vars.and_then(|vars| vars.get(name)) + .and_then(Value::as_str) + .map(str::to_string) + }; + let session = sessions(&header).ok_or(CellDispatchError::Forbidden)?; + cell.dispatch_idempotent(command, &identity, parsed.input, session) + .await + .map(|dispatch| dispatch.wait_path_wire()) +} + /// A watched world cell's watchers ([`crate::world::watch`]): zones that /// read the cell with `?watch=` get its view pushed after every commit (and /// every alarm) that moves its version. Memory only. diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index 38ac23da3..a60eef4e6 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -268,7 +268,6 @@ impl CelldObjectHost { } async fn post(&self, call: &FactCall<'_>) -> std::result::Result { - let retry = |e: worker::Error| FactError::Retryable(e.to_string()); let headers = Headers::new(); let set = |name: &str, value: &str| { headers @@ -285,24 +284,124 @@ impl CelldObjectHost { set(name, value)?; } } - let body = json!({ "commandId": call.command_id, "input": call.input }).to_string(); + let body = json!({ "commandId": call.command_id, "input": call.input }); + let (status, body) = self + .send(call.kind, &call.shard, call.command, headers, body) + .await?; + classify_wait_path(status, &body) + } + + /// `calls` (one aggregate) in one `POST /{kind}/{shard}/__batch` + /// ([`super::cells::serve_cell`]); each entry carries its own command + /// id, principal and stamp. A cell without the route gets them one by + /// one. + async fn post_batch( + &self, + calls: Vec>, + ) -> Vec> { + let Some(first) = calls.first() else { + return Vec::new(); + }; + let (kind, shard) = (first.kind, first.shard.clone()); + let entries: Vec = calls + .iter() + .map(|call| { + let mut session = serde_json::Map::new(); + session.insert(USER_ID_KEY.into(), json!(call.principal)); + for (name, value) in call.session.variables() { + if name.starts_with(crate::stamp::SESSION_PREFIX) { + session.insert(name.clone(), json!(value)); + } + } + json!({ + "command": call.command, + "commandId": call.command_id, + "input": call.input, + "partition": call.principal, + "session": session, + }) + }) + .collect(); + let headers = Headers::new(); + let set = |name: &str, value: &str| headers.set(name, value).is_ok(); + if !(set(CELL_INTERNAL_SECRET_HEADER, &self.secret) + && set(CELL_SERVICE_ID_HEADER, &self.service_id) + && set("content-type", "application/json")) + { + return vec![Err(FactError::Permanent("batch headers".into()))]; + } + let sent = self + .send( + kind, + &shard, + super::cells::BATCH_ROUTE, + headers, + json!({ "entries": entries }), + ) + .await; + match sent { + Ok((404, _)) => { + let mut results = Vec::with_capacity(calls.len()); + for call in &calls { + let result = self.post(call).await; + let stop = matches!(result, Err(FactError::Retryable(_))); + results.push(result); + if stop { + break; + } + } + results + } + Ok((status, body)) if (200..300).contains(&status) => body + .get("results") + .and_then(Value::as_array) + .map(|results| { + results + .iter() + .map(|result| { + let status = result + .get("status") + .and_then(Value::as_u64) + .and_then(|status| u16::try_from(status).ok()) + .unwrap_or(500); + classify_wait_path(status, result.get("body").unwrap_or(&Value::Null)) + }) + .collect() + }) + .unwrap_or_default(), + Ok((status, body)) => vec![classify_wait_path(status, &body)], + Err(error) => vec![Err(error)], + } + } + + /// `POST https://cell/{kind}/{shard}/{route}` to the cell's stub: + /// `(status, JSON body)`. + async fn send( + &self, + kind: &str, + shard: &str, + route: &str, + headers: Headers, + body: Value, + ) -> std::result::Result<(u16, Value), FactError> { + let retry = |e: worker::Error| FactError::Retryable(e.to_string()); let mut init = RequestInit::new(); init.with_method(Method::Post) .with_headers(headers) - .with_body(Some(body.into())); + .with_body(Some(body.to_string().into())); let url = format!( "https://cell/{}/{}/{}", - encode_segment(call.kind), - encode_segment(&call.shard), - encode_segment(call.command) + encode_segment(kind), + encode_segment(shard), + encode_segment(route) ); let request = Request::new_with_init(&url, &init).map_err(retry)?; - let stub = stub_for(&self.env, call.kind, &call.shard) - .map_err(|e| FactError::Permanent(format!("cell `{}`: {e}", call.kind)))?; + let stub = stub_for(&self.env, kind, shard) + .map_err(|e| FactError::Permanent(format!("cell `{kind}`: {e}")))?; let mut response = stub.fetch_with_request(request).await.map_err(retry)?; let status = response.status_code(); let body: Value = response.json().await.unwrap_or(Value::Null); - classify_wait_path(status, &body) + Ok((status, body)) } } @@ -313,6 +412,19 @@ impl FactHost for CelldObjectHost { ) -> BoxFuture<'a, std::result::Result> { Box::pin(SendFuture::new(async move { self.post(&call).await })) } + + fn dispatch_fact_batch<'a>( + &'a self, + calls: Vec>, + ) -> BoxFuture<'a, Vec>> { + Box::pin(SendFuture::new(async move { + if calls.len() == 1 { + let call = calls.into_iter().next().expect("one call"); + return vec![self.post(&call).await]; + } + self.post_batch(calls).await + })) + } } /// The durable fact outbox around the sink: every emitted fact is written @@ -352,6 +464,43 @@ impl FactSink for DurableFacts { result })) } + + /// One batched delete for the facts the batch settled. + fn commit_batch<'a>( + &'a self, + facts: &'a [Fact], + ) -> BoxFuture<'a, Vec>> { + Box::pin(SendFuture::new(async move { + let results = self.sink.commit_batch(facts).await; + let mut settled = Vec::new(); + for (fact, result) in facts.iter().zip(&results) { + match result { + Err(FactError::Permanent(e)) => { + worker::console_error!("fact {} refused: {e}", fact.key()) + } + Err(FactError::Retryable(e)) => { + worker::console_warn!("fact {} will be retried: {e}", fact.key()); + continue; + } + Ok(_) => {} + } + settled.push(StoredFact::storage_key(fact.key())); + } + if !settled.is_empty() { + let _ = self.storage.delete_multiple(settled).await; + } + results + })) + } +} + +/// Facts per cell batch from the zone's options. +fn fact_batch(options: &Option) -> usize { + options + .as_ref() + .map_or(crate::facts::DEFAULT_MAX_BATCH, |options| { + options.fact_batch + }) } // ------------------------------------------------------------------ time @@ -1094,8 +1243,10 @@ impl ZoneObject { }; let (queue, feed) = fact_channel(FACT_CAPACITY, retry_policy()); let queue = queue.with_journal(); + let batch = fact_batch(&self.inner.borrow().options); wasm_bindgen_futures::spawn_local( feed.with_max_concurrent_commits(4) + .with_max_batch(batch) .run(durable.clone(), sleep), ); { @@ -1194,7 +1345,9 @@ impl ZoneObject { let (queue, feed) = fact_channel(FACT_CAPACITY, retry_policy()); if let Some(sink) = inner.sink.clone() { wasm_bindgen_futures::spawn_local( - feed.with_max_concurrent_commits(4).run(sink, sleep), + feed.with_max_concurrent_commits(4) + .with_max_batch(fact_batch(&inner.options)) + .run(sink, sleep), ); host = host.with_facts(queue.with_journal()); } diff --git a/distributed_game/src/celld_zone/plan.rs b/distributed_game/src/celld_zone/plan.rs index f753beff7..7f4d9991f 100644 --- a/distributed_game/src/celld_zone/plan.rs +++ b/distributed_game/src/celld_zone/plan.rs @@ -37,6 +37,10 @@ pub const RECONNECT_GRACE_MS_VAR: &str = "ZONE_RECONNECT_GRACE_MS"; /// Env var: command-ledger service id of the zone's facts (default /// `celld-zone`). pub const FACT_SERVICE_VAR: &str = "ZONE_FACT_SERVICE"; +/// Env var: facts of a lane settled in one cell request at most (default +/// [`DEFAULT_MAX_BATCH`](crate::facts::DEFAULT_MAX_BATCH), 1..=256; 1 +/// sends each fact alone). +pub const FACT_BATCH_VAR: &str = "ZONE_FACT_BATCH"; /// Env var (debug only, off by default): `name=host:port,…` celld operator /// listeners asked which node holds the room (celld does not tell /// application code its node id). @@ -82,6 +86,8 @@ pub struct ObjectOptions { pub reconnect_grace: Duration, /// Fact command-ledger service id. pub fact_service: String, + /// Facts per cell batch at most ([`FACT_BATCH_VAR`]). + pub fact_batch: usize, /// Debug node lookup: `(name, operator host:port)`. pub nodes: Vec<(String, String)>, /// Debug fixed node name. @@ -118,6 +124,10 @@ impl ObjectOptions { return Err(format!("{TICK_HZ_VAR} must be within 1..=120")); } let patch_hz: u32 = parse(&get, PATCH_HZ_VAR, 20)?; + let fact_batch: usize = parse(&get, FACT_BATCH_VAR, crate::facts::DEFAULT_MAX_BATCH)?; + if !(1..=256).contains(&fact_batch) { + return Err(format!("{FACT_BATCH_VAR} must be within 1..=256")); + } let nodes = get(NODES_VAR) .unwrap_or_default() .split(',') @@ -139,6 +149,7 @@ impl ObjectOptions { fact_service: get(FACT_SERVICE_VAR) .filter(|service| !service.trim().is_empty()) .unwrap_or_else(|| "celld-zone".into()), + fact_batch, nodes, node: get(NODE_VAR).filter(|node| !node.is_empty()), dev_settings: get(DEV_SETTINGS_VAR) diff --git a/distributed_game/src/command_sink.rs b/distributed_game/src/command_sink.rs index b5acc84b1..759353ffb 100644 --- a/distributed_game/src/command_sink.rs +++ b/distributed_game/src/command_sink.rs @@ -152,6 +152,28 @@ pub trait FactHost: Send + Sync + 'static { &'a self, call: FactCall<'a>, ) -> BoxFuture<'a, Result>; + + /// Run `calls` (consecutive facts for one aggregate: same kind and + /// shard) in order, under [`FactSink::commit_batch`]'s contract: the + /// results end at the first retryable failure. The default runs them + /// one by one; a host with a batch route sends them in one request. + fn dispatch_fact_batch<'a>( + &'a self, + calls: Vec>, + ) -> BoxFuture<'a, Vec>> { + Box::pin(async move { + let mut results = Vec::with_capacity(calls.len()); + for call in calls { + let result = self.dispatch_fact(call).await; + let stop = matches!(result, Err(FactError::Retryable(_))); + results.push(result); + if stop { + break; + } + } + results + }) + } } /// See the module docs. @@ -278,21 +300,72 @@ impl CommandFactSink { } } +impl CommandFactSink { + fn call_for<'a>(&'a self, fact: &'a Fact) -> Result, FactError> { + let (route, shard, principal, session, command_id) = self.resolve(fact)?; + Ok(FactCall { + command: fact.command(), + kind: route.kind(), + shard, + command_id, + input: fact.input(), + principal, + session, + }) + } +} + impl FactSink for CommandFactSink { fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { self.host.dispatch_fact(self.call_for(fact)?).await }) + } + + /// Consecutive facts for the same aggregate go to the host together + /// ([`FactHost::dispatch_fact_batch`]). + fn commit_batch<'a>( + &'a self, + facts: &'a [Fact], + ) -> BoxFuture<'a, Vec>> { Box::pin(async move { - let (route, shard, principal, session, command_id) = self.resolve(fact)?; - self.host - .dispatch_fact(FactCall { - command: fact.command(), - kind: route.kind(), - shard, - command_id, - input: fact.input(), - principal, - session, - }) - .await + let mut results = Vec::with_capacity(facts.len()); + let mut i = 0; + while i < facts.len() { + let call = match self.call_for(&facts[i]) { + Ok(call) => call, + Err(error) => { + results.push(Err(error)); + i += 1; + continue; + } + }; + let mut run = vec![call]; + let mut next = i + 1; + while let Some(fact) = facts.get(next) { + match self.call_for(fact) { + Ok(call) if call.kind == run[0].kind && call.shard == run[0].shard => { + run.push(call); + next += 1; + } + _ => break, + } + } + let n = run.len(); + let batch: Vec<_> = self + .host + .dispatch_fact_batch(run) + .await + .into_iter() + .take(n) + .collect(); + let complete = + batch.len() == n && !matches!(batch.last(), Some(Err(FactError::Retryable(_)))); + results.extend(batch); + if !complete { + break; + } + i = next; + } + results }) } } @@ -572,3 +645,108 @@ mod tests { assert!(CelldFactSink::celld("http://x", secret(), "s", " ").is_err()); } } + +#[cfg(test)] +mod batch_tests { + use std::sync::Mutex; + + use serde_json::json; + + use super::*; + + /// Records each batch as `(shard, commands)`; `busy` answers retryable. + #[derive(Default)] + struct RecordingHost { + batches: Mutex>, + busy: Option, + } + + impl FactHost for RecordingHost { + fn dispatch_fact<'a>( + &'a self, + _: FactCall<'a>, + ) -> BoxFuture<'a, Result> { + unreachable!("batches only") + } + + fn dispatch_fact_batch<'a>( + &'a self, + calls: Vec>, + ) -> BoxFuture<'a, Vec>> { + Box::pin(async move { + let shard = calls[0].shard.clone(); + self.batches.lock().unwrap().push((shard, calls.len())); + let mut results = Vec::new(); + for call in &calls { + if self.busy.as_deref() == Some(call.input["n"].as_str().unwrap()) { + results.push(Err(FactError::Retryable("busy".into()))); + break; + } + results.push(Ok(FactReceipt { + payload: Value::Null, + replayed: false, + })); + } + results + }) + } + } + + fn sink(host: RecordingHost) -> CommandFactSink { + CommandFactSink::new(host, "zone-test", "zone:z1") + .unwrap() + .route(FactRoute::new("credit", "room", |input, _| { + input.get("room")?.as_str().map(str::to_string) + })) + } + + fn facts(rooms: &[&str]) -> Vec { + rooms + .iter() + .enumerate() + .map(|(i, room)| { + let command = if room.is_empty() { + "unrouted" + } else { + "credit" + }; + Fact::new( + format!("k{i}"), + command, + json!({ "room": room, "n": format!("n{i}") }), + ) + }) + .collect() + } + + #[tokio::test] + async fn consecutive_facts_for_one_aggregate_travel_together() { + let sink = sink(RecordingHost::default()); + let facts = facts(&["r1", "r1", "", "r2", "r2", "r2", "r1"]); + let results = sink.commit_batch(&facts).await; + assert_eq!(results.len(), 7); + assert!(matches!(results[2], Err(FactError::Permanent(_)))); + assert!(results.iter().enumerate().all(|(i, r)| i == 2 || r.is_ok())); + assert_eq!( + *sink.host.batches.lock().unwrap(), + [("r1".into(), 2), ("r2".into(), 3), ("r1".into(), 1)] + ); + } + + #[tokio::test] + async fn a_retryable_fact_ends_the_batch() { + let sink = sink(RecordingHost { + busy: Some("n3".into()), + ..RecordingHost::default() + }); + let facts = facts(&["r1", "r1", "r2", "r2", "r2", "r3"]); + let results = sink.commit_batch(&facts).await; + // r1 ×2, then r2 stops at its second fact; r3 is not attempted. + assert_eq!(results.len(), 4); + assert!(matches!(results[3], Err(FactError::Retryable(_)))); + assert_eq!( + *sink.host.batches.lock().unwrap(), + [("r1".into(), 2), ("r2".into(), 3)] + ); + } +} diff --git a/distributed_game/src/facts.rs b/distributed_game/src/facts.rs index 66e784662..40eb74aa8 100644 --- a/distributed_game/src/facts.rs +++ b/distributed_game/src/facts.rs @@ -28,6 +28,15 @@ //! different principals overlap. A key is never in flight twice (the queue //! refuses duplicates), so per-key order always holds. Settlements arrive //! in completion order. +//! +//! Batching: the feed hands a sink up to +//! [`max_batch`](FactFeed::with_max_batch) (default +//! [`DEFAULT_MAX_BATCH`]) consecutive facts of one lane at once +//! ([`FactSink::commit_batch`]). A sink that can settle many facts in +//! one round trip (the celld cell hosts: one `__batch` request per cell) +//! overrides it; the default commits them one by one. A batch commits in +//! order and stops at its first retryable failure, which is retried from +//! there, so order and exactly-once hold as for single facts. use std::collections::{HashSet, VecDeque}; use std::future::Future; @@ -236,8 +245,33 @@ impl FactSettlement { pub trait FactSink: Send + Sync + 'static { /// Commit one fact. Must be idempotent per [`Fact::key`]. fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result>; + + /// Commit `facts` in order: one result per fact attempted. The + /// results end at the first [`FactError::Retryable`] (included), or + /// earlier; facts past the end were not attempted and are retried. + /// The default commits them one by one. + fn commit_batch<'a>( + &'a self, + facts: &'a [Fact], + ) -> BoxFuture<'a, Vec>> { + Box::pin(async move { + let mut results = Vec::with_capacity(facts.len()); + for fact in facts { + let result = self.commit(fact).await; + let stop = matches!(result, Err(FactError::Retryable(_))); + results.push(result); + if stop { + break; + } + } + results + }) + } } +/// Facts per [`FactSink::commit_batch`] unless the feed says otherwise. +pub const DEFAULT_MAX_BATCH: usize = 16; + /// Retry/backoff policy. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct RetryPolicy { @@ -330,6 +364,7 @@ pub struct FactFeed { settlements: mpsc::Sender, policy: RetryPolicy, max_concurrent: usize, + max_batch: usize, } /// Create a queue allowing `capacity` unsettled facts. @@ -350,6 +385,7 @@ pub fn fact_channel(capacity: usize, policy: RetryPolicy) -> (FactQueue, FactFee settlements: tx, policy, max_concurrent: 1, + max_batch: DEFAULT_MAX_BATCH, }, ) } @@ -470,18 +506,22 @@ impl Drop for FactQueue { } } -struct NextFact<'a>(&'a Shared); +/// The next facts in emission order (at most `.1`). +struct NextFact<'a>(&'a Shared, usize); impl Future for NextFact<'_> { - type Output = Option; + type Output = Option>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let shared = self.0; // Register before checking so a concurrent push cannot be missed. *shared.waker.lock().expect("waker lock") = Some(cx.waker().clone()); - if let Some(fact) = shared.queue.lock().expect("fact queue lock").pop_front() { - return Poll::Ready(Some(fact)); + let mut queue = shared.queue.lock().expect("fact queue lock"); + if !queue.is_empty() { + let n = queue.len().min(self.1); + return Poll::Ready(Some(queue.drain(..n).collect())); } + drop(queue); if shared.closed.load(Ordering::SeqCst) { return Poll::Ready(None); } @@ -502,6 +542,18 @@ impl FactFeed { self.max_concurrent } + /// Hand the sink up to `n` facts of a lane at once (default + /// [`DEFAULT_MAX_BATCH`]; 1 commits fact by fact). + pub fn with_max_batch(mut self, n: usize) -> Self { + self.max_batch = n.max(1); + self + } + + /// Facts per batch at most. + pub fn max_batch(&self) -> usize { + self.max_batch + } + /// Commit facts until the queue is dropped and drained: in emission /// order, or with bounded concurrency in per-principal lanes /// ([`with_max_concurrent_commits`](Self::with_max_concurrent_commits)). @@ -514,17 +566,19 @@ impl FactFeed { { let policy = self.policy; if self.max_concurrent <= 1 { - while let Some(fact) = NextFact(&self.shared).await { - let settlement = commit_with_retries(&sink, fact, policy, &sleep).await; - let _ = self.settlements.send(settlement); + while let Some(facts) = NextFact(&self.shared, self.max_batch).await { + for settlement in commit_with_retries(&sink, facts, policy, &sleep).await { + let _ = self.settlements.send(settlement); + } } return; } - let start = |fact: Fact| Box::pin(commit_with_retries(&sink, fact, policy, &sleep)); + let start = |facts: Vec| Box::pin(commit_with_retries(&sink, facts, policy, &sleep)); Pump { shared: &self.shared, settlements: &self.settlements, limit: self.max_concurrent, + max_batch: self.max_batch, start, waiting: VecDeque::new(), active: Vec::new(), @@ -533,33 +587,68 @@ impl FactFeed { } } +/// Commit `facts` in order through [`FactSink::commit_batch`], retrying +/// from the first unsettled fact; `policy` bounds the attempts at each +/// fact as if it were committed alone. async fn commit_with_retries( sink: &S, - fact: Fact, + facts: Vec, policy: RetryPolicy, sleep: &F, -) -> FactSettlement +) -> Vec where S: FactSink, F: Fn(Duration) -> Fut, Fut: Future, { + let max_attempts = policy.max_attempts.max(1); + let mut settled = Vec::with_capacity(facts.len()); + let mut pending: VecDeque = facts.into(); + // Attempts at the head of `pending`. let mut attempts = 0; - let outcome = loop { + while !pending.is_empty() { attempts += 1; - match sink.commit(&fact).await { - Ok(receipt) => break FactOutcome::Committed(receipt), - Err(FactError::Retryable(_)) if attempts < policy.max_attempts.max(1) => { - sleep(policy.backoff(attempts)).await; - } - Err(error) => break FactOutcome::Failed { error, attempts }, + let results = sink.commit_batch(pending.make_contiguous()).await; + let answered = results.len().min(pending.len()); + // A sink that answered for nothing is retried like a retryable head. + let mut retry = answered == 0; + for result in results.into_iter().take(answered) { + let outcome = match result { + Ok(receipt) => FactOutcome::Committed(receipt), + Err(FactError::Retryable(_)) if attempts < max_attempts => { + retry = true; + break; + } + Err(error) => FactOutcome::Failed { error, attempts }, + }; + let fact = pending.pop_front().expect("a result per attempted fact"); + settled.push(FactSettlement { + key: fact.key, + command: fact.command, + outcome, + }); + // The next result is the next fact's first attempt. + attempts = 1; + } + if !retry { + // The facts left (if any) were not attempted yet. + attempts = 0; + } else if answered == 0 && attempts >= max_attempts { + let fact = pending.pop_front().expect("a pending fact"); + settled.push(FactSettlement { + key: fact.key, + command: fact.command, + outcome: FactOutcome::Failed { + error: FactError::Retryable("the sink committed nothing".into()), + attempts, + }, + }); + attempts = 0; + } else { + sleep(policy.backoff(attempts)).await; } - }; - FactSettlement { - key: fact.key, - command: fact.command, - outcome, } + settled } /// Concurrent commit driver: at most `limit` commits in flight, one per @@ -568,6 +657,7 @@ struct Pump<'a, T, M> { shared: &'a Shared, settlements: &'a mpsc::Sender, limit: usize, + max_batch: usize, start: M, waiting: VecDeque, active: Vec<(Option, Pin>)>, @@ -575,8 +665,8 @@ struct Pump<'a, T, M> { impl Future for Pump<'_, T, M> where - T: Future, - M: FnMut(Fact) -> Pin> + Unpin, + T: Future>, + M: FnMut(Vec) -> Pin> + Unpin, { type Output = (); @@ -598,16 +688,27 @@ where i += 1; continue; } - let fact = this.waiting.remove(i).expect("index in range"); + // The lane's waiting facts, in order, up to a batch. + let mut batch = vec![this.waiting.remove(i).expect("index in range")]; + let mut k = i; + while k < this.waiting.len() && batch.len() < this.max_batch { + if this.waiting[k].principal == lane { + batch.push(this.waiting.remove(k).expect("index in range")); + } else { + k += 1; + } + } blocked.insert(lane.clone()); - this.active.push((lane, (this.start)(fact))); + this.active.push((lane, (this.start)(batch))); } let mut progressed = false; let mut j = 0; while j < this.active.len() { match this.active[j].1.as_mut().poll(cx) { - Poll::Ready(settlement) => { - let _ = this.settlements.send(settlement); + Poll::Ready(settlements) => { + for settlement in settlements { + let _ = this.settlements.send(settlement); + } this.active.remove(j); progressed = true; } @@ -678,4 +779,182 @@ impl FactSink for Arc { fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { (**self).commit(fact) } + + fn commit_batch<'a>( + &'a self, + facts: &'a [Fact], + ) -> BoxFuture<'a, Vec>> { + (**self).commit_batch(facts) + } +} + +#[cfg(test)] +mod batch_tests { + use super::*; + + /// Commits whole batches, recording each; `flaky` answers retryable + /// `failures` times when it reaches that key. + #[derive(Default)] + struct BatchSink { + batches: Mutex>>, + committed: Mutex>, + flaky: Option<(String, Mutex)>, + } + + impl FactSink for BatchSink { + fn commit<'a>(&'a self, _: &'a Fact) -> BoxFuture<'a, Result> { + unreachable!("batches only") + } + + fn commit_batch<'a>( + &'a self, + facts: &'a [Fact], + ) -> BoxFuture<'a, Vec>> { + Box::pin(async move { + let keys = facts.iter().map(|f| f.key().to_string()).collect(); + self.batches.lock().unwrap().push(keys); + let mut results = Vec::new(); + for fact in facts { + if let Some((key, failures)) = &self.flaky { + let mut failures = failures.lock().unwrap(); + if fact.key() == key && *failures > 0 { + *failures -= 1; + results.push(Err(FactError::Retryable("busy".into()))); + break; + } + } + let mut committed = self.committed.lock().unwrap(); + let replayed = committed.iter().any(|k| k == fact.key()); + if !replayed { + committed.push(fact.key().to_string()); + } + results.push(Ok(FactReceipt { + payload: Value::Null, + replayed, + })); + } + results + }) + } + } + + fn fact(key: &str, principal: Option<&str>) -> Fact { + Fact::new(key, "credit", Value::Null) + .bind(principal.map(str::to_string), FactStamp::default()) + } + + async fn settle(queue: &mut FactQueue, n: usize) -> Vec { + let mut settled = Vec::new(); + while settled.len() < n { + settled.extend(queue.drain()); + tokio::time::sleep(Duration::from_millis(1)).await; + } + settled + } + + fn keys(settled: &[FactSettlement]) -> Vec<&str> { + settled.iter().map(|s| s.key.as_str()).collect() + } + + fn policy(max_attempts: u32) -> RetryPolicy { + RetryPolicy { + max_attempts, + base_backoff: Duration::from_millis(1), + max_backoff: Duration::from_millis(2), + } + } + + #[tokio::test] + async fn a_lane_settles_in_batches_in_order() { + let (mut queue, feed) = fact_channel(64, policy(3)); + for i in 0..5 { + queue + .try_send(fact(&format!("a{i}"), Some("alice"))) + .unwrap(); + } + for i in 0..3 { + queue.try_send(fact(&format!("b{i}"), Some("bob"))).unwrap(); + } + let sink = Arc::new(BatchSink::default()); + let feed = feed.with_max_concurrent_commits(4).with_max_batch(4); + let worker = tokio::spawn(feed.run(sink.clone(), tokio::time::sleep)); + let settled = settle(&mut queue, 8).await; + drop(queue); + worker.await.unwrap(); + + let batches = sink.batches.lock().unwrap().clone(); + assert!(batches.contains(&vec!["a0".into(), "a1".into(), "a2".into(), "a3".into()])); + assert!(batches.contains(&vec!["b0".into(), "b1".into(), "b2".into()])); + assert!(batches.contains(&vec!["a4".into()])); + assert_eq!(batches.len(), 3); + let lane = |p: char| -> Vec<&str> { + keys(&settled) + .into_iter() + .filter(|k| k.starts_with(p)) + .collect() + }; + assert_eq!(lane('a'), ["a0", "a1", "a2", "a3", "a4"]); + assert_eq!(lane('b'), ["b0", "b1", "b2"]); + assert!(settled + .iter() + .all(|s| matches!(s.outcome, FactOutcome::Committed(_)))); + } + + #[tokio::test] + async fn a_batch_retries_from_its_first_retryable_fact() { + let (mut queue, feed) = fact_channel(64, policy(3)); + for i in 0..5 { + queue.try_send(fact(&format!("f{i}"), None)).unwrap(); + } + let sink = Arc::new(BatchSink { + flaky: Some(("f2".into(), Mutex::new(1))), + ..BatchSink::default() + }); + let worker = tokio::spawn(feed.with_max_batch(8).run(sink.clone(), tokio::time::sleep)); + let settled = settle(&mut queue, 5).await; + drop(queue); + worker.await.unwrap(); + + assert_eq!(keys(&settled), ["f0", "f1", "f2", "f3", "f4"]); + assert!(settled + .iter() + .all(|s| matches!(s.outcome, FactOutcome::Committed(_)))); + // Exactly once each, in order; the retry resent only f2 onward. + assert_eq!( + *sink.committed.lock().unwrap(), + ["f0", "f1", "f2", "f3", "f4"] + ); + assert_eq!( + *sink.batches.lock().unwrap(), + [vec!["f0", "f1", "f2", "f3", "f4"], vec!["f2", "f3", "f4"]] + ); + } + + #[tokio::test] + async fn attempts_count_per_fact_in_a_batch() { + let (mut queue, feed) = fact_channel(64, policy(2)); + for i in 0..3 { + queue.try_send(fact(&format!("f{i}"), None)).unwrap(); + } + let sink = Arc::new(BatchSink { + flaky: Some(("f1".into(), Mutex::new(u32::MAX))), + ..BatchSink::default() + }); + let worker = tokio::spawn(feed.with_max_batch(8).run(sink.clone(), tokio::time::sleep)); + let settled = settle(&mut queue, 3).await; + drop(queue); + worker.await.unwrap(); + + assert_eq!(keys(&settled), ["f0", "f1", "f2"]); + assert!(matches!(settled[0].outcome, FactOutcome::Committed(_))); + assert!(matches!( + settled[1].outcome, + FactOutcome::Failed { + error: FactError::Retryable(_), + attempts: 2 + } + )); + assert!(matches!(settled[2].outcome, FactOutcome::Committed(_))); + assert_eq!(*sink.committed.lock().unwrap(), ["f0", "f2"]); + } } From cd04f3e4c9d1b1a9c275af372887f413c3e9007c Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 00:16:20 -0500 Subject: [PATCH 03/22] perf(game): encode deltas once per patch, thin far update rates Every entity state is now a shared revision. Each changed entity's update record is encoded once per patch from its previous state, and clients with an older baseline share one record per baseline. Per-client work is selection and copy, and the bytes stay identical (a randomized test checks this against the old per-client path). Distance-based update rates (InterestConfig::update_rates, on by default): a known entity updates every patch within 0.5x the view radius, every 2nd patch up to 0.8x, and every 4th beyond, staggered by id. Creates, removes, globals and #[replicated(.., every_patch)] kinds are never held. [[replication-encode-once-lod]] --- distributed_game/PROTOCOL.md | 3 + distributed_game/README.md | 20 + distributed_game/src/host.rs | 11 +- distributed_game/src/interest.rs | 66 +- distributed_game/src/lib.rs | 2 +- distributed_game/src/replay.rs | 2 + distributed_game/src/replication.rs | 950 ++++++++++++++++++--- distributed_game/src/schema.rs | 10 + distributed_game/tests/authoring/main.rs | 3 +- distributed_game/tests/replication/main.rs | 125 ++- distributed_game_macros/src/lib.rs | 11 +- distributed_game_macros/src/replicated.rs | 15 +- 12 files changed, 1081 insertions(+), 137 deletions(-) diff --git a/distributed_game/PROTOCOL.md b/distributed_game/PROTOCOL.md index 276140542..f9a4ef703 100644 --- a/distributed_game/PROTOCOL.md +++ b/distributed_game/PROTOCOL.md @@ -425,6 +425,9 @@ update := varuint id, varuint mask, value(field_i) for each set bit i, ascendin accumulator; the highest accumulators are sent first and reset; deferred entities keep accumulating, so none starve. At least one candidate is sent per Delta. Removes are never budgeted. +- Update rates (default): a changed, already-known, non-global entity + beyond half the radius is a candidate only every 2nd patch, beyond 0.8 + of it every 4th. Clients see this only as less frequent updates. ### 7.4 Worked example diff --git a/distributed_game/README.md b/distributed_game/README.md index 00980704c..0b9f4cbe6 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -418,6 +418,21 @@ client already has: (4 KiB). Creates and updates compete by priority (kind × entity × field × nearness). Anything deferred builds up priority, so it is always sent eventually. +- **Update rates.** Far entities are updated less often: within half the + view radius every patch, up to 0.8 of it every 2nd patch, beyond that + every 4th (`HostConfig::interest.update_rates`, staggered by entity id, + so a far entity's latest state is at most 3 patches late). Creates, + removes and global entities are never held, nor are kinds marked + `#[replicated(.., every_patch)]`; keep your own entity near the view + centre (it is when the view follows the player) or mark its kind. + `UpdateRates::every_patch()` turns the rates off. +- **Encoded once.** Each changed entity's update is encoded once per patch + and copied into every Delta of a client that was current; clients + further behind (budget or update rates) share one encoding per + baseline. The bytes are exactly those of a per-client encoding (a + randomized test checks it), so a dense room's cost per client is + selection and copying: 128 mutually visible players build a patch + about 5× faster than encoding per client. - **Events** carry one-off messages such as "kill", "saved" or "weather changed" next to the Deltas. Declare them as a `#[derive(GameEvent)]` enum (each variant `#[game_event(id = N)]`: unit, named fields, or one @@ -997,6 +1012,11 @@ unlimited rewind. client last received, which only works because nothing is lost or reordered. Loss shows up as delay, which the adaptive interpolation absorbs. +- **Shared encodings, per-client baselines.** Deltas stay per client + (each knows only what it was sent), but each distinct entity state and + each update between two states is encoded once and shared, so dense + rooms pay for copying, not encoding. Lowering far update rates on top + is a server choice: the wire and the client are unchanged. ## Status and limits diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index 8b2a87153..c2b78c891 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -841,6 +841,9 @@ pub struct ZoneHost { token_source: Box String + Send>, pending: AdvanceReport, hash_ticks: bool, + /// The last patch's world: the next one shares its unchanged entities + /// and encodes each change once for every client. + last_world: Option, } macro_rules! hook { @@ -917,6 +920,7 @@ impl ZoneHost { token_source: Box::new(default_token), pending: AdvanceReport::default(), hash_ticks: false, + last_world: None, }; host.process_kicks(); host @@ -1975,7 +1979,11 @@ impl ZoneHost { let view = self.world_view(); let core = &mut self.core; self.pending.entities = Some(view.entities().len()); - let world = EncodedWorld::build(&self.schema, view, &core.config.interest); + let interest = &core.config.interest; + let world = match self.last_world.take() { + Some(last) => last.next(&self.schema, view, interest), + None => EncodedWorld::build(&self.schema, view, interest), + }; let tick = core.tick as u32; let budget = core.config.byte_budget; let mut frames = Vec::with_capacity(core.clients.len()); @@ -2010,6 +2018,7 @@ impl ZoneHost { }); } core.outbound.extend(frames); + self.last_world = Some(world); } /// Run due ticks and patches for the time elapsed on the clock. diff --git a/distributed_game/src/interest.rs b/distributed_game/src/interest.rs index b041cdc66..b11dcaa1f 100644 --- a/distributed_game/src/interest.rs +++ b/distributed_game/src/interest.rs @@ -5,6 +5,10 @@ //! when within `radius` and *leaves* only beyond `radius * (1 + leave_margin)` //! (hysteresis prevents create/remove flicker at the edge). Entities marked //! global are visible to everyone regardless of distance. +//! +//! [`UpdateRates`] thin out updates by distance: a known entity far from +//! the view centre is updated only every few patches (creates, removes and +//! global entities are never delayed). use std::collections::HashMap; @@ -31,13 +35,16 @@ impl View { } } -/// Grid and hysteresis settings. +/// Grid, hysteresis and update-rate settings. #[derive(Debug, Clone, Copy, PartialEq)] pub struct InterestConfig { /// Grid cell edge length (world units); ~view radius / 2 works well. pub cell_size: f32, /// Extra fraction of the radius before a known entity leaves the view. pub leave_margin: f32, + /// Distance-based update rates ([`UpdateRates::every_patch`] turns them + /// off). + pub update_rates: UpdateRates, } impl Default for InterestConfig { @@ -45,6 +52,63 @@ impl Default for InterestConfig { Self { cell_size: 16.0, leave_margin: 0.1, + update_rates: UpdateRates::default(), + } + } +} + +/// Distance-based update rates (level of detail), as fractions of the view +/// radius. A changed entity the client already has is an update candidate +/// every patch within `near × radius`, every `mid_every`-th patch within +/// `mid × radius` and every `far_every`-th patch beyond. Which patches is +/// staggered by entity id, so a far entity's latest state arrives at most +/// `far_every - 1` patches late (budget permitting; keep `mid_every` a +/// divisor of `far_every`). Creates, removes, global entities and kinds +/// marked `#[replicated(.., every_patch)]` are never delayed. The default +/// is `near = 0.5`, `mid = 0.8`, every 2nd and every 4th patch: at 20 +/// patches/s, 10 Hz past half the radius and 5 Hz near its edge. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct UpdateRates { + /// Fraction of the radius updated every patch. + pub near: f32, + /// Fraction of the radius updated every `mid_every`-th patch. + pub mid: f32, + /// Patch interval between `near` and `mid` (1 = every patch). + pub mid_every: u32, + /// Patch interval beyond `mid` (1 = every patch). + pub far_every: u32, +} + +impl UpdateRates { + /// Every entity every patch (no distance-based rates). + pub fn every_patch() -> Self { + Self { + near: f32::INFINITY, + mid: f32::INFINITY, + mid_every: 1, + far_every: 1, + } + } + + /// Patch interval at `distance` from a view of `radius`. + pub fn interval(&self, distance: f32, radius: f32) -> u32 { + if distance <= self.near * radius { + 1 + } else if distance <= self.mid * radius { + self.mid_every.max(1) + } else { + self.far_every.max(1) + } + } +} + +impl Default for UpdateRates { + fn default() -> Self { + Self { + near: 0.5, + mid: 0.8, + mid_every: 2, + far_every: 4, } } } diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index 51766bc20..c0430baa5 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -119,7 +119,7 @@ pub use handoff::{HandoffError, HandoffRequest, Transfer}; pub use host::{AdvanceReport, Ctx, FrameStats, HostConfig, JoinError, Joined, Outbound, ZoneHost}; pub use inbox::{Delivered, NoMessages, ZoneInbound}; pub use input::{ConsumePolicy, InputBuffer, InputCredits, PushOutcome, SequencedInput}; -pub use interest::{InterestConfig, View}; +pub use interest::{InterestConfig, UpdateRates, View}; pub use prediction::{PredictContext, PredictStep}; pub use protocol::{ClientMessage, ClientMirror, Role, ServerMessage, PROTOCOL_VERSION}; pub use pure::GamePure; diff --git a/distributed_game/src/replay.rs b/distributed_game/src/replay.rs index 7081e43da..653a559f0 100644 --- a/distributed_game/src/replay.rs +++ b/distributed_game/src/replay.rs @@ -654,9 +654,11 @@ fn get_config(r: &mut Reader<'_>) -> Result { reconnect_grace: get_dur(r)?, byte_budget: get_u64(r)? as usize, default_view_radius: r.f32()?, + // Update rates only shape Deltas, never the replayed world. interest: InterestConfig { cell_size: r.f32()?, leave_margin: r.f32()?, + ..InterestConfig::default() }, content_version: r.string()?, world_seed: get_u64(r)?, diff --git a/distributed_game/src/replication.rs b/distributed_game/src/replication.rs index 15f54ab21..08c69ceb5 100644 --- a/distributed_game/src/replication.rs +++ b/distributed_game/src/replication.rs @@ -17,9 +17,30 @@ //! eventually sent. At least one candidate is sent per patch even if it alone //! exceeds the budget. Change detection compares encoded bytes, so //! sub-quantum jitter never costs bandwidth. +//! +//! **Encode once.** Each distinct encoded state of an entity is a shared +//! *revision*; a client remembers the revision it was sent, not a copy. A +//! world built with [`EncodedWorld::next`] carries, per changed entity, the +//! update record from its previous state, so a client that is current costs +//! a revision compare and a copy. Clients with an older baseline (deferred +//! by the budget or by update rates) share one record per baseline, encoded +//! on first use in the patch. The bytes are those of encoding per client. +//! +//! **Update rates.** A changed entity the client already has is an update +//! candidate only every [`UpdateRates::interval`]-th patch for its distance +//! (staggered by id) unless it is global or its kind is `every_patch`; held +//! entities keep their accumulated priority. Creates and removes are never +//! held. -use std::collections::{HashMap, HashSet}; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::hash::{BuildHasherDefault, Hasher}; +use std::ops::Range; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +#[cfg(doc)] +use crate::interest::UpdateRates; use crate::interest::{InterestConfig, SpatialGrid, View}; use crate::protocol::{tag, Writer}; use crate::schema::{EntitySchema, FieldValue}; @@ -92,6 +113,107 @@ impl WorldView { } } +/// Multiplicative hasher for the integer keys of the per-client maps, hashed +/// once per visible entity per client per patch (ids are assigned by the +/// game, not chosen by clients). +#[derive(Debug, Default, Clone, Copy)] +struct IdHasher(u64); + +impl IdHasher { + const K: u64 = 0x517c_c1b7_2722_0a95; + + fn mix(&mut self, n: u64) { + self.0 = (self.0.rotate_left(5) ^ n).wrapping_mul(Self::K); + } +} + +impl Hasher for IdHasher { + fn finish(&self) -> u64 { + self.0 + } + + fn write(&mut self, bytes: &[u8]) { + for b in bytes { + self.mix(u64::from(*b)); + } + } + + fn write_u32(&mut self, n: u32) { + self.mix(u64::from(n)); + } + + fn write_u64(&mut self, n: u64) { + self.mix(n); + } + + fn write_usize(&mut self, n: usize) { + self.mix(n as u64); + } +} + +type IdMap = HashMap>; + +/// Revisions are unique across all worlds: equal revisions are equal bytes, +/// whichever world or replica they came from. +static NEXT_REV: AtomicU64 = AtomicU64::new(1); + +/// One encoded state of an entity, shared by the worlds it is unchanged in +/// and by every client it was sent to. +#[derive(Debug)] +struct Payload { + rev: u64, + data: Vec, + offsets: Vec, + /// The create record: `varuint id, varuint kind, data`. + create: Vec, +} + +impl Payload { + fn new(id: u32, kind: u16, data: Vec, offsets: Vec) -> Self { + let mut create = Writer::new(); + create.varuint(id).varuint(u32::from(kind)).raw(&data); + Self { + rev: NEXT_REV.fetch_add(1, Ordering::Relaxed), + data, + offsets, + create: create.into_bytes(), + } + } + + fn field(&self, index: usize) -> &[u8] { + &self.data[self.offsets[index]..self.offsets[index + 1]] + } + + /// Mask of the first `fields` fields whose bytes differ from `old`. + fn changed_since(&self, old: &Payload, fields: usize) -> u32 { + let mut mask = 0u32; + for i in 0..fields { + if old.field(i) != self.field(i) { + mask |= 1 << i; + } + } + mask + } + + /// The update record `varuint id, varuint mask, masked fields`. + fn update(&self, id: u32, mask: u32, w: &mut Writer) { + w.varuint(id).varuint(mask); + for i in 0..self.offsets.len() - 1 { + if mask & (1 << i) != 0 { + w.raw(self.field(i)); + } + } + } +} + +/// An entity's update record from revision `base` to its current one. +#[derive(Debug)] +struct SharedUpdate { + base: u64, + mask: u32, + record: Vec, +} + #[derive(Debug, Clone)] struct EncodedEntity { id: u32, @@ -100,34 +222,71 @@ struct EncodedEntity { y: f32, priority: f32, global: bool, - data: Vec, - offsets: Vec, + payload: Arc, + /// From its state in the previous world (kept while unchanged). + update: Option>, } -impl EncodedEntity { - fn field(&self, index: usize) -> &[u8] { - &self.data[self.offsets[index]..self.offsets[index + 1]] - } +/// Update records for older baselines: `(entity index, baseline rev)` → +/// `(mask, record range)`, encoded on first use in a patch. +#[derive(Debug, Default)] +struct UpdateCache { + records: IdMap<(u32, u64), (u32, Range)>, + bytes: Vec, } /// A patch's encoded world, shared by every client. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct EncodedWorld { entities: Vec, - by_id: HashMap, + by_id: IdMap, grid: SpatialGrid, globals: Vec, + patch: u32, + cache: Mutex, +} + +impl Clone for EncodedWorld { + fn clone(&self) -> Self { + Self { + entities: self.entities.clone(), + by_id: self.by_id.clone(), + grid: self.grid.clone(), + globals: self.globals.clone(), + patch: self.patch, + cache: Mutex::default(), + } + } } impl EncodedWorld { - /// Encode `view` against `schema`. Entities of unknown kinds or duplicate - /// ids are skipped. + /// Encode `view` against `schema` as the first patch. Entities of + /// unknown kinds or duplicate ids are skipped. pub fn build(schema: &EntitySchema, view: WorldView, interest: &InterestConfig) -> Self { + Self::encode(schema, view, interest, None) + } + + /// Encode the patch after `self`: unchanged entities keep their + /// revision, changed ones get one update record from their state here, + /// shared by every client that has it. Update rates count patches along + /// this chain, so build every patch after the first with `next`. + pub fn next(&self, schema: &EntitySchema, view: WorldView, interest: &InterestConfig) -> Self { + Self::encode(schema, view, interest, Some(self)) + } + + fn encode( + schema: &EntitySchema, + view: WorldView, + interest: &InterestConfig, + previous: Option<&EncodedWorld>, + ) -> Self { let mut world = Self { entities: Vec::with_capacity(view.entities.len()), - by_id: HashMap::new(), + by_id: IdMap::default(), grid: SpatialGrid::new(interest.cell_size), globals: Vec::new(), + patch: previous.map_or(0, |p| p.patch.wrapping_add(1)), + cache: Mutex::default(), }; let zero = FieldValue::U(0); for e in view.entities { @@ -144,6 +303,29 @@ impl EncodedWorld { def.ty.encode(&mut w, e.fields.get(index).unwrap_or(&zero)); } offsets.push(w.len()); + let data = w.into_bytes(); + let before = previous + .and_then(|p| p.by_id.get(&e.id).map(|&i| &p.entities[i])) + .filter(|p| p.kind == e.kind && p.payload.offsets.len() == offsets.len()); + let (payload, update) = match before { + Some(p) if p.payload.data == data && p.payload.offsets == offsets => { + (p.payload.clone(), p.update.clone()) + } + _ => { + let payload = Arc::new(Payload::new(e.id, e.kind, data, offsets)); + let update = before.map(|p| { + let mask = payload.changed_since(&p.payload, kind.fields.len()); + let mut w = Writer::new(); + payload.update(e.id, mask, &mut w); + Arc::new(SharedUpdate { + base: p.payload.rev, + mask, + record: w.into_bytes(), + }) + }); + (payload, update) + } + }; let index = world.entities.len() as u32; if e.global { world.globals.push(index); @@ -158,8 +340,8 @@ impl EncodedWorld { y: e.y, priority: e.priority, global: e.global, - data: w.into_bytes(), - offsets, + payload, + update, }); } world @@ -176,12 +358,6 @@ impl EncodedWorld { } } -#[derive(Debug, Clone)] -struct Known { - data: Vec, - offsets: Vec, -} - /// Counters from one Delta build. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct DeltaStats { @@ -197,16 +373,27 @@ pub struct DeltaStats { pub budget_bytes: usize, } +/// A visible or known entity of one client. +#[derive(Debug, Clone, Default)] +struct Slot { + /// What it was last sent. + known: Option>, + accum: f32, + /// Build that last saw it visible. + seen: u32, +} + /// What one client has been sent. #[derive(Debug, Clone, Default)] pub struct ClientReplica { - known: HashMap, - accum: HashMap, + slots: IdMap, + builds: u32, } enum Candidate { Create, - Update(u32), + Shared, + Cached(Range), } impl ClientReplica { @@ -217,13 +404,15 @@ impl ClientReplica { /// Forget everything (e.g. after reconnect the client gets fresh creates). pub fn reset(&mut self) { - self.known.clear(); - self.accum.clear(); + self.slots.clear(); } /// Ids the client currently knows. pub fn known_ids(&self) -> impl Iterator + '_ { - self.known.keys().copied() + self.slots + .iter() + .filter(|(_, slot)| slot.known.is_some()) + .map(|(id, _)| *id) } /// Build this client's Delta frame and update the last-sent state. @@ -238,57 +427,54 @@ impl ClientReplica { server_tick: u32, ack_seq: u32, ) -> (Vec, DeltaStats) { + self.builds = self.builds.wrapping_add(1); + let stamp = self.builds; let radius = view.radius.max(0.0); let stay_radius = radius * (1.0 + interest.leave_margin.max(0.0)); let mut hits = Vec::new(); world.grid.query(view.x, view.y, stay_radius, &mut hits); hits.extend_from_slice(&world.globals); + let mut cache = world.cache.lock().unwrap_or_else(PoisonError::into_inner); + let cache = &mut *cache; - // Visible entities with their distance. - let mut visible: Vec<(usize, f32)> = Vec::new(); - let mut seen = HashSet::new(); + // Visible entities (indices are unique) become candidates with + // accumulated priority. + let mut visible = 0usize; + let mut candidates: Vec<(usize, Candidate, f32)> = Vec::new(); for index in hits { let index = index as usize; let e = &world.entities[index]; - if !seen.insert(e.id) { - continue; - } let d = if e.global { 0.0 } else { view.distance(e.x, e.y) }; - let limit = if self.known.contains_key(&e.id) { - stay_radius - } else { - radius + let slot = match self.slots.entry(e.id) { + Entry::Occupied(slot) => { + let limit = if slot.get().known.is_some() { + stay_radius + } else { + radius + }; + if !e.global && d > limit { + continue; + } + slot.into_mut() + } + Entry::Vacant(slot) => { + if !e.global && d > radius { + continue; + } + slot.insert(Slot::default()) + } }; - if e.global || d <= limit { - visible.push((index, d)); - } - } - - let visible_ids: HashSet = - visible.iter().map(|(i, _)| world.entities[*i].id).collect(); - let mut removes: Vec = self - .known - .keys() - .copied() - .filter(|id| !visible_ids.contains(id)) - .collect(); - removes.sort_unstable(); - for id in &removes { - self.known.remove(id); - } - - // Candidates with accumulated priority. - let mut candidates: Vec<(usize, Candidate, f32)> = Vec::new(); - for (index, d) in visible { - let e = &world.entities[index]; + slot.seen = stamp; + visible += 1; let Some(kind) = schema.get(e.kind) else { + slot.accum = 0.0; continue; }; - let (candidate, field_prio) = match self.known.get(&e.id) { + let (candidate, field_prio) = match &slot.known { None => ( Candidate::Create, kind.fields @@ -297,32 +483,71 @@ impl ClientReplica { .fold(1.0_f32, f32::max), ), Some(known) => { - let mut mask = 0u32; - let mut prio = 0.0_f32; - for (i, def) in kind.fields.iter().enumerate() { - let old = &known.data[known.offsets[i]..known.offsets[i + 1]]; - if old != e.field(i) { - mask |= 1 << i; - prio = prio.max(def.priority); + if known.rev == e.payload.rev { + slot.accum = 0.0; + continue; + } + if !e.global && !kind.every_patch { + let every = interest.update_rates.interval(d, radius); + if every > 1 && !world.patch.wrapping_add(e.id).is_multiple_of(every) { + // Held: keeps its accumulated priority. + continue; } } + let fields = kind.fields.len(); + let (mask, candidate) = match &e.update { + Some(update) + if update.base == known.rev + && fields + 1 == e.payload.offsets.len() => + { + (update.mask, Candidate::Shared) + } + _ => { + let (mask, range) = cache + .records + .entry((index as u32, known.rev)) + .or_insert_with(|| { + let mask = e.payload.changed_since(known, fields); + let start = cache.bytes.len(); + let mut w = Writer::new(); + e.payload.update(e.id, mask, &mut w); + cache.bytes.extend_from_slice(w.as_bytes()); + (mask, start..cache.bytes.len()) + }) + .clone(); + (mask, Candidate::Cached(range)) + } + }; if mask == 0 { + slot.accum = 0.0; continue; } - (Candidate::Update(mask), prio) + let prio = kind + .fields + .iter() + .enumerate() + .filter(|(i, _)| mask & (1 << i) != 0) + .fold(0.0_f32, |p, (_, def)| p.max(def.priority)); + (candidate, prio) } }; let distance_factor = 1.0 / (1.0 + d / radius.max(f32::EPSILON)); let gain = (kind.priority * e.priority * field_prio * distance_factor).max(1e-6); - let accum = self.accum.entry(e.id).or_insert(0.0); - *accum += gain; - candidates.push((index, candidate, *accum)); + slot.accum += gain; + candidates.push((index, candidate, slot.accum)); + } + + // Known entities gone or out of view. + let mut removes: Vec = Vec::new(); + if visible != self.slots.len() { + self.slots.retain(|id, slot| { + if slot.seen != stamp && slot.known.is_some() { + removes.push(*id); + } + slot.seen == stamp + }); + removes.sort_unstable(); } - let candidate_ids: HashSet = candidates - .iter() - .map(|(i, _, _)| world.entities[*i].id) - .collect(); - self.accum.retain(|id, _| candidate_ids.contains(id)); candidates.sort_by(|a, b| { b.2.total_cmp(&a.2) .then(world.entities[a.0].id.cmp(&world.entities[b.0].id)) @@ -334,49 +559,28 @@ impl ClientReplica { }; let mut creates = Writer::new(); let mut updates = Writer::new(); - let mut scratch = Writer::new(); for (index, candidate, _) in candidates { let e = &world.entities[index]; - scratch.clear(); - match candidate { - Candidate::Create => { - scratch - .varuint(e.id) - .varuint(u32::from(e.kind)) - .raw(&e.data); - } - Candidate::Update(mask) => { - scratch.varuint(e.id).varuint(mask); - for i in 0..e.offsets.len() - 1 { - if mask & (1 << i) != 0 { - scratch.raw(e.field(i)); - } - } - } - } + let record = match &candidate { + Candidate::Create => &e.payload.create[..], + Candidate::Shared => &e.update.as_ref().expect("shared update").record[..], + Candidate::Cached(range) => &cache.bytes[range.clone()], + }; let used = creates.len() + updates.len(); - if used > 0 && used + scratch.len() > budget { + if used > 0 && used + record.len() > budget { stats.deferred += 1; continue; } - match candidate { - Candidate::Create => { - creates.raw(scratch.as_bytes()); - stats.creates += 1; - } - Candidate::Update(_) => { - updates.raw(scratch.as_bytes()); - stats.updates += 1; - } + if let Candidate::Create = candidate { + creates.raw(record); + stats.creates += 1; + } else { + updates.raw(record); + stats.updates += 1; } - self.accum.remove(&e.id); - self.known.insert( - e.id, - Known { - data: e.data.clone(), - offsets: e.offsets.clone(), - }, - ); + let slot = self.slots.get_mut(&e.id).expect("candidate slot"); + slot.accum = 0.0; + slot.known = Some(e.payload.clone()); } stats.budget_bytes = creates.len() + updates.len(); @@ -397,3 +601,527 @@ impl ClientReplica { (frame.into_bytes(), stats) } } + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::time::{Duration, Instant}; + + use super::*; + use crate::interest::UpdateRates; + use crate::rng::ZoneRng; + use crate::schema::{FieldType, KindDef}; + + struct Known { + rev: u64, + data: Vec, + offsets: Vec, + } + + enum RefCandidate { + Create, + Update(u32), + } + + /// The per-client path encode-once replaced (each client compares, + /// encodes and copies every entity itself) with the same update-rate + /// hold: the reference for byte equality, and "before" in the bench. + #[derive(Default)] + struct Reference { + known: HashMap, + accum: HashMap, + } + + impl Reference { + fn reset(&mut self) { + self.known.clear(); + self.accum.clear(); + } + + #[allow(clippy::too_many_arguments)] + fn build_delta( + &mut self, + schema: &EntitySchema, + world: &EncodedWorld, + interest: &InterestConfig, + view: &View, + budget: usize, + server_tick: u32, + ack_seq: u32, + ) -> (Vec, DeltaStats) { + let radius = view.radius.max(0.0); + let stay_radius = radius * (1.0 + interest.leave_margin.max(0.0)); + let mut hits = Vec::new(); + world.grid.query(view.x, view.y, stay_radius, &mut hits); + hits.extend_from_slice(&world.globals); + + let mut visible: Vec<(usize, f32)> = Vec::new(); + let mut seen = HashSet::new(); + for index in hits { + let index = index as usize; + let e = &world.entities[index]; + if !seen.insert(e.id) { + continue; + } + let d = if e.global { + 0.0 + } else { + view.distance(e.x, e.y) + }; + let limit = if self.known.contains_key(&e.id) { + stay_radius + } else { + radius + }; + if e.global || d <= limit { + visible.push((index, d)); + } + } + + let visible_ids: HashSet = + visible.iter().map(|(i, _)| world.entities[*i].id).collect(); + let mut removes: Vec = self + .known + .keys() + .copied() + .filter(|id| !visible_ids.contains(id)) + .collect(); + removes.sort_unstable(); + for id in &removes { + self.known.remove(id); + } + + let mut held = HashSet::new(); + let mut candidates: Vec<(usize, RefCandidate, f32)> = Vec::new(); + for (index, d) in visible { + let e = &world.entities[index]; + let p = &e.payload; + let Some(kind) = schema.get(e.kind) else { + continue; + }; + let (candidate, field_prio) = match self.known.get(&e.id) { + None => ( + RefCandidate::Create, + kind.fields + .iter() + .map(|f| f.priority) + .fold(1.0_f32, f32::max), + ), + Some(known) => { + let every = interest.update_rates.interval(d, radius); + if known.rev != p.rev + && !e.global + && !kind.every_patch + && every > 1 + && !world.patch.wrapping_add(e.id).is_multiple_of(every) + { + held.insert(e.id); + continue; + } + let mut mask = 0u32; + let mut prio = 0.0_f32; + for (i, def) in kind.fields.iter().enumerate() { + let old = &known.data[known.offsets[i]..known.offsets[i + 1]]; + if old != p.field(i) { + mask |= 1 << i; + prio = prio.max(def.priority); + } + } + if mask == 0 { + continue; + } + (RefCandidate::Update(mask), prio) + } + }; + let distance_factor = 1.0 / (1.0 + d / radius.max(f32::EPSILON)); + let gain = (kind.priority * e.priority * field_prio * distance_factor).max(1e-6); + let accum = self.accum.entry(e.id).or_insert(0.0); + *accum += gain; + candidates.push((index, candidate, *accum)); + } + let candidate_ids: HashSet = candidates + .iter() + .map(|(i, _, _)| world.entities[*i].id) + .collect(); + self.accum + .retain(|id, _| candidate_ids.contains(id) || held.contains(id)); + candidates.sort_by(|a, b| { + b.2.total_cmp(&a.2) + .then(world.entities[a.0].id.cmp(&world.entities[b.0].id)) + }); + + let mut stats = DeltaStats { + removes: removes.len(), + ..DeltaStats::default() + }; + let mut creates = Writer::new(); + let mut updates = Writer::new(); + let mut scratch = Writer::new(); + for (index, candidate, _) in candidates { + let e = &world.entities[index]; + let p = &e.payload; + scratch.clear(); + match candidate { + RefCandidate::Create => { + scratch + .varuint(e.id) + .varuint(u32::from(e.kind)) + .raw(&p.data); + } + RefCandidate::Update(mask) => { + scratch.varuint(e.id).varuint(mask); + for i in 0..p.offsets.len() - 1 { + if mask & (1 << i) != 0 { + scratch.raw(p.field(i)); + } + } + } + } + let used = creates.len() + updates.len(); + if used > 0 && used + scratch.len() > budget { + stats.deferred += 1; + continue; + } + match candidate { + RefCandidate::Create => { + creates.raw(scratch.as_bytes()); + stats.creates += 1; + } + RefCandidate::Update(_) => { + updates.raw(scratch.as_bytes()); + stats.updates += 1; + } + } + self.accum.remove(&e.id); + self.known.insert( + e.id, + Known { + rev: p.rev, + data: p.data.clone(), + offsets: p.offsets.clone(), + }, + ); + } + stats.budget_bytes = creates.len() + updates.len(); + + let mut frame = Writer::new(); + frame + .u8(tag::DELTA) + .varuint(server_tick) + .varuint(ack_seq) + .varuint(removes.len() as u32); + for id in &removes { + frame.varuint(*id); + } + frame + .varuint(stats.creates as u32) + .raw(creates.as_bytes()) + .varuint(stats.updates as u32) + .raw(updates.as_bytes()); + (frame.into_bytes(), stats) + } + } + + fn schema() -> EntitySchema { + let pos = FieldType::QuantF32 { + min: -200.0, + max: 200.0, + bits: 14, + }; + EntitySchema::new() + .kind( + KindDef::new(1, "thing") + .field("x", pos) + .field("y", pos) + .field_with_priority("hp", FieldType::U16, 3.0) + .field("name", FieldType::ShortStr) + .field("up", FieldType::Bool), + ) + .kind( + KindDef::new(2, "tag") + .field("x", FieldType::F32) + .field("y", FieldType::F32) + .field_with_priority("label", FieldType::ShortStr, 0.5) + .priority(0.5), + ) + .kind( + KindDef::new(3, "beacon") + .field("x", pos) + .field("y", pos) + .field("blob", FieldType::Bytes) + .priority(2.0) + .every_patch(), + ) + } + + /// Random entity pool evolving patch by patch: moves, small value sets + /// (so fields revert), spawns, despawns, duplicates and unknown kinds. + struct Pool { + rng: ZoneRng, + alive: Vec>, + } + + impl Pool { + fn new(seed: u64, n: usize) -> Self { + Self { + rng: ZoneRng::from_seed(seed), + alive: vec![None; n], + } + } + + fn fields(rng: &mut ZoneRng, kind: u16, x: f32, y: f32) -> Vec { + let names = ["", "a", "bc", "def"]; + match kind { + 1 => vec![ + FieldValue::F(x), + FieldValue::F(y), + FieldValue::U(rng.range_u32(0, 4) * 50), + FieldValue::Str(names[rng.range_u32(0, 4) as usize].into()), + FieldValue::Bool(rng.chance(0.5)), + ], + 2 => vec![ + FieldValue::F(x), + FieldValue::F(y), + FieldValue::Str(names[rng.range_u32(0, 4) as usize].into()), + ], + _ => vec![ + FieldValue::F(x), + FieldValue::F(y), + FieldValue::Bytes(vec![1; rng.range_u32(0, 3) as usize]), + ], + } + } + + fn step(&mut self) -> WorldView { + let rng = &mut self.rng; + for (i, slot) in self.alive.iter_mut().enumerate() { + let id = i as u32 + 1; + match slot { + None if rng.chance(0.2) => { + // A reused id keeps its kind: a kind change under a + // known id is outside the delta model. + let kind = 1 + (id % 3) as u16; + let (x, y) = (rng.next_f32() * 160.0 - 80.0, rng.next_f32() * 160.0 - 80.0); + let mut e = EntityState::new(id, kind, x, y, Self::fields(rng, kind, x, y)) + .priority(0.5 + rng.next_f32() * 2.0); + e.global = id.is_multiple_of(9); + *slot = Some(e); + } + Some(_) if rng.chance(0.03) => *slot = None, + Some(e) => { + if rng.chance(0.6) { + e.x = (e.x + rng.next_f32() * 6.0 - 3.0).clamp(-100.0, 100.0); + e.y = (e.y + rng.next_f32() * 6.0 - 3.0).clamp(-100.0, 100.0); + e.fields[0] = FieldValue::F(e.x); + e.fields[1] = FieldValue::F(e.y); + } + if rng.chance(0.2) { + let fresh = Self::fields(rng, e.kind, e.x, e.y); + let i = 2 + rng.range_u32(0, fresh.len() as u32 - 2) as usize; + e.fields[i] = fresh[i].clone(); + } + } + None => {} + } + } + let mut view = WorldView::default(); + for e in self.alive.iter().flatten() { + view.push(e.clone()); + } + if rng.chance(0.3) { + if let Some(e) = self.alive.iter().flatten().next() { + let mut dup = e.clone(); + dup.x += 1.0; + view.push(dup); + } + } + if rng.chance(0.2) { + view.push(EntityState::new(999, 9, 0.0, 0.0, vec![])); + } + view + } + } + + struct Pair { + new: ClientReplica, + old: Reference, + view: View, + budget: usize, + } + + fn compare(seed: u64, rates: UpdateRates) { + let schema = schema(); + let interest = InterestConfig { + cell_size: 12.0, + update_rates: rates, + ..InterestConfig::default() + }; + let mut pool = Pool::new(seed, 50); + let mut rng = ZoneRng::from_seed(seed ^ 0x5eed); + let budgets = [12, 40, 160, 600, 4096, 1 << 20]; + let mut pairs: Vec = (0..6) + .map(|_| Pair { + new: ClientReplica::new(), + old: Reference::default(), + view: View::new(0.0, 0.0, 20.0 + rng.next_f32() * 60.0), + budget: budgets[rng.range_u32(0, 6) as usize], + }) + .collect(); + let mut world: Option = None; + for patch in 0..150u32 { + let view = pool.step(); + let next = match &world { + // Sometimes a fresh chain: no shared updates, patch 0. + Some(prev) if !rng.chance(0.05) => prev.next(&schema, view, &interest), + _ => EncodedWorld::build(&schema, view, &interest), + }; + for (i, pair) in pairs.iter_mut().enumerate() { + pair.view.x = (pair.view.x + rng.next_f32() * 8.0 - 4.0).clamp(-90.0, 90.0); + pair.view.y = (pair.view.y + rng.next_f32() * 8.0 - 4.0).clamp(-90.0, 90.0); + if rng.chance(0.05) { + pair.budget = budgets[rng.range_u32(0, 6) as usize]; + } + if rng.chance(0.01) { + pair.new.reset(); + pair.old.reset(); + } + // Skipped clients fall behind: their baselines go stale. + if rng.chance(0.15) { + continue; + } + let new = pair.new.build_delta( + &schema, + &next, + &interest, + &pair.view, + pair.budget, + patch, + i as u32, + ); + let old = pair.old.build_delta( + &schema, + &next, + &interest, + &pair.view, + pair.budget, + patch, + i as u32, + ); + assert_eq!(new, old, "seed {seed} patch {patch} client {i}"); + let mut known: Vec = pair.new.known_ids().collect(); + let mut want: Vec = pair.old.known.keys().copied().collect(); + known.sort_unstable(); + want.sort_unstable(); + assert_eq!(known, want, "seed {seed} patch {patch} client {i}"); + } + world = Some(next); + } + } + + #[test] + fn encode_once_matches_per_client_encoding() { + for seed in 0..24 { + compare(seed, UpdateRates::every_patch()); + compare(seed, UpdateRates::default()); + compare( + seed, + UpdateRates { + near: 0.2, + mid: 0.6, + mid_every: 3, + far_every: 6, + }, + ); + } + } + + /// `n` players inside a disk of radius 10, each viewing 24 around + /// itself (everyone sees everyone), all moving every patch. + fn dense_room(n: u32, patches: u32, rates: UpdateRates, before: bool) -> (Duration, usize) { + let pos = FieldType::QuantF32 { + min: -40.0, + max: 40.0, + bits: 14, + }; + let schema = EntitySchema::new().kind( + KindDef::new(1, "player") + .field("x", pos) + .field("y", pos) + .field("hp", FieldType::U8) + .field("facing", FieldType::Angle { bits: 8 }) + .priority(2.0), + ); + let interest = InterestConfig { + update_rates: rates, + ..InterestConfig::default() + }; + let mut rng = ZoneRng::from_seed(u64::from(n)); + let mut players: Vec<(f32, f32, f32)> = (0..n) + .map(|_| { + let a = rng.next_f32() * std::f32::consts::TAU; + let r = rng.next_f32().sqrt() * 10.0; + (r * a.cos(), r * a.sin(), a) + }) + .collect(); + let mut new: Vec = (0..n).map(|_| ClientReplica::new()).collect(); + let mut old: Vec = (0..n).map(|_| Reference::default()).collect(); + let mut world: Option = None; + let (mut spent, mut bytes) = (Duration::ZERO, 0usize); + let warmup = 10; + for patch in 0..warmup + patches { + let mut view = WorldView::default(); + for (i, p) in players.iter_mut().enumerate() { + p.2 += 0.05; + p.0 = (p.0 + p.2.cos() * 0.2).clamp(-10.0, 10.0); + p.1 = (p.1 + p.2.sin() * 0.2).clamp(-10.0, 10.0); + view.push(EntityState::new( + i as u32 + 1, + 1, + p.0, + p.1, + vec![ + FieldValue::F(p.0), + FieldValue::F(p.1), + FieldValue::U(100), + FieldValue::F(p.2), + ], + )); + } + let started = Instant::now(); + let next = match (&world, before) { + (Some(prev), false) => prev.next(&schema, view, &interest), + _ => EncodedWorld::build(&schema, view, &interest), + }; + let mut sent = 0; + for (i, p) in players.iter().enumerate() { + let v = View::new(p.0, p.1, 24.0); + let (frame, _) = if before { + old[i].build_delta(&schema, &next, &interest, &v, 4096, patch, 0) + } else { + new[i].build_delta(&schema, &next, &interest, &v, 4096, patch, 0) + }; + sent += frame.len(); + } + if patch >= warmup { + spent += started.elapsed(); + bytes += sent; + } + world = Some(next); + } + (spent / patches, bytes / (patches as usize * n as usize)) + } + + /// `cargo test -p distributed_game --release --lib dense_room_patch_build -- --ignored --nocapture` + #[test] + #[ignore = "timing"] + fn dense_room_patch_build() { + for n in [64, 128] { + let (before, b0) = dense_room(n, 200, UpdateRates::every_patch(), true); + let (once, b1) = dense_room(n, 200, UpdateRates::every_patch(), false); + let (rates, b2) = dense_room(n, 200, UpdateRates::default(), false); + println!( + "{n} clients: per-client {before:?} ({b0} B/client), encode once {once:?} ({b1} B), + update rates {rates:?} ({b2} B)" + ); + } + } +} diff --git a/distributed_game/src/schema.rs b/distributed_game/src/schema.rs index 86cc28d7b..1702b05c9 100644 --- a/distributed_game/src/schema.rs +++ b/distributed_game/src/schema.rs @@ -357,6 +357,9 @@ pub struct KindDef { pub fields: Vec, /// Server-side priority multiplier (not sent on the wire). pub priority: f32, + /// Exempt from distance-based update rates + /// ([`UpdateRates`](crate::UpdateRates); server-side, not on the wire). + pub every_patch: bool, } impl KindDef { @@ -367,6 +370,7 @@ impl KindDef { name: name.into(), fields: Vec::new(), priority: 1.0, + every_patch: false, } } @@ -407,6 +411,12 @@ impl KindDef { self.priority = p; self } + + /// Update every patch at any distance (no distance-based rates). + pub fn every_patch(mut self) -> Self { + self.every_patch = true; + self + } } /// All replicated kinds of a zone type. diff --git a/distributed_game/tests/authoring/main.rs b/distributed_game/tests/authoring/main.rs index ac5e38bdf..5345ec6f6 100644 --- a/distributed_game/tests/authoring/main.rs +++ b/distributed_game/tests/authoring/main.rs @@ -64,7 +64,7 @@ pub struct MarkerView { } #[derive(Debug, Clone, PartialEq, Replicated)] -#[replicated(kind = "stats", id = 4, global)] +#[replicated(kind = "stats", id = 4, global, every_patch)] pub struct StatsView { pub slow_ms: u16, pub tick_ms: f32, @@ -363,6 +363,7 @@ fn replicated_kind_def_and_values_share_one_field_order() { (kind.id, kind.name.as_str(), kind.priority), (1, "player", 2.0) ); + assert!(!kind.every_patch && StatsView::kind_def().every_patch); let names: Vec<_> = kind.fields.iter().map(|f| f.name.as_str()).collect(); assert_eq!( names, diff --git a/distributed_game/tests/replication/main.rs b/distributed_game/tests/replication/main.rs index 3b51285d3..fdcb2d7c7 100644 --- a/distributed_game/tests/replication/main.rs +++ b/distributed_game/tests/replication/main.rs @@ -1,13 +1,13 @@ //! P3 replication: interest enter/leave with hysteresis, global entities, -//! change detection and byte-budget fairness. +//! change detection, byte-budget fairness and distance-based update rates. use std::collections::BTreeMap; use distributed_game::protocol::{Delta, ServerMessage}; use distributed_game::replication::{ClientReplica, EncodedWorld}; use distributed_game::{ - ClientMirror, EntitySchema, EntityState, FieldType, FieldValue, InterestConfig, KindDef, View, - WorldView, + ClientMirror, EntitySchema, EntityState, FieldType, FieldValue, InterestConfig, KindDef, + UpdateRates, View, WorldView, }; fn schema() -> EntitySchema { @@ -16,12 +16,20 @@ fn schema() -> EntitySchema { max: 1000.0, bits: 20, }; - EntitySchema::new().kind( - KindDef::new(1, "thing") - .field("x", pos) - .field("y", pos) - .field("hp", FieldType::U16), - ) + EntitySchema::new() + .kind( + KindDef::new(1, "thing") + .field("x", pos) + .field("y", pos) + .field("hp", FieldType::U16), + ) + .kind( + KindDef::new(2, "beacon") + .field("x", pos) + .field("y", pos) + .field("hp", FieldType::U16) + .every_patch(), + ) } fn thing(id: u32, x: f32, y: f32, hp: u32) -> EntityState { @@ -39,10 +47,16 @@ struct Client { mirror: ClientMirror, view: View, budget: usize, + interest: InterestConfig, + world: Option, } impl Client { fn new(view: View, budget: usize) -> Self { + Self::with_rates(view, budget, UpdateRates::every_patch()) + } + + fn with_rates(view: View, budget: usize, update_rates: UpdateRates) -> Self { // Seed the mirror with the schema as a Welcome would. let mut mirror = ClientMirror::new(); mirror.welcome = Some(distributed_game::protocol::Welcome { @@ -62,6 +76,12 @@ impl Client { mirror, view, budget, + interest: InterestConfig { + cell_size: 8.0, + leave_margin: 0.1, + update_rates, + }, + world: None, } } @@ -70,20 +90,20 @@ impl Client { for e in entities { view.push(e); } - let interest = InterestConfig { - cell_size: 8.0, - leave_margin: 0.1, + let world = match &self.world { + Some(last) => last.next(&schema(), view, &self.interest), + None => EncodedWorld::build(&schema(), view, &self.interest), }; - let world = EncodedWorld::build(&schema(), view, &interest); let (frame, _) = self.replica.build_delta( &schema(), &world, - &interest, + &self.interest, &self.view, self.budget, tick, 0, ); + self.world = Some(world); match self.mirror.apply(&frame).unwrap() { ServerMessage::Delta(d) => d, other => panic!("{other:?}"), @@ -176,3 +196,80 @@ fn byte_budget_prioritizes_but_never_starves() { // Mirror converges to the server state for everything sent recently. assert_eq!(c.mirror.entities.len(), n as usize); } + +fn hp(c: &Client, id: u32) -> u32 { + match c.mirror.entities[&id].fields[2] { + FieldValue::U(hp) => hp, + ref other => panic!("{other:?}"), + } +} + +#[test] +fn far_updates_thin_out_and_converge_within_their_interval() { + // Defaults: within 50 every patch, within 80 every 2nd, beyond every 4th. + let mut c = Client::with_rates(View::new(0.0, 0.0, 100.0), 1 << 20, UpdateRates::default()); + let world = |hp: u32| { + vec![ + thing(1, 10.0, 0.0, hp), + thing(2, 65.0, 0.0, hp), + thing(3, 95.0, 0.0, hp), + thing(4, 0.0, 95.0, hp).global(), + EntityState::new( + 5, + 2, + -95.0, + 0.0, + vec![FieldValue::F(-95.0), FieldValue::F(0.0), FieldValue::U(hp)], + ), + ] + }; + let d = c.patch(world(0), 0); + assert_eq!(d.creates.len(), 5, "creates are never held"); + + let mut updates: BTreeMap = BTreeMap::new(); + for tick in 1..=40 { + let d = c.patch(world(tick), tick); + for u in &d.updates { + *updates.entry(u.id).or_insert(0) += 1; + } + // Every entity shows a state at most its interval - 1 patches old. + for (id, every) in [(1, 1), (2, 2), (3, 4), (4, 1), (5, 1)] { + assert!(hp(&c, id) + every > tick, "entity {id} at tick {tick}"); + } + } + assert_eq!(updates[&1], 40, "near: every patch"); + assert_eq!(updates[&2], 20, "mid: every 2nd"); + assert_eq!(updates[&3], 10, "far: every 4th"); + assert_eq!(updates[&4], 40, "global: never held"); + assert_eq!(updates[&5], 40, "every_patch kind: never held"); + + // The last change arrives within 4 patches, then nothing more is sent. + for tick in 41..=44 { + c.patch(world(40), tick); + } + assert!((1..=5).all(|id| hp(&c, id) == 40)); + let d = c.patch(world(40), 45); + assert!(d.updates.is_empty()); +} + +#[test] +fn update_rates_never_hold_entering_spawning_or_leaving_entities() { + let mut c = Client::with_rates(View::new(0.0, 0.0, 100.0), 1 << 20, UpdateRates::default()); + c.patch(vec![thing(1, 95.0, 0.0, 1)], 0); + // A far spawn is created at once, whatever the patch. + for tick in 1..=4 { + let d = c.patch( + vec![thing(1, 95.0, 0.0, 1), thing(10 + tick, 0.0, 95.0, 1)], + tick, + ); + assert_eq!( + d.creates.iter().map(|e| e.id).collect::>(), + vec![10 + tick] + ); + assert_eq!(d.removes, if tick == 1 { vec![] } else { vec![9 + tick] }); + } + // Leaving the view (beyond 110) and despawning are removed at once. + let d = c.patch(vec![thing(1, 111.0, 0.0, 2)], 5); + assert_eq!(d.removes, vec![1, 14]); + assert!(c.mirror.entities.is_empty()); +} diff --git a/distributed_game_macros/src/lib.rs b/distributed_game_macros/src/lib.rs index 9779a4d12..ae7f2134f 100644 --- a/distributed_game_macros/src/lib.rs +++ b/distributed_game_macros/src/lib.rs @@ -45,11 +45,12 @@ fn finish(result: syn::Result) -> TokenStream { } /// One replicated entity kind: `#[replicated(kind = "mob", id = 2, -/// priority = 2.0, global, x = field, y = field, of = Entity)]` on a struct -/// with named fields (Rust type → wire type: `bool`, `u8`, `u16`, `u32`, -/// `i32`, `f32`, `String` → short_str, `Vec` → bytes). The interest -/// position is the `x`/`y` fields unless overridden; a kind without a -/// position must be `global`. +/// priority = 2.0, global, every_patch, x = field, y = field, of = +/// Entity)]` on a struct with named fields (Rust type → wire type: `bool`, +/// `u8`, `u16`, `u32`, `i32`, `f32`, `String` → short_str, `Vec` → +/// bytes). The interest position is the `x`/`y` fields unless overridden; +/// a kind without a position must be `global`. `every_patch` exempts the +/// kind from distance-based update rates (`distributed_game::UpdateRates`). /// /// `of = Entity` makes the struct a view projected from a zone entity /// (`distributed_game::entity`): fields marked `#[from]` (same name) or diff --git a/distributed_game_macros/src/replicated.rs b/distributed_game_macros/src/replicated.rs index 3d7d724cb..43b959498 100644 --- a/distributed_game_macros/src/replicated.rs +++ b/distributed_game_macros/src/replicated.rs @@ -18,6 +18,7 @@ struct KindAttrs { id: u16, priority: Option, global: bool, + every_patch: bool, x: Option, y: Option, of: Option, @@ -30,8 +31,8 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { "#[derive(Replicated)] needs #[replicated(kind = \"name\", id = N)]", )); }; - let (mut kind, mut id, mut priority, mut global, mut x, mut y, mut of) = - (None, None, None, false, None, None, None); + let (mut kind, mut id, mut priority, mut global, mut every_patch, mut x, mut y, mut of) = + (None, None, None, false, false, None, None, None); attr.parse_nested_meta(|meta| { let key = meta .path @@ -46,12 +47,13 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { } "priority" => priority = Some(meta.value()?.parse::()?), "global" => global = true, + "every_patch" => every_patch = true, "x" => x = Some(meta.value()?.parse::()?), "y" => y = Some(meta.value()?.parse::()?), "of" => of = Some(meta.value()?.parse::()?), _ => { return Err(meta.error( - "expected `kind`, `id`, `priority`, `global`, `x`, `y` or `of` in #[replicated(..)]", + "expected `kind`, `id`, `priority`, `global`, `every_patch`, `x`, `y` or `of` in #[replicated(..)]", )) } } @@ -66,6 +68,7 @@ fn parse_kind_attrs(input: &DeriveInput) -> syn::Result { })?, priority, global, + every_patch, x, y, of, @@ -275,6 +278,11 @@ pub(crate) fn expand(input: DeriveInput) -> syn::Result { .as_ref() .map(|e| quote!(.priority((#e) as f32))) .unwrap_or_default(); + let every_patch = if attrs.every_patch { + quote!(.every_patch()) + } else { + TokenStream::new() + }; Ok(quote! { impl #rt::Replicated for #ident { const KIND_ID: u16 = #id; @@ -285,6 +293,7 @@ pub(crate) fn expand(input: DeriveInput) -> syn::Result { #p::KindDef::new(#id, #kind) #(#defs)* #kind_priority + #every_patch } #[allow(clippy::unnecessary_cast)] From e4af1940fac1e613128ab6873c81d619cb4fd0b7 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 03:33:18 -0500 Subject: [PATCH 04/22] fix(game): batch facts only for sinks that settle batches A sink that commits a batch one by one delivered no settlement until the whole batch was done (16 facts behind slow storage held back every settlement). FactSink::max_batch (default 1) and FactHost::max_fact_batch let a sink declare batching; the celld object host does, other sinks get one fact at a time as before. [[zone-fact-batches]] --- distributed_game/README.md | 14 ++++---- distributed_game/src/celld_zone/object.rs | 10 ++++++ distributed_game/src/command_sink.rs | 11 ++++++ distributed_game/src/facts.rs | 43 ++++++++++++++++------- 4 files changed, 60 insertions(+), 18 deletions(-) diff --git a/distributed_game/README.md b/distributed_game/README.md index 0b9f4cbe6..9941d8f11 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -1163,13 +1163,15 @@ a time (a slow commit delays every later fact). Set without a principal) stay sequential in emission order; different principals commit concurrently; settlements arrive in completion order. -**Batches.** The feed hands its sink up to `max_batch` (default 16, +**Batches.** A sink that settles many facts in one round trip says so +(`FactSink::max_batch`, `FactHost::max_fact_batch`; the celld object host +does) and the feed then hands it up to `max_batch` (default 16, `FactFeed::with_max_batch`; celld zones: `ZONE_FACT_BATCH`) waiting facts -of a lane at once, `FactSink::commit_batch`. Game code is unchanged -(`ctx.emit`, `#[derive(FactCommand)]`, `#[command]` handlers). The default -commits them one by one; `CommandFactSink` groups consecutive facts for -the same aggregate and calls `FactHost::dispatch_fact_batch`, which the -celld object host sends as one `POST {kind}/{shard}/__batch` (each entry +of a lane at once, `FactSink::commit_batch`. Other sinks keep getting one +fact at a time. Game code is unchanged (`ctx.emit`, +`#[derive(FactCommand)]`, `#[command]` handlers). `CommandFactSink` +groups consecutive facts for the same aggregate and calls +`FactHost::dispatch_fact_batch`, which the celld object host sends as one `POST {kind}/{shard}/__batch` (each entry with its own command id, principal and stamp; `serve_cell` runs them in order through the same idempotent wait path and drains the cell outbox once). A batch stops at its first retryable failure and is retried from diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index a60eef4e6..32147631a 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -413,6 +413,12 @@ impl FactHost for CelldObjectHost { Box::pin(SendFuture::new(async move { self.post(&call).await })) } + /// Any number: one `__batch` request per cell (the feed's + /// `ZONE_FACT_BATCH` bounds it). + fn max_fact_batch(&self) -> usize { + usize::MAX + } + fn dispatch_fact_batch<'a>( &'a self, calls: Vec>, @@ -465,6 +471,10 @@ impl FactSink for DurableFacts { })) } + fn max_batch(&self) -> usize { + self.sink.max_batch() + } + /// One batched delete for the facts the batch settled. fn commit_batch<'a>( &'a self, diff --git a/distributed_game/src/command_sink.rs b/distributed_game/src/command_sink.rs index 759353ffb..5b720f8a7 100644 --- a/distributed_game/src/command_sink.rs +++ b/distributed_game/src/command_sink.rs @@ -153,6 +153,13 @@ pub trait FactHost: Send + Sync + 'static { call: FactCall<'a>, ) -> BoxFuture<'a, Result>; + /// Facts this host runs per [`dispatch_fact_batch`](Self::dispatch_fact_batch) + /// at most (default 1). A host with a batch route returns more, and + /// the sink's [`FactSink::max_batch`] follows it. + fn max_fact_batch(&self) -> usize { + 1 + } + /// Run `calls` (consecutive facts for one aggregate: same kind and /// shard) in order, under [`FactSink::commit_batch`]'s contract: the /// results end at the first retryable failure. The default runs them @@ -320,6 +327,10 @@ impl FactSink for CommandFactSink { Box::pin(async move { self.host.dispatch_fact(self.call_for(fact)?).await }) } + fn max_batch(&self) -> usize { + self.host.max_fact_batch() + } + /// Consecutive facts for the same aggregate go to the host together /// ([`FactHost::dispatch_fact_batch`]). fn commit_batch<'a>( diff --git a/distributed_game/src/facts.rs b/distributed_game/src/facts.rs index 40eb74aa8..a82df4c9d 100644 --- a/distributed_game/src/facts.rs +++ b/distributed_game/src/facts.rs @@ -29,14 +29,15 @@ //! refuses duplicates), so per-key order always holds. Settlements arrive //! in completion order. //! -//! Batching: the feed hands a sink up to -//! [`max_batch`](FactFeed::with_max_batch) (default -//! [`DEFAULT_MAX_BATCH`]) consecutive facts of one lane at once -//! ([`FactSink::commit_batch`]). A sink that can settle many facts in -//! one round trip (the celld cell hosts: one `__batch` request per cell) -//! overrides it; the default commits them one by one. A batch commits in -//! order and stops at its first retryable failure, which is retried from -//! there, so order and exactly-once hold as for single facts. +//! Batching: a sink that settles many facts in one round trip (the +//! celld object host: one `__batch` request per cell) says so with +//! [`FactSink::max_batch`]; the feed then hands it up to that many (and +//! at most [`FactFeed::with_max_batch`], default [`DEFAULT_MAX_BATCH`]) +//! consecutive facts of one lane at once ([`FactSink::commit_batch`]). +//! Other sinks keep getting one fact at a time, so their settlements flow +//! back per fact. A batch commits in order and stops at its first +//! retryable failure, which is retried from there, so order and +//! exactly-once hold as for single facts. use std::collections::{HashSet, VecDeque}; use std::future::Future; @@ -246,6 +247,14 @@ pub trait FactSink: Send + Sync + 'static { /// Commit one fact. Must be idempotent per [`Fact::key`]. fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result>; + /// Facts this sink settles per [`commit_batch`](Self::commit_batch) + /// round trip at most (default 1: one fact at a time, each settlement + /// delivered as soon as it commits). Override when a batch is cheaper + /// than its facts one by one. + fn max_batch(&self) -> usize { + 1 + } + /// Commit `facts` in order: one result per fact attempted. The /// results end at the first [`FactError::Retryable`] (included), or /// earlier; facts past the end were not attempted and are retried. @@ -542,8 +551,9 @@ impl FactFeed { self.max_concurrent } - /// Hand the sink up to `n` facts of a lane at once (default - /// [`DEFAULT_MAX_BATCH`]; 1 commits fact by fact). + /// Hand a batching sink ([`FactSink::max_batch`] > 1) up to `n` facts + /// of a lane at once (default [`DEFAULT_MAX_BATCH`]; 1 commits fact by + /// fact). pub fn with_max_batch(mut self, n: usize) -> Self { self.max_batch = n.max(1); self @@ -565,8 +575,9 @@ impl FactFeed { Fut: Future, { let policy = self.policy; + let max_batch = self.max_batch.min(sink.max_batch()).max(1); if self.max_concurrent <= 1 { - while let Some(facts) = NextFact(&self.shared, self.max_batch).await { + while let Some(facts) = NextFact(&self.shared, max_batch).await { for settlement in commit_with_retries(&sink, facts, policy, &sleep).await { let _ = self.settlements.send(settlement); } @@ -578,7 +589,7 @@ impl FactFeed { shared: &self.shared, settlements: &self.settlements, limit: self.max_concurrent, - max_batch: self.max_batch, + max_batch, start, waiting: VecDeque::new(), active: Vec::new(), @@ -780,6 +791,10 @@ impl FactSink for Arc { (**self).commit(fact) } + fn max_batch(&self) -> usize { + (**self).max_batch() + } + fn commit_batch<'a>( &'a self, facts: &'a [Fact], @@ -806,6 +821,10 @@ mod batch_tests { unreachable!("batches only") } + fn max_batch(&self) -> usize { + usize::MAX + } + fn commit_batch<'a>( &'a self, facts: &'a [Fact], From 2053dc545ff3ae0e3bf73b0586e9f553113a5b00 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 03:51:29 -0500 Subject: [PATCH 05/22] docs(cli): distributed-game skill for zones, facts and watches Agents get one skill for the game layer: plain Rust zones, facts settled into cells (batched on celld), interest-managed replication with update rates, and zones watching world cells as pushed read models. [[world-scale-game-services]] --- .../skills/distributed-game/SKILL.md | 136 ++++++++++++++++++ distributed_cli/src/skills.rs | 9 +- distributed_cli/tests/cli_skills_init.rs | 2 +- 3 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 distributed_cli/skills/distributed-game/SKILL.md diff --git a/distributed_cli/skills/distributed-game/SKILL.md b/distributed_cli/skills/distributed-game/SKILL.md new file mode 100644 index 000000000..46eb43d4e --- /dev/null +++ b/distributed_cli/skills/distributed-game/SKILL.md @@ -0,0 +1,136 @@ +--- +name: distributed-game +description: Build authoritative game zones with distributed_game - a plain Rust Zone for the simulation, replicated entities with interest management, durable facts settled into cells, and zones that watch world cells as pushed read models. Use when writing or scaling a game zone, its facts, its replication, or its world cells (native host or celld). +--- + +# Game zones with distributed_game + +A zone is a plain Rust struct that owns one authoritative simulation +(movement, physics, combat ticks). The framework hosts it, replicates it, +settles its outcomes and keeps it informed about the world. Game code holds +no storage, transport, retry or celld code. `distributed_game/README.md` is +the full reference; this skill is the shortest path to the common cases. + +## The three rules + +1. **Ticks are never stored.** Only meaningful outcomes leave the zone, as + *facts* (`ctx.emit`), and the tick never waits for them. +2. **Identity is bound by the host.** A fact's principal comes from the + player's verified ticket. Inputs never carry owner identity, XP amounts + or rewards chosen by a client. +3. **Read models are for display.** Watched world state flavours the zone + (weather, a boss tier, an event); rewards still go through the fact → + command path and are validated by the cell's handler. + +## Facts: outcomes into cells + +```rust +#[derive(Serialize, Deserialize, FactCommand)] +#[fact(command = RECORD_KILL)] // the same constant the cell mounts +pub struct RecordKillInput { pub mob: u32 } + +// in a zone hook: never blocks, returns the key +ctx.emit(player, RecordKillInput { mob: id })?; // principal bound from the ticket +ctx.emit_zone(RoomAwakeInput { .. })?; // zone/system fact, no principal +``` + +Route facts once, where the host is built (celld: `CelldZone::fact_routes`): + +```rust +sink.route(FactRoute::by_principal::("crawler")) // crawler/{principal} + .route(FactRoute::by_input_field::("borough", "borough")) +``` + +You get: +- **Exactly once.** The key (zone + boot epoch + parts) becomes the command + id, and the cell's ledger turns a retry into a replay. +- **Order per principal.** Facts of one player commit in emission order; + different players overlap (`max_concurrent_commits`). +- **Backpressure, not loss.** Past `capacity` unsettled facts, `emit` + returns `Backpressure` and the game decides what to do. +- **Batches by default on celld.** A lane's waiting facts for one cell go + as one `POST …/__batch` (`ZONE_FACT_BATCH`, default 16), and the zone + outbox clears them with one delete. A batch stops at its first retryable + failure and resumes from there, so order and exactly-once hold. A custom + sink opts in by overriding `FactSink::max_batch` and `commit_batch`, or + `FactHost::max_fact_batch` and `dispatch_fact_batch`. Otherwise it gets + one fact at a time. + +Results come back on a later tick in `on_fact_settled`. + +## Replication: what each client sees + +```rust +// one #[replicated] per view struct (Mob Pit: examples/game/server/rooms/src/zone.rs) +#[replicated(kind = "mob", id = 2, of = Mob)] // interest-managed +#[replicated(kind = "pillar", id = 3, global, of = Pillar)] // everyone, always +#[replicated(kind = "hero", id = 4, every_patch, of = Hero)] // never rate-thinned +``` + +- Clients see entities within their view radius, under a per-Delta byte + budget (4 KiB) ranked by priority. Deferred entities gain priority until + they are sent. +- Far entities update less often (`HostConfig::interest.update_rates`): + - within 0.5 of the radius: every patch; + - up to 0.8: every 2nd patch; + - beyond that: every 4th. + + Creates and removes are never held. `UpdateRates::every_patch()` turns + the rates off. +- Each changed entity is encoded once per patch and shared by every client + that was up to date. A dense room costs selection plus copying, not one + encode per client. +- The wire format and the JS SDK's prediction are unchanged by these + knobs. + +## The world around: watched cells (CQRS) + +World cells (hood, borough, city, content rollout, dev settings) are the +write model. Each zone keeps a pushed read model of the cells it declares, +and each view decodes into a plain struct that becomes a normal zone +message: + +```rust +impl CelldZone for MobPit { + fn watch(zone: &str) -> Watches { + Watches::new() + .cell(HOOD_KIND, &hood_of(zone), |read: HoodRead| read.state.map(PitMessage::hood)) + .cell(CITY_KIND, CITY_SHARD, |read: CityRead| read.state.map(PitMessage::city)) + } +} +``` + +A cell opts in with one field and two calls: + +```rust +pub struct HoodCell { cell: AggregateCell, env: Env, watchers: CellWatchers } +// fetch: self.watchers.serve(&self.env, &self.cell, req, read_hood, sessions).await +// alarm: self.watchers.alarm(&self.env, &self.cell, read_hood).await +``` + +How it behaves: +- **Reads.** A zone reads each watched cell on wake. The read leaves an + in-memory lease on the cell; nothing is written. +- **Pushes.** The cell pushes `{topic, version, view}` after a commit or + alarm that changes the view. A heartbeat that only bumps the version + pushes nothing. +- **Ordering.** The zone applies only newer versions. +- **Repair.** The zone re-reads every 45 ± 15 s, which renews its lease + and repairs a lost push. +- **Sleeping zones.** A sleeping zone is never woken by a push. +- **Scale.** Cost grows with view changes × watchers, not with zones × + seconds. +- **Other reads.** `refresh`/`refresh_every` remain for async reads that + are not cell views. + +## Checking your work + +- `cargo test -p distributed_game --all-features` runs the native host, the + facts, replication and interest management with in-memory sinks. +- `mobpit-load` (in `examples/game/server/examples`) measures: + - `--steps 32,64` for one room; + - `--steps 64x1,128x1` for many rooms; + - `--no-attack` for movement only; + - `--var ZONE_FACT_BATCH=1` to A/B fact batching. + + Compare runs only on a quiet machine. diff --git a/distributed_cli/src/skills.rs b/distributed_cli/src/skills.rs index 5930c5d9d..3ea07cd22 100644 --- a/distributed_cli/src/skills.rs +++ b/distributed_cli/src/skills.rs @@ -40,7 +40,7 @@ pub struct EmbeddedSkill { pub files: &'static [EmbeddedFile], } -static SKILLS: [EmbeddedSkill; 4] = [ +static SKILLS: [EmbeddedSkill; 5] = [ EmbeddedSkill { name: "distributed-usage", description: "Build Distributed CQRS/event-sourced Rust services where you mostly write models and handlers while the framework and distributed generate persistence, transports, manifests, and deploy wiring. Use model-first TDD to specify plain aggregate behavior with fast unit tests before implementing models and thin handlers. Use when designing, testing, writing, or modifying a Distributed service or domain model.", @@ -72,6 +72,13 @@ static SKILLS: [EmbeddedSkill; 4] = [ relative_path: "SKILL.md", contents: include_str!("../skills/distributed-graphql/SKILL.md"), }], + }, EmbeddedSkill { + name: "distributed-game", + description: "Build authoritative game zones with distributed_game - a plain Rust Zone for the simulation, replicated entities with interest management, durable facts settled into cells, and zones that watch world cells as pushed read models. Use when writing or scaling a game zone, its facts, its replication, or its world cells (native host or celld).", + files: &[EmbeddedFile { + relative_path: "SKILL.md", + contents: include_str!("../skills/distributed-game/SKILL.md"), + }], }, ]; diff --git a/distributed_cli/tests/cli_skills_init.rs b/distributed_cli/tests/cli_skills_init.rs index 86c7d305e..3b8b8efb5 100644 --- a/distributed_cli/tests/cli_skills_init.rs +++ b/distributed_cli/tests/cli_skills_init.rs @@ -78,7 +78,7 @@ fn init_bootstraps_a_fresh_project_and_reruns_are_noops() { let agents_md = read(&dir, "AGENTS.md"); assert!(agents_md.contains(BEGIN) && agents_md.contains(END)); assert!(stdout.contains("created .distributed/skills/distributed-ci/SKILL.md")); - assert!(stdout.contains("Initialized 4 skills at .distributed/skills (wired: claude, agents)")); + assert!(stdout.contains("Initialized 5 skills at .distributed/skills (wired: claude, agents)")); // No strays at the harness skill roots (only skill folders). for root in [".agents/skills", ".claude/skills"] { From 36054d8ee2db58e96fff9853208ea9bc64118fbb Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 11:13:41 -0500 Subject: [PATCH 06/22] feat(game): neighbours follow the player's view (observer Focus) A watched neighbour room gave each observer the whole room from its centre, so interest management stopped at the border. Clients now send Focus {x, y} (tag 0x07, observers only) with the player's position in the neighbour's coordinates; Zone::on_observer_focus recentres that observer's view. Mob Pit applies VIEW_RADIUS, so interest and update rates continue across rooms. JS: ZoneWatch.setFocus, throttled and resent after reconnect. [[observer-focus-neighbour-interest]] --- .../skills/distributed-game/SKILL.md | 4 ++ distributed_game/PROTOCOL.md | 12 ++++ distributed_game/README.md | 7 ++ distributed_game/src/celld_zone/object.rs | 6 ++ distributed_game/src/host.rs | 16 +++++ distributed_game/src/net/mod.rs | 10 +++ distributed_game/src/protocol/messages.rs | 20 ++++++ distributed_game/src/zone.rs | 16 +++++ distributed_game/tests/fixtures/messages.json | 10 +++ distributed_game/tests/golden/main.rs | 2 + examples/game/client/src/mobpit/client.ts | 14 +++- examples/game/client/tests/helpers/mobpit.mjs | 46 ++++++++++++- .../tests/mobpit-neighbours-headless.test.mjs | 45 +++++++++++-- .../tests/mobpit-wake-headless.test.mjs | 47 +------------ examples/game/server/rooms/src/zone.rs | 16 ++++- examples/game/server/rooms/tests/rooms.rs | 66 +++++++++++++++++++ examples/game/sveltekit/src/lib/pit/loop.ts | 13 +++- js/src/game/client.ts | 10 +++ js/src/game/connection.ts | 34 ++++++++++ js/src/game/index.ts | 2 + js/src/game/messages.ts | 12 +++- js/src/game/watch.ts | 17 +++++ js/tests/game-codec.test.mjs | 6 +- js/tests/game-connection.test.mjs | 38 +++++++++++ 24 files changed, 412 insertions(+), 57 deletions(-) diff --git a/distributed_cli/skills/distributed-game/SKILL.md b/distributed_cli/skills/distributed-game/SKILL.md index 46eb43d4e..6d606ae9c 100644 --- a/distributed_cli/skills/distributed-game/SKILL.md +++ b/distributed_cli/skills/distributed-game/SKILL.md @@ -77,6 +77,10 @@ Results come back on a later tick in `on_fact_settled`. Creates and removes are never held. `UpdateRates::every_patch()` turns the rates off. +- A client watching the zone next door tells it where the player is + (`watchZones(...).setFocus(at)`, zone-local coordinates). The + neighbour's `Zone::on_observer_focus` then centres that observer's view + on the player, so interest continues across the border. - Each changed entity is encoded once per patch and shared by every client that was up to date. A dense room costs selection plus copying, not one encode per client. diff --git a/distributed_game/PROTOCOL.md b/distributed_game/PROTOCOL.md index f9a4ef703..b9c76fb94 100644 --- a/distributed_game/PROTOCOL.md +++ b/distributed_game/PROTOCOL.md @@ -196,6 +196,7 @@ replication priorities are not transmitted. | `0x04` | C→S | Leave | | `0x05` | C→S | Content | | `0x06` | C→S | Near | +| `0x07` | C→S | Focus | | `0x80` | S→C | Welcome | | `0x81` | S→C | Delta | | `0x82` | S→C | Event | @@ -221,6 +222,7 @@ Ping := 0x03, f64 client_time Leave := 0x04 Content := 0x05, string content_version Near := 0x06, bool near +Focus := 0x07, f32 x, f32 y ``` - **Hello** MUST be the first message, sent within the server's hello @@ -261,6 +263,13 @@ Near := 0x06, bool near `Error(WAKE_DENIED)` (the connection stays open). Servers that keep observed zones running anyway (the native `net` server) ignore it; players' are ignored. +- **Focus**: where an observer is looking, in the zone's local + coordinates (e.g. its player next door, seen from this zone). The zone + decides what it means; by default the observer's view centre moves + there and keeps its radius. It only steers what that observer is sent. + Clients SHOULD throttle it (the JS SDK: at most every 200 ms, after a + move of 0.25 or more) and send it again after each Welcome. Players' + are ignored; non-finite values are ignored. Worked example — Hello (`version 1, content "c1", caps 5, ticket "t.k.n"`): @@ -601,6 +610,9 @@ error codes and treat them as "session over" / "request failed"): - Error code `10` ASLEEP. - Error code `11` WAKE_DENIED. - `0x06 Near` (client → server) and the ticket claim `wake`. +- `0x07 Focus` (client → server). A server built before it answers + `Error(BAD_MESSAGE)` (the native server keeps the connection; a celld + zone closes it), so deploy servers before clients that send it. Not additive (made before the first release, so the version stays 1): Hello's trailing `bool near` — clients built before it send one byte diff --git a/distributed_game/README.md b/distributed_game/README.md index 9941d8f11..71cd9b877 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -414,6 +414,13 @@ client already has: the client when they leave it. A spatial grid makes this cheap. Mark entities that everyone must always see (players in a small room, a boss, pillars) with `.global()`. +- **Across borders.** A client watching the zone next door (an observer) + tells it where the player is in that zone's coordinates (`Focus`; JS: + `watchZones(...).setFocus(at)`, throttled for you). By default the + observer's view centre follows (`Zone::on_observer_focus`; override it + to resize or clamp, as Mob Pit does with the player's own radius), so + interest and update rates carry on past the wall instead of the + neighbour sending all it has. - **Bandwidth budget.** Each client has `byte_budget` bytes per Delta (4 KiB). Creates and updates compete by priority (kind × entity × field × nearness). Anything deferred builds up priority, so it is always diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index 32147631a..14158850d 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -1109,6 +1109,12 @@ impl ZoneObject { self.keep_awake().await; } } + (Some(client), ClientMessage::Focus { x, y }) => { + let mut inner = self.inner.borrow_mut(); + if let Some(host) = inner.host.as_mut() { + host.observer_focus(ClientId(client), x, y); + } + } (Some(client), ClientMessage::Content { content_version }) => { let mut inner = self.inner.borrow_mut(); if let Some(host) = inner.host.as_mut() { diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index c2b78c891..9acbb921c 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -712,6 +712,14 @@ impl Ctx<'_, Z> { self.broadcast_event(event.event_id(), event.to_payload()); } + /// A viewer's area of interest (None for an unknown viewer). + pub fn view_of(&self, viewer: impl Into) -> Option { + match viewer.into() { + Viewer::Player(p) => self.core.players.get(&p).map(|slot| slot.view), + Viewer::Observer(o) => self.core.observers.get(&o).map(|slot| slot.view), + } + } + /// Set a viewer's area of interest. pub fn set_view(&mut self, viewer: impl Into, view: View) { match viewer.into() { @@ -1473,6 +1481,14 @@ impl ZoneHost { } } + /// An observer's `Focus` (zone-local): [`Zone::on_observer_focus`]. + /// Ignored for players and unknown clients. + pub fn observer_focus(&mut self, client: ClientId, x: f32, y: f32) { + if let Some(Viewer::Observer(observer)) = self.viewer_of(client) { + hook!(self.on_observer_focus(observer, x, y)); + } + } + /// Content version a player's client reported. pub fn player_content_version(&self, player: PlayerId) -> Option<&str> { self.core diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index e7fab362c..3a9a1a545 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -271,6 +271,12 @@ enum ZoneCmd { client: ClientId, version: String, }, + /// An observer's `Focus`. + Focus { + client: ClientId, + x: f32, + y: f32, + }, /// A zone inbox message (decoded into `Z::Message` on the zone task). Deliver { inbound: inbox::WireInbound, @@ -1452,6 +1458,7 @@ fn handle_cmd( ZoneCmd::Content { client, version } => { host.content_patched(client, &version); } + ZoneCmd::Focus { client, x, y } => host.observer_focus(client, x, y), ZoneCmd::ChildEvent(event) => host.deliver_child_event(event), ZoneCmd::Deliver { inbound, reply } => { let _ = reply.send(inbound.decode::().map(|m| host.deliver(m))); @@ -1634,6 +1641,7 @@ enum Inbound { }, Ping(f64), Content(String), + Focus(f32, f32), } async fn connection(inner: Arc>, zone_id: String, mut socket: WebSocket) @@ -1895,6 +1903,7 @@ where // Native zones keep observers' zones running already; // nearness only matters to celld zones. Ok(ClientMessage::Near { .. }) => continue, + Ok(ClientMessage::Focus { x, y }) => Inbound::Focus(x, y), Ok(ClientMessage::Leave) => { graceful = true; break; } Ok(ClientMessage::Hello(_)) => { let _ = socket.send(Message::Binary(error_frame(error::BAD_MESSAGE, "already joined").into())).await; @@ -1993,6 +2002,7 @@ async fn forward_in( .send(ZoneCmd::Content { client, version }) .await .is_ok(), + Inbound::Focus(x, y) => zone.send(ZoneCmd::Focus { client, x, y }).await.is_ok(), Inbound::Ping(client_time) => { let pong = ServerMessage::Pong { client_time, diff --git a/distributed_game/src/protocol/messages.rs b/distributed_game/src/protocol/messages.rs index d560788ea..3def9af5f 100644 --- a/distributed_game/src/protocol/messages.rs +++ b/distributed_game/src/protocol/messages.rs @@ -23,6 +23,8 @@ pub mod tag { /// Client → server Near (an observer asks to keep the zone awake, or /// lets go). pub const NEAR: u8 = 0x06; + /// Client → server Focus (where an observer is looking, zone-local). + pub const FOCUS: u8 = 0x07; /// Server → client Welcome. pub const WELCOME: u8 = 0x80; /// Server → client Delta. @@ -149,6 +151,17 @@ pub enum ClientMessage { /// Near (true) or passive (false). near: bool, }, + /// Where an observer is looking, in the zone's local coordinates (a + /// player next door: its position seen from this zone). The zone + /// decides what it means ([`Zone::on_observer_focus`](crate::Zone::on_observer_focus); + /// by default the observer's view centre follows). Ignored for + /// players. + Focus { + /// x (zone-local). + x: f32, + /// y (zone-local). + y: f32, + }, } impl ClientMessage { @@ -187,6 +200,9 @@ impl ClientMessage { Self::Near { near } => { w.u8(tag::NEAR).bool(*near); } + Self::Focus { x, y } => { + w.u8(tag::FOCUS).f32(*x).f32(*y); + } } w.into_bytes() } @@ -216,6 +232,10 @@ impl ClientMessage { content_version: r.string()?, }, tag::NEAR => Self::Near { near: r.bool()? }, + tag::FOCUS => Self::Focus { + x: r.f32()?, + y: r.f32()?, + }, other => return Err(DecodeError::UnknownMessage(other)), }; r.finish()?; diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index 56d45541e..69bda36b3 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -13,6 +13,7 @@ use crate::claims::TicketExtensions; use crate::facts::FactSettlement; use crate::host::Ctx; +use crate::interest::View; use crate::protocol::Role; use crate::replication::WorldView; use crate::rewind::RewindFrame; @@ -281,6 +282,21 @@ pub trait Zone: Sized + Send + 'static { /// returns it during this hook (it can no longer receive events). fn on_unobserve(&mut self, _ctx: &mut Ctx<'_, Self>, _observer: ObserverId) {} + /// An observer says where it is looking, in this zone's local + /// coordinates (`Focus`; e.g. a player next door, so this zone's + /// interest management follows that player across the border). The + /// default moves the observer's view centre there and keeps its + /// radius; override to clamp, resize or ignore it. Untrusted: it only + /// steers what the observer sees. + fn on_observer_focus(&mut self, ctx: &mut Ctx<'_, Self>, observer: ObserverId, x: f32, y: f32) { + if !(x.is_finite() && y.is_finite()) { + return; + } + if let Some(view) = ctx.view_of(observer) { + ctx.set_view(observer, View::new(x, y, view.radius)); + } + } + /// Decode raw input bytes from the wire; `None` drops the input. fn decode_input(&self, player: PlayerId, payload: &[u8]) -> Option; diff --git a/distributed_game/tests/fixtures/messages.json b/distributed_game/tests/fixtures/messages.json index 40f07d39d..3c2ae4981 100644 --- a/distributed_game/tests/fixtures/messages.json +++ b/distributed_game/tests/fixtures/messages.json @@ -97,6 +97,16 @@ "hex": "0600", "name": "passive" }, + { + "decoded": { + "type": "Focus", + "x": -25.5, + "y": 3.25 + }, + "direction": "client_to_server", + "hex": "070000ccc100005040", + "name": "focus" + }, { "decoded": { "content_version": "c1", diff --git a/distributed_game/tests/golden/main.rs b/distributed_game/tests/golden/main.rs index b6eb18574..7f1fa8c92 100644 --- a/distributed_game/tests/golden/main.rs +++ b/distributed_game/tests/golden/main.rs @@ -104,6 +104,7 @@ fn client_json(m: &ClientMessage) -> Value { json!({ "type": "Content", "content_version": content_version }) } ClientMessage::Near { near } => json!({ "type": "Near", "near": near }), + ClientMessage::Focus { x, y } => json!({ "type": "Focus", "x": x, "y": y }), } } @@ -373,6 +374,7 @@ fn messages() { ), ("near", ClientMessage::Near { near: true }), ("passive", ClientMessage::Near { near: false }), + ("focus", ClientMessage::Focus { x: -25.5, y: 3.25 }), ]; for (name, msg) in client { let bytes = msg.encode(); diff --git a/examples/game/client/src/mobpit/client.ts b/examples/game/client/src/mobpit/client.ts index aa026a84a..98f1f4414 100644 --- a/examples/game/client/src/mobpit/client.ts +++ b/examples/game/client/src/mobpit/client.ts @@ -17,7 +17,10 @@ * `game_sim::mobpit`). * * Neighbours: the client also watches the 8 rooms around its own as an - * observer (`watchZones` over `neighbourRooms`), so their mobs are drawn where they are. On + * observer (`watchZones` over `neighbourRooms`), so their mobs are drawn where they are. + * Each watcher sends the player's position in that room's coordinates + * (`setFocus`), and the room replicates only what is within the player's + * own view radius around it. On * celld an observer is a passive watcher: it keeps no room awake and wakes * none (a sleeping room answers `ASLEEP`; it is tried again every few * seconds and shows nothing meanwhile). @@ -538,6 +541,15 @@ export function createMobPitClient(options: MobPitClientOptions): MobPitClient { reachBits = 0; } const meNow = me(); + // Each neighbour applies the player's own view around it (its + // interest radius carries on across the wall). + const atWorld = meNow === undefined ? undefined : worldOf(meNow); + if (atWorld !== undefined) { + watcher.setFocus((zone) => { + const r = parseRoom(zone); + return r === undefined ? undefined : { x: atWorld.x - r[0] * ROOM_SIZE, y: atWorld.y - r[1] * ROOM_SIZE }; + }); + } let bits = 0; if (wake && at !== undefined && meNow !== undefined && meNow.zoneId === room && pures.loaded()) { const p = pit.playerFields(meNow); diff --git a/examples/game/client/tests/helpers/mobpit.mjs b/examples/game/client/tests/helpers/mobpit.mjs index 38e7bd784..9a5edb99e 100644 --- a/examples/game/client/tests/helpers/mobpit.mjs +++ b/examples/game/client/tests/helpers/mobpit.mjs @@ -8,7 +8,7 @@ import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { BUTTON, isMob, isPlayer, mobFields, playerFields } from '../../src/generated/mobpit/index.ts'; +import { BUTTON, isMob, isPlayer, mobFields, PIT_WORLD, playerFields } from '../../src/generated/mobpit/index.ts'; import { freePort, startCelld } from './celld.mjs'; import { sleep, startDemo } from './demo-server.mjs'; import { startFleet } from './fleet.mjs'; @@ -97,6 +97,50 @@ export async function startPitFleet(demo = 'mobpit', vars = {}, nodes = 3) { }; } +/** + * Walk to room-local `(tx, ty)` (one command frame per tick), steering + * around the room's obstacles, and record in `seen.max` how many rooms + * were near at most. + */ +export async function walkTo(pit, obstacles, [tx, ty], seen = { max: 0 }, timeoutMs = 20_000) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const me = pit.me(); + const p = me && me.zoneId === pit.room() ? playerFields(me) : undefined; + seen.max = Math.max(seen.max, pit.near().size); + if (p !== undefined && Math.hypot(tx - p.x, ty - p.y) < 0.4) break; + if (Date.now() > deadline) throw new Error(`walk to ${tx},${ty} timed out at ${p?.x},${p?.y}`); + let input = { mx: 0, my: 0, buttons: 0 }; + if (p !== undefined && !p.dead) { + let dx = tx - p.x; + let dy = ty - p.y; + const len = Math.max(Math.hypot(dx, dy), 1e-3); + dx /= len; + dy /= len; + // Push away from obstacles close ahead (slide around them). + for (const o of obstacles) { + const ox = p.x - o.x; + const oy = p.y - o.y; + const d = Math.hypot(ox, oy); + const clear = d - o.r - PIT_WORLD.playerRadius; + if (clear < 1.5 && ox * dx + oy * dy < 0) { + const k = 0.8 / Math.max(clear, 0.15); + // Sideways around it, on the side the target is. + const side = Math.sign(-oy * (tx - o.x) + ox * (ty - o.y)) || 1; + dx += ((side * -oy) / d) * k + (ox / d) * k * 0.3; + dy += ((side * ox) / d) * k + (oy / d) * k * 0.3; + } + } + const n = Math.max(Math.hypot(dx, dy), 1e-3); + const slow = Math.min(1, len / 1.5); + input = { mx: Math.round((dx / n) * 127 * Math.max(slow, 0.35)), my: Math.round((dy / n) * 127 * Math.max(slow, 0.35)), buttons: 0 }; + } + pit.client.sendInput(input); + await sleep(1000 / 30); + } + pit.client.sendInput({ mx: 0, my: 0, buttons: 0 }); +} + /** * Walk to the doorway on side `[dx, dy]` and through it (one command frame * per tick) until the client's authority is the next room and the own diff --git a/examples/game/client/tests/mobpit-neighbours-headless.test.mjs b/examples/game/client/tests/mobpit-neighbours-headless.test.mjs index 283e85676..03a5ce6f0 100644 --- a/examples/game/client/tests/mobpit-neighbours-headless.test.mjs +++ b/examples/game/client/tests/mobpit-neighbours-headless.test.mjs @@ -5,16 +5,18 @@ * sleeping room is not woken (it answers `ASLEEP`, nothing is drawn), an * awake one streams its mobs, and the watcher does not keep it awake — when * its last player leaves it goes dormant and then hibernates as if nobody - * looked, and the watcher shows nothing again. + * looked, and the watcher shows nothing again. A neighbour applies the + * player's own view radius around the player (`Focus`): only its mobs + * within reach of the player are sent, also after the player moves. */ import assert from 'node:assert/strict'; import test from 'node:test'; -import { isMob } from '../src/generated/mobpit/index.ts'; +import { isMob, mobFields } from '../src/generated/mobpit/index.ts'; import { createMobPitClient, MobPitApi, neighbourRooms } from '../src/mobpit/client.ts'; import { DevApi } from '../src/shared/dev-api.ts'; import { eventually, memoryStorage, until } from './helpers/demo-server.mjs'; -import { startPit } from './helpers/mobpit.mjs'; +import { startPit, walkTo } from './helpers/mobpit.mjs'; test('neighbour rooms: passive watchers see mobs, never wake or keep a room awake', { timeout: 600_000 }, async () => { const server = await startPit('mobpit'); @@ -25,7 +27,9 @@ test('neighbour rooms: passive watchers see mobs, never wake or keep a room awak const pit = createMobPitClient({ api: new DevApi({ base: server.httpUrl, name: () => name, storage: memoryStorage() }), roomsBase: server.roomsUrl, - neighbours + neighbours, + // Passive watchers only (proximity wake has its own test). + wake: false }); clients.push(pit); return pit; @@ -49,8 +53,37 @@ test('neighbour rooms: passive watchers see mobs, never wake or keep a room awak await other.ready; const layout = watcher.pures.pitRoom({ seed: hello.seed, x: 1, y: 0 }); await until(() => watcher.neighbours().get('pit.1.0') === 'watching', 15_000, 'watching pit.1.0'); - await until(() => mobsIn('pit.1.0') === layout.mobs.length, 10_000, 'every mob of pit.1.0'); - console.error(`watching pit.1.0: ${mobsIn('pit.1.0')} mobs`); + // Only what the player's own view covers, measured from the player + // (pit.1.0's frame: world x − 40), leaving at +10 %. + const leave = hello.viewRadius * 1.1; + const eastMobs = () => + watcher.client + .render() + .filter((e) => isMob(e) && !e.stale && e.zoneId === 'pit.1.0') + .map((e) => mobFields(e)); + const focus = () => { + const at = watcher.position(); + return { x: at.x - 40, y: at.y }; + }; + const withinView = (slack) => { + const f = focus(); + return eastMobs().every((m) => Math.hypot(m.x - f.x, m.y - f.y) <= leave + slack); + }; + const farMobs = () => { + const f = focus(); + return layout.mobs.filter((m) => Math.hypot(m.x - f.x, m.y - f.y) > leave + 6).length; + }; + await until(() => withinView(0.5), 10_000, 'pit.1.0 sends only what the player sees'); + if (farMobs() > 0) assert.ok(eastMobs().length < layout.mobs.length, 'far mobs of pit.1.0 are not sent'); + console.error(`watching pit.1.0 from the room's middle: ${mobsIn('pit.1.0')} of ${layout.mobs.length} mobs`); + + // Walking up to the shared wall brings pit.1.0's west half into view. + const obstacles = watcher.pures.pitRoom({ seed: hello.seed, x: 0, y: 0 }).obstacles; + const middle = mobsIn('pit.1.0'); + await walkTo(watcher, obstacles, [15, 3]); + await until(() => withinView(0.5) && mobsIn('pit.1.0') > middle, 10_000, 'more of pit.1.0 near its wall'); + console.error(`watching pit.1.0 from beside its wall: ${mobsIn('pit.1.0')} of ${layout.mobs.length} mobs`); + await walkTo(watcher, obstacles, [-3, 3]); // Its player leaves: the room sleeps although it is watched. other.close(); diff --git a/examples/game/client/tests/mobpit-wake-headless.test.mjs b/examples/game/client/tests/mobpit-wake-headless.test.mjs index a223ef79f..5c7340f19 100644 --- a/examples/game/client/tests/mobpit-wake-headless.test.mjs +++ b/examples/game/client/tests/mobpit-wake-headless.test.mjs @@ -11,54 +11,11 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { PIT_WORLD, playerFields } from '../src/generated/mobpit/index.ts'; +import { PIT_WORLD } from '../src/generated/mobpit/index.ts'; import { createMobPitClient, MobPitApi } from '../src/mobpit/client.ts'; import { DevApi } from '../src/shared/dev-api.ts'; import { eventually, memoryStorage, sleep, until } from './helpers/demo-server.mjs'; -import { startPit } from './helpers/mobpit.mjs'; - -/** - * Walk to room-local `(tx, ty)` (one command frame per tick), steering - * around the room's obstacles, and record how many rooms were near. - */ -async function walkTo(pit, obstacles, [tx, ty], seen, timeoutMs = 20_000) { - const deadline = Date.now() + timeoutMs; - for (;;) { - const me = pit.me(); - const p = me && me.zoneId === pit.room() ? playerFields(me) : undefined; - seen.max = Math.max(seen.max, pit.near().size); - if (p !== undefined && Math.hypot(tx - p.x, ty - p.y) < 0.4) break; - if (Date.now() > deadline) throw new Error(`walk to ${tx},${ty} timed out at ${p?.x},${p?.y}`); - let input = { mx: 0, my: 0, buttons: 0 }; - if (p !== undefined && !p.dead) { - let dx = tx - p.x; - let dy = ty - p.y; - const len = Math.max(Math.hypot(dx, dy), 1e-3); - dx /= len; - dy /= len; - // Push away from obstacles close ahead (slide around them). - for (const o of obstacles) { - const ox = p.x - o.x; - const oy = p.y - o.y; - const d = Math.hypot(ox, oy); - const clear = d - o.r - PIT_WORLD.playerRadius; - if (clear < 1.5 && ox * dx + oy * dy < 0) { - const k = 0.8 / Math.max(clear, 0.15); - // Sideways around it, on the side the target is. - const side = Math.sign(-oy * (tx - o.x) + ox * (ty - o.y)) || 1; - dx += ((side * -oy) / d) * k + (ox / d) * k * 0.3; - dy += ((side * ox) / d) * k + (oy / d) * k * 0.3; - } - } - const n = Math.max(Math.hypot(dx, dy), 1e-3); - const slow = Math.min(1, len / 1.5); - input = { mx: Math.round((dx / n) * 127 * Math.max(slow, 0.35)), my: Math.round((dy / n) * 127 * Math.max(slow, 0.35)), buttons: 0 }; - } - pit.client.sendInput(input); - await sleep(1000 / 30); - } - pit.client.sendInput({ mx: 0, my: 0, buttons: 0 }); -} +import { startPit, walkTo } from './helpers/mobpit.mjs'; /** Keep sending idle frames for `ms` (the room keeps simulating us). */ async function idle(pit, ms, seen) { diff --git a/examples/game/server/rooms/src/zone.rs b/examples/game/server/rooms/src/zone.rs index 2302110c7..001868b1d 100644 --- a/examples/game/server/rooms/src/zone.rs +++ b/examples/game/server/rooms/src/zone.rs @@ -1860,11 +1860,25 @@ impl Zone for MobPit { } } - /// A watcher (a player next door, a board page): the whole room. + /// A watcher (a player next door, a board page): the whole room, until + /// it says where it stands (`on_observer_focus`). fn on_observe(&mut self, ctx: &mut Ctx<'_, Self>, observer: ObserverId) { ctx.set_view(observer, View::new(0.0, 0.0, WATCH_RADIUS)); } + /// A player next door, at `(x, y)` in this room's coordinates, sees this + /// room as it sees its own: [`VIEW_RADIUS`] around it, so interest and + /// update rates carry on across the wall. + fn on_observer_focus(&mut self, ctx: &mut Ctx<'_, Self>, observer: ObserverId, x: f32, y: f32) { + if !(x.is_finite() && y.is_finite()) { + return; + } + // A watcher stands at most one room away. + let reach = 3.0 * movement::HALF; + let view = View::new(x.clamp(-reach, reach), y.clamp(-reach, reach), VIEW_RADIUS); + ctx.set_view(observer, view); + } + fn decode_input(&self, _player: PlayerId, payload: &[u8]) -> Option { PitInput::from_payload(payload) } diff --git a/examples/game/server/rooms/tests/rooms.rs b/examples/game/server/rooms/tests/rooms.rs index ba25abccf..9133c60c5 100644 --- a/examples/game/server/rooms/tests/rooms.rs +++ b/examples/game/server/rooms/tests/rooms.rs @@ -847,3 +847,69 @@ fn inputs_sent_through_a_stall_drain_but_a_fast_client_is_still_paced() { "{ack} inputs in {wall_ticks} ticks of wall time" ); } + +/// Mob positions an observer's mirror holds (kind 2 = mob). +fn mob_positions(mirror: &distributed_game::ClientMirror) -> Vec<(f32, f32)> { + use distributed_game::FieldValue; + mirror + .entities + .values() + .filter(|e| e.kind == 2) + .filter_map(|e| match (&e.fields[0], &e.fields[1]) { + (FieldValue::F(x), FieldValue::F(y)) => Some((*x, *y)), + _ => None, + }) + .collect() +} + +fn observe( + room: &mut ZoneHost, + clock: &ManualClock, + client: ClientId, + mirror: &mut distributed_game::ClientMirror, +) { + for _ in 0..6 { + for o in tick(room, clock) { + if let Outbound::Frame { client: c, frame } = o { + if c == client { + mirror.apply(&frame).expect("frame"); + } + } + } + } +} + +#[test] +fn a_player_next_door_sees_this_room_from_where_it_stands() { + let clock = ManualClock::new(); + let mut room = host("pit.0.0", &clock); + let claims = TicketClaims::new("ana", room.zone_id(), u64::MAX, "ana-w", Role::Observer); + let watcher = room.join(&claims, 0, "").expect("observe").client; + let mut mirror = distributed_game::ClientMirror::new(); + observe(&mut room, &clock, watcher, &mut mirror); + let whole = mob_positions(&mirror); + assert!( + whole.iter().any(|&(x, _)| x > 5.0), + "before Focus a watcher sees the whole room: {whole:?}" + ); + + // Ana stands in the west room, 5 m from the shared wall: (-25, 0) here. + let focus = (-25.0_f32, 0.0_f32); + room.observer_focus(watcher, focus.0, focus.1); + observe(&mut room, &clock, watcher, &mut mirror); + let seen = mob_positions(&mirror); + // Her own view radius (24 m, leaving at +10 %), as in her own room. + let leave = mobpit_rooms::VIEW_RADIUS * 1.1; + for &(x, y) in &seen { + assert!( + (x - focus.0).hypot(y - focus.1) <= leave, + "mob at ({x}, {y}) is beyond her view: {seen:?}" + ); + } + let near = whole + .iter() + .filter(|&&(x, y)| (x - focus.0).hypot(y - focus.1) < mobpit_rooms::VIEW_RADIUS - 2.0) + .count(); + assert!(seen.len() >= near.min(1), "mobs near the wall stay visible"); + assert!(seen.len() < whole.len(), "far mobs are no longer sent"); +} diff --git a/examples/game/sveltekit/src/lib/pit/loop.ts b/examples/game/sveltekit/src/lib/pit/loop.ts index 99bb41bd4..1f97832b7 100644 --- a/examples/game/sveltekit/src/lib/pit/loop.ts +++ b/examples/game/sveltekit/src/lib/pit/loop.ts @@ -67,7 +67,10 @@ export function reachRooms(room: readonly [number, number], near: number): strin * beyond `wakeRelease`, at most the two of a corner, corner rooms never) * are made near on `watch` — the rooms wake and stay live (the ticket * route grants `wake` only for rooms within two doorways of the player's - * last room). `enabled()` false: none (passive watchers). Returns stop. + * last room). `enabled()` false: none (passive watchers). Every tick also + * tells each watched neighbour where the player stands in its coordinates + * (`setFocus`), so it sends only what the player's own view covers. + * Returns stop. */ export function startProximityWake( client: GameClient, @@ -87,6 +90,14 @@ export function startProximityWake( bits = 0; } const me = client.render().find((e) => e.predicted && e.zoneId === zone && isPlayer(e)); + if (at !== undefined && me !== undefined) { + // Each neighbour applies the player's own view around it. + const p = playerFields(me); + watch.setFocus((neighbour) => { + const r = parseRoom(neighbour); + return r === undefined ? undefined : { x: p.x + (at[0] - r[0]) * ROOM, y: p.y + (at[1] - r[1]) * ROOM }; + }); + } let next = 0; if (enabled() && at !== undefined && me !== undefined && pures.loaded()) { const p = playerFields(me); diff --git a/js/src/game/client.ts b/js/src/game/client.ts index 143b7432e..3300a9d6c 100644 --- a/js/src/game/client.ts +++ b/js/src/game/client.ts @@ -320,6 +320,16 @@ export class GameClient { if (zone?.role === 'observer') zone.connection.setNear(near); } + /** + * Tell an observed zone where this client is looking, in that zone's + * local coordinates (`GameConnection.setFocus`). No-op for zones not + * observed. + */ + setFocus(zoneId: string, x: number, y: number): void { + const zone = this.#zones.get(zoneId); + if (zone?.role === 'observer') zone.connection.setFocus(x, y); + } + /** Stop observing a zone. */ unobserve(zoneId: string): void { const zone = this.#zones.get(zoneId); diff --git a/js/src/game/connection.ts b/js/src/game/connection.ts index 23cc6f4f0..7233cad6e 100644 --- a/js/src/game/connection.ts +++ b/js/src/game/connection.ts @@ -200,6 +200,11 @@ type Attempt = { const defaultBackoff = (attempt: number): number => Math.min(5000, 250 * 2 ** (attempt - 1)); +/** Observers: at most one `Focus` per this many ms (see `setFocus`). */ +export const FOCUS_INTERVAL_MS = 200; +/** Observers: a `Focus` is sent only after a move of this many metres. */ +export const FOCUS_MIN_MOVE = 0.25; + export class GameConnection { readonly zoneId: string; readonly clock: ClockSync; @@ -217,6 +222,9 @@ export class GameConnection { #url: string; #contentVersion: string; #near: boolean; + /** Observers: the focus asked for, and the last one sent (and when). */ + #focus: { x: number; y: number } | undefined; + #focusSent: { x: number; y: number; at: number } | undefined; #redirects = 0; #lastResumeVersion: string | undefined; #attempt: Attempt | undefined; @@ -443,6 +451,30 @@ export class GameConnection { } } + /** + * Observers: where this watcher is looking, in the zone's local + * coordinates (a player next door: its position seen from this zone, so + * the zone's interest management follows it). Call it as often as you + * like (e.g. every frame): it is sent at most every + * {@link FOCUS_INTERVAL_MS} and only after a move of + * {@link FOCUS_MIN_MOVE} or more, and again after every (re)connect. + */ + setFocus(x: number, y: number): void { + if (!Number.isFinite(x) || !Number.isFinite(y)) return; + this.#focus = { x, y }; + this.#sendFocus(); + } + + #sendFocus(): void { + const focus = this.#focus; + if (focus === undefined || this.#state !== 'open' || this.mirror.welcome?.role !== 'observer') return; + const now = this.#scheduler.now(); + const last = this.#focusSent; + if (last !== undefined && (now - last.at < FOCUS_INTERVAL_MS || Math.hypot(focus.x - last.x, focus.y - last.y) < FOCUS_MIN_MOVE)) return; + this.#focusSent = { ...focus, at: now }; + this.#send({ type: 'Focus', x: focus.x, y: focus.y }); + } + /** * Continue after a content reload (state `reload-required`): sends the * new `contentVersion` in Hello, with the reconnect token while the grace @@ -645,6 +677,8 @@ export class GameConnection { this.clock.reset(); this.clock.seed(welcome.serverTime, this.#scheduler.now()); this.#setState('open'); + this.#focusSent = undefined; + this.#sendFocus(); this.#startPing(); this.#emitter.emit('welcome', welcome, { resumed: attempt.mode === 'resume', reconnected }); const waiters = this.#connectWaiters; diff --git a/js/src/game/index.ts b/js/src/game/index.ts index 1bb9b53b1..1aaa64566 100644 --- a/js/src/game/index.ts +++ b/js/src/game/index.ts @@ -109,6 +109,8 @@ export { } from './typed.js'; export { ClockSync, type ClockSample, type ClockSyncOptions } from './clock.js'; export { + FOCUS_INTERVAL_MS, + FOCUS_MIN_MOVE, GameConnection, GameConnectionClosedError, defaultScheduler, diff --git a/js/src/game/messages.ts b/js/src/game/messages.ts index a86f31279..f7da0ab0e 100644 --- a/js/src/game/messages.ts +++ b/js/src/game/messages.ts @@ -24,6 +24,8 @@ export const GameMessageTag = Object.freeze({ CONTENT: 0x05, /** Client → server: an observer is near (keep the zone awake) or passive again. */ NEAR: 0x06, + /** Client → server: where an observer is looking (zone-local `x`, `y`). */ + FOCUS: 0x07, WELCOME: 0x80, DELTA: 0x81, EVENT: 0x82, @@ -98,7 +100,9 @@ export type LeaveMessage = Readonly<{ type: 'Leave' }>; export type ContentMessage = Readonly<{ type: 'Content'; contentVersion: string }>; /** An observer is near the zone (keep it awake, with a `wake` ticket) or passive again. */ export type NearMessage = Readonly<{ type: 'Near'; near: boolean }>; -export type GameClientMessage = HelloMessage | InputMessage | PingMessage | LeaveMessage | ContentMessage | NearMessage; +/** Where an observer is looking, in the zone's local coordinates (the zone decides what it means; by default its view follows). */ +export type FocusMessage = Readonly<{ type: 'Focus'; x: number; y: number }>; +export type GameClientMessage = HelloMessage | InputMessage | PingMessage | LeaveMessage | ContentMessage | NearMessage | FocusMessage; export type GameRole = 'player' | 'observer'; @@ -198,6 +202,9 @@ export function encodeClientMessage(message: GameClientMessage): Uint8Array { case 'Near': w.u8(GameMessageTag.NEAR).bool(message.near); break; + case 'Focus': + w.u8(GameMessageTag.FOCUS).f32(message.x).f32(message.y); + break; } return w.finish(); } @@ -234,6 +241,9 @@ export function decodeClientMessage(bytes: Uint8Array): GameClientMessage { case GameMessageTag.NEAR: message = { type: 'Near', near: r.bool() }; break; + case GameMessageTag.FOCUS: + message = { type: 'Focus', x: r.f32(), y: r.f32() }; + break; default: throw new GameDecodeError('UNKNOWN_MESSAGE', `message type 0x${tag.toString(16)}`); } diff --git a/js/src/game/watch.ts b/js/src/game/watch.ts index c182a4549..ffa936d48 100644 --- a/js/src/game/watch.ts +++ b/js/src/game/watch.ts @@ -20,6 +20,12 @@ * before the issuer knew where the player is) is reopened with a fresh * ticket while its zone stays near — at once the first time, then backing * off like a sleeping zone. + * + * **Focus** (interest across borders): `setFocus(at)` tells every watched + * zone where the player is in that zone's local coordinates (`at(zone)`), + * so each neighbour applies the player's own view around it instead of + * sending everything. Call it every frame; connections throttle it and + * resend after reconnects. */ import type { GameClient } from './client.js'; import { GameErrorCode } from './messages.js'; @@ -53,6 +59,11 @@ export type ZoneWatch = Readonly<{ setNear(zones: Iterable): void; /** The zones last passed to `setNear`. */ near(): ReadonlySet; + /** + * Where the player is, per watched zone, in that zone's local + * coordinates (`undefined`: no focus for it). + */ + setFocus(at: (zone: string) => { x: number; y: number } | undefined): void; }>; /** @@ -176,6 +187,12 @@ export function watchZones( else for (const zone of wanted) client.setNear(zone, near.has(zone)); }, near: () => near, + setFocus: (at) => { + for (const zone of wanted) { + const point = at(zone); + if (point !== undefined) client.setFocus(zone, point.x, point.y); + } + }, stop: () => { stopped = true; clear(timer); diff --git a/js/tests/game-codec.test.mjs b/js/tests/game-codec.test.mjs index 3efb66967..5fb7a92de 100644 --- a/js/tests/game-codec.test.mjs +++ b/js/tests/game-codec.test.mjs @@ -149,6 +149,8 @@ function messageFromJson(decoded, schema) { return { type: 'Content', contentVersion: decoded.content_version }; case 'Near': return { type: 'Near', near: decoded.near }; + case 'Focus': + return { type: 'Focus', x: decoded.x, y: decoded.y }; case 'Welcome': return { type: 'Welcome', @@ -234,6 +236,8 @@ function messageToJson(message, kinds) { return { content_version: message.contentVersion, type: 'Content' }; case 'Near': return { near: message.near, type: 'Near' }; + case 'Focus': + return { type: 'Focus', x: message.x, y: message.y }; case 'Welcome': return { content_version: message.contentVersion, @@ -442,7 +446,7 @@ test('decoders reject malformed input like the Rust reference', () => { assert.throws(() => new WireReader(fromHex('ffffffff10')).varuint(), /VARINT_OVERFLOW/); assert.throws(() => new WireReader(fromHex('02')).bool(), /INVALID/); assert.throws(() => new WireReader(fromHex('02c328')).string(), /INVALID_UTF8/); - assert.throws(() => decodeClientMessage(fromHex('07')), /UNKNOWN_MESSAGE/); + assert.throws(() => decodeClientMessage(fromHex('08')), /UNKNOWN_MESSAGE/); // A delta update for an unknown entity id. const { messages } = fixture('messages.json'); const delta = messages.find((m) => m.name === 'delta'); diff --git a/js/tests/game-connection.test.mjs b/js/tests/game-connection.test.mjs index 17192ca55..1ef2e910e 100644 --- a/js/tests/game-connection.test.mjs +++ b/js/tests/game-connection.test.mjs @@ -127,6 +127,44 @@ test('an observer says it is near in Hello and with Near, without reconnecting', player.close(); }); +test('an observer sends Focus throttled, and again after a reconnect', async () => { + const network = new FakeNetwork(); + const scheduler = new FakeScheduler(); + network.onMessage = (conn, message) => { + if (message.type === 'Hello') conn.send(welcome({ zoneId: conn.zoneId, role: 'observer', serverTime: 10_000 })); + }; + const { conn } = connection(network, scheduler, { role: 'observer' }); + await conn.connect(); + const focus = () => network.last().inbox.filter((m) => m.type === 'Focus'); + conn.setFocus(-25, 0); + // Within the interval, or moved too little: not sent. + conn.setFocus(-24, 0); + await scheduler.advance(250); + conn.setFocus(-25.125, 0); + conn.setFocus(-23, 1); + assert.deepEqual(focus(), [ + { type: 'Focus', x: -25, y: 0 }, + { type: 'Focus', x: -23, y: 1 } + ]); + conn.setFocus(Number.NaN, 0); + assert.equal(focus().length, 2, 'non-finite focus is ignored'); + // A new connection hears the last focus right after its Welcome. + network.last().close(1006, "lost"); + await scheduler.advance(5_000); + assert.ok(network.connections.length >= 2, 'reconnected'); + assert.deepEqual(focus(), [{ type: 'Focus', x: -23, y: 1 }]); + conn.close(); + // A player's focus is never sent. + network.onMessage = (c, message) => { + if (message.type === 'Hello') c.send(welcome({ zoneId: c.zoneId, serverTime: 10_000 })); + }; + const player = connection(network, scheduler).conn; + await player.connect(); + player.setFocus(1, 2); + assert.deepEqual(network.last().inbox.filter((m) => m.type === 'Focus'), []); + player.close(); +}); + test('ping/pong estimates offset, rtt and jitter', async () => { const network = new FakeNetwork(); const scheduler = new FakeScheduler(); From da77ccbb1cef61a68e58a03e134fc60e4cc5c5ed Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:08:29 -0500 Subject: [PATCH 07/22] refactor(game): opt into fact batches with one override A sink opted into batches by overriding both `FactSink::max_batch` and `FactSink::commit_batch` (a host: `max_fact_batch` and `dispatch_fact_batch`). Overriding only the size once batched a sink through the default one-by-one `commit_batch`, which held settlements back until the whole batch committed. Now one override opts in and supplies the batch commit: `FactSink::batches() -> Option<&dyn FactBatchSink>` and `FactHost::fact_batches() -> Option<&dyn FactBatchHost>`, both `None` by default. The feed batches only a sink that returns one (up to `FactFeed::with_max_batch` / `ZONE_FACT_BATCH`); the rest get one fact at a time. `DurableFacts`, `CommandFactSink`, `CelldObjectHost` and the test sinks moved over; behaviour is unchanged. New test: a sink without batches settles each fact as it commits. [[zone-fact-batches]] --- .../skills/distributed-game/SKILL.md | 7 +- distributed_game/README.md | 22 +-- distributed_game/src/celld_zone/object.rs | 22 +-- distributed_game/src/command_sink.rs | 59 ++++---- distributed_game/src/facts.rs | 130 ++++++++++++------ distributed_game/src/lib.rs | 6 +- 6 files changed, 155 insertions(+), 91 deletions(-) diff --git a/distributed_cli/skills/distributed-game/SKILL.md b/distributed_cli/skills/distributed-game/SKILL.md index 6d606ae9c..f1ecc782a 100644 --- a/distributed_cli/skills/distributed-game/SKILL.md +++ b/distributed_cli/skills/distributed-game/SKILL.md @@ -52,9 +52,10 @@ You get: as one `POST …/__batch` (`ZONE_FACT_BATCH`, default 16), and the zone outbox clears them with one delete. A batch stops at its first retryable failure and resumes from there, so order and exactly-once hold. A custom - sink opts in by overriding `FactSink::max_batch` and `commit_batch`, or - `FactHost::max_fact_batch` and `dispatch_fact_batch`. Otherwise it gets - one fact at a time. + sink opts in with one override, `FactSink::batches` returning + `Some(self)` as a `&dyn FactBatchSink` (its `commit_batch`); a host, + `FactHost::fact_batches` returning a `&dyn FactBatchHost`. Otherwise it + gets one fact at a time. Results come back on a later tick in `on_fact_settled`. diff --git a/distributed_game/README.md b/distributed_game/README.md index 71cd9b877..658eb4161 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -1170,15 +1170,19 @@ a time (a slow commit delays every later fact). Set without a principal) stay sequential in emission order; different principals commit concurrently; settlements arrive in completion order. -**Batches.** A sink that settles many facts in one round trip says so -(`FactSink::max_batch`, `FactHost::max_fact_batch`; the celld object host -does) and the feed then hands it up to `max_batch` (default 16, -`FactFeed::with_max_batch`; celld zones: `ZONE_FACT_BATCH`) waiting facts -of a lane at once, `FactSink::commit_batch`. Other sinks keep getting one -fact at a time. Game code is unchanged (`ctx.emit`, -`#[derive(FactCommand)]`, `#[command]` handlers). `CommandFactSink` -groups consecutive facts for the same aggregate and calls -`FactHost::dispatch_fact_batch`, which the celld object host sends as one `POST {kind}/{shard}/__batch` (each entry +**Batches.** A sink that settles many facts in one round trip opts in +with one override that also supplies the batch commit: +`fn batches(&self) -> Option<&dyn FactBatchSink> { Some(self) }` plus +`impl FactBatchSink for MySink { fn commit_batch(…) }` (a host: +`FactHost::fact_batches` → `FactBatchHost::dispatch_fact_batch`; the +celld object host does). The feed then hands it up to +`FactFeed::with_max_batch` (default 16; celld zones: `ZONE_FACT_BATCH`) +waiting facts of a lane at once. Other sinks (`batches` is `None` by +default) get one fact at a time and each settlement as soon as it +commits. Game code is unchanged (`ctx.emit`, `#[derive(FactCommand)]`, +`#[command]` handlers). `CommandFactSink` groups consecutive facts for +the same aggregate and hands them to the host's batch dispatch, which the +celld object host sends as one `POST {kind}/{shard}/__batch` (each entry with its own command id, principal and stamp; `serve_cell` runs them in order through the same idempotent wait path and drains the cell outbox once). A batch stops at its first retryable failure and is retried from diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index 14158850d..f792cce62 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -26,9 +26,10 @@ use super::plan::{ cell_binding, classify_wait_path, zone_from_path, ObjectOptions, RoomState, StoredFact, TickSchedule, Window, CLOSE_RESTARTED, NETSIM_SHARD, OUTBOX_PREFIX, ZONE_DEV_KIND, ZONE_HEADER, }; -use crate::command_sink::{CommandFactSink, FactCall, FactHost, FactRoute}; +use crate::command_sink::{CommandFactSink, FactBatchHost, FactCall, FactHost, FactRoute}; use crate::facts::{ - fact_channel, BoxFuture, Fact, FactError, FactQueue, FactReceipt, FactSink, RetryPolicy, + fact_channel, BoxFuture, Fact, FactBatchSink, FactError, FactQueue, FactReceipt, FactSink, + RetryPolicy, }; use crate::handoff::{HandoffRequest, Transfer}; use crate::inbox::ZoneInbound; @@ -413,12 +414,14 @@ impl FactHost for CelldObjectHost { Box::pin(SendFuture::new(async move { self.post(&call).await })) } - /// Any number: one `__batch` request per cell (the feed's - /// `ZONE_FACT_BATCH` bounds it). - fn max_fact_batch(&self) -> usize { - usize::MAX + /// One `__batch` request per cell (the feed's `ZONE_FACT_BATCH` bounds + /// it). + fn fact_batches(&self) -> Option<&dyn FactBatchHost> { + Some(self) } +} +impl FactBatchHost for CelldObjectHost { fn dispatch_fact_batch<'a>( &'a self, calls: Vec>, @@ -471,10 +474,13 @@ impl FactSink for DurableFacts { })) } - fn max_batch(&self) -> usize { - self.sink.max_batch() + /// Batches when the cells take them. + fn batches(&self) -> Option<&dyn FactBatchSink> { + self.sink.batches().map(|_| self as &dyn FactBatchSink) } +} +impl FactBatchSink for DurableFacts { /// One batched delete for the facts the batch settled. fn commit_batch<'a>( &'a self, diff --git a/distributed_game/src/command_sink.rs b/distributed_game/src/command_sink.rs index 5b720f8a7..ccbfdfd58 100644 --- a/distributed_game/src/command_sink.rs +++ b/distributed_game/src/command_sink.rs @@ -44,7 +44,9 @@ use distributed::microsvc::Session; use serde_json::Value; use crate::cell::{command_id_for, session_stamp, trusted_session}; -use crate::facts::{BoxFuture, Fact, FactCommand, FactError, FactReceipt, FactSink}; +use crate::facts::{ + commit_each, BoxFuture, Fact, FactBatchSink, FactCommand, FactError, FactReceipt, FactSink, +}; use crate::lifecycle::LIFECYCLE_COMMANDS; #[cfg(feature = "command-host")] use crate::stamp::SESSION_PREFIX; @@ -153,34 +155,25 @@ pub trait FactHost: Send + Sync + 'static { call: FactCall<'a>, ) -> BoxFuture<'a, Result>; - /// Facts this host runs per [`dispatch_fact_batch`](Self::dispatch_fact_batch) - /// at most (default 1). A host with a batch route returns more, and - /// the sink's [`FactSink::max_batch`] follows it. - fn max_fact_batch(&self) -> usize { - 1 + /// The host's batch route, if it runs many facts for one aggregate in + /// one request (default none: one by one). Opting in and supplying the + /// batch dispatch are the same override, usually `Some(self)`; the + /// sink's [`FactSink::batches`] follows it. + fn fact_batches(&self) -> Option<&dyn FactBatchHost> { + None } +} +/// A [`FactHost`] with a batch route, handed out by +/// [`FactHost::fact_batches`]. +pub trait FactBatchHost: Send + Sync { /// Run `calls` (consecutive facts for one aggregate: same kind and - /// shard) in order, under [`FactSink::commit_batch`]'s contract: the - /// results end at the first retryable failure. The default runs them - /// one by one; a host with a batch route sends them in one request. + /// shard) in order, under [`FactBatchSink::commit_batch`]'s contract: + /// the results end at the first retryable failure. fn dispatch_fact_batch<'a>( &'a self, calls: Vec>, - ) -> BoxFuture<'a, Vec>> { - Box::pin(async move { - let mut results = Vec::with_capacity(calls.len()); - for call in calls { - let result = self.dispatch_fact(call).await; - let stop = matches!(result, Err(FactError::Retryable(_))); - results.push(result); - if stop { - break; - } - } - results - }) - } + ) -> BoxFuture<'a, Vec>>; } /// See the module docs. @@ -327,17 +320,22 @@ impl FactSink for CommandFactSink { Box::pin(async move { self.host.dispatch_fact(self.call_for(fact)?).await }) } - fn max_batch(&self) -> usize { - self.host.max_fact_batch() + fn batches(&self) -> Option<&dyn FactBatchSink> { + self.host.fact_batches().map(|_| self as &dyn FactBatchSink) } +} +impl FactBatchSink for CommandFactSink { /// Consecutive facts for the same aggregate go to the host together - /// ([`FactHost::dispatch_fact_batch`]). + /// ([`FactBatchHost::dispatch_fact_batch`]). fn commit_batch<'a>( &'a self, facts: &'a [Fact], ) -> BoxFuture<'a, Vec>> { Box::pin(async move { + let Some(host) = self.host.fact_batches() else { + return commit_each(self, facts).await; + }; let mut results = Vec::with_capacity(facts.len()); let mut i = 0; while i < facts.len() { @@ -361,8 +359,7 @@ impl FactSink for CommandFactSink { } } let n = run.len(); - let batch: Vec<_> = self - .host + let batch: Vec<_> = host .dispatch_fact_batch(run) .await .into_iter() @@ -680,6 +677,12 @@ mod batch_tests { unreachable!("batches only") } + fn fact_batches(&self) -> Option<&dyn FactBatchHost> { + Some(self) + } + } + + impl FactBatchHost for RecordingHost { fn dispatch_fact_batch<'a>( &'a self, calls: Vec>, diff --git a/distributed_game/src/facts.rs b/distributed_game/src/facts.rs index a82df4c9d..45a28731f 100644 --- a/distributed_game/src/facts.rs +++ b/distributed_game/src/facts.rs @@ -30,10 +30,10 @@ //! in completion order. //! //! Batching: a sink that settles many facts in one round trip (the -//! celld object host: one `__batch` request per cell) says so with -//! [`FactSink::max_batch`]; the feed then hands it up to that many (and -//! at most [`FactFeed::with_max_batch`], default [`DEFAULT_MAX_BATCH`]) -//! consecutive facts of one lane at once ([`FactSink::commit_batch`]). +//! celld object host: one `__batch` request per cell) returns its +//! [`FactBatchSink`] from [`FactSink::batches`]; the feed then hands it up +//! to [`FactFeed::with_max_batch`] (default [`DEFAULT_MAX_BATCH`]) +//! consecutive facts of one lane at once ([`FactBatchSink::commit_batch`]). //! Other sinks keep getting one fact at a time, so their settlements flow //! back per fact. A batch commits in order and stops at its first //! retryable failure, which is retried from there, so order and @@ -247,38 +247,46 @@ pub trait FactSink: Send + Sync + 'static { /// Commit one fact. Must be idempotent per [`Fact::key`]. fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result>; - /// Facts this sink settles per [`commit_batch`](Self::commit_batch) - /// round trip at most (default 1: one fact at a time, each settlement - /// delivered as soon as it commits). Override when a batch is cheaper - /// than its facts one by one. - fn max_batch(&self) -> usize { - 1 + /// The sink's batch committer, if a batch is cheaper than its facts + /// one by one (default none: one fact at a time, each settlement + /// delivered as soon as it commits). Opting in and supplying the + /// batch commit are the same override, usually `Some(self)`. + fn batches(&self) -> Option<&dyn FactBatchSink> { + None } +} +/// A [`FactSink`] that settles many facts per round trip, handed to the +/// feed by [`FactSink::batches`]. +pub trait FactBatchSink: Send + Sync { /// Commit `facts` in order: one result per fact attempted. The /// results end at the first [`FactError::Retryable`] (included), or /// earlier; facts past the end were not attempted and are retried. - /// The default commits them one by one. fn commit_batch<'a>( &'a self, facts: &'a [Fact], - ) -> BoxFuture<'a, Vec>> { - Box::pin(async move { - let mut results = Vec::with_capacity(facts.len()); - for fact in facts { - let result = self.commit(fact).await; - let stop = matches!(result, Err(FactError::Retryable(_))); - results.push(result); - if stop { - break; - } - } - results - }) + ) -> BoxFuture<'a, Vec>>; +} + +/// [`FactBatchSink::commit_batch`] for a sink without batches: its facts +/// one by one, stopping after the first retryable result. +pub(crate) async fn commit_each( + sink: &S, + facts: &[Fact], +) -> Vec> { + let mut results = Vec::with_capacity(facts.len()); + for fact in facts { + let result = sink.commit(fact).await; + let stop = matches!(result, Err(FactError::Retryable(_))); + results.push(result); + if stop { + break; + } } + results } -/// Facts per [`FactSink::commit_batch`] unless the feed says otherwise. +/// Facts per [`FactBatchSink::commit_batch`] unless the feed says otherwise. pub const DEFAULT_MAX_BATCH: usize = 16; /// Retry/backoff policy. @@ -551,7 +559,7 @@ impl FactFeed { self.max_concurrent } - /// Hand a batching sink ([`FactSink::max_batch`] > 1) up to `n` facts + /// Hand a batching sink ([`FactSink::batches`]) up to `n` facts /// of a lane at once (default [`DEFAULT_MAX_BATCH`]; 1 commits fact by /// fact). pub fn with_max_batch(mut self, n: usize) -> Self { @@ -575,7 +583,11 @@ impl FactFeed { Fut: Future, { let policy = self.policy; - let max_batch = self.max_batch.min(sink.max_batch()).max(1); + let max_batch = if sink.batches().is_some() { + self.max_batch + } else { + 1 + }; if self.max_concurrent <= 1 { while let Some(facts) = NextFact(&self.shared, max_batch).await { for settlement in commit_with_retries(&sink, facts, policy, &sleep).await { @@ -598,7 +610,8 @@ impl FactFeed { } } -/// Commit `facts` in order through [`FactSink::commit_batch`], retrying +/// Commit `facts` in order (through [`FactSink::batches`] if more than +/// one), retrying /// from the first unsettled fact; `policy` bounds the attempts at each /// fact as if it were committed alone. async fn commit_with_retries( @@ -619,7 +632,10 @@ where let mut attempts = 0; while !pending.is_empty() { attempts += 1; - let results = sink.commit_batch(pending.make_contiguous()).await; + let results = match sink.batches() { + Some(batches) => batches.commit_batch(pending.make_contiguous()).await, + None => commit_each(sink, pending.make_contiguous()).await, + }; let answered = results.len().min(pending.len()); // A sink that answered for nothing is retried like a retryable head. let mut retry = answered == 0; @@ -791,15 +807,8 @@ impl FactSink for Arc { (**self).commit(fact) } - fn max_batch(&self) -> usize { - (**self).max_batch() - } - - fn commit_batch<'a>( - &'a self, - facts: &'a [Fact], - ) -> BoxFuture<'a, Vec>> { - (**self).commit_batch(facts) + fn batches(&self) -> Option<&dyn FactBatchSink> { + (**self).batches() } } @@ -821,10 +830,12 @@ mod batch_tests { unreachable!("batches only") } - fn max_batch(&self) -> usize { - usize::MAX + fn batches(&self) -> Option<&dyn FactBatchSink> { + Some(self) } + } + impl FactBatchSink for BatchSink { fn commit_batch<'a>( &'a self, facts: &'a [Fact], @@ -919,6 +930,45 @@ mod batch_tests { .all(|s| matches!(s.outcome, FactOutcome::Committed(_)))); } + /// Holds `f1` until released. + struct Gated(tokio::sync::Notify); + + impl FactSink for Gated { + fn commit<'a>(&'a self, fact: &'a Fact) -> BoxFuture<'a, Result> { + Box::pin(async move { + if fact.key() == "f1" { + self.0.notified().await; + } + Ok(FactReceipt { + payload: Value::Null, + replayed: false, + }) + }) + } + } + + #[tokio::test] + async fn a_sink_without_batches_settles_each_fact_as_it_commits() { + let (mut queue, feed) = fact_channel(64, policy(3)); + for i in 0..2 { + queue.try_send(fact(&format!("f{i}"), None)).unwrap(); + } + let sink = Arc::new(Gated(tokio::sync::Notify::new())); + let worker = tokio::spawn( + feed.with_max_batch(16) + .run(sink.clone(), tokio::time::sleep), + ); + // f0 settles while f1 is still committing. + let first = tokio::time::timeout(Duration::from_secs(5), settle(&mut queue, 1)) + .await + .expect("f0 settles before f1 commits"); + assert_eq!(keys(&first), ["f0"]); + sink.0.notify_one(); + assert_eq!(keys(&settle(&mut queue, 1).await), ["f1"]); + drop(queue); + worker.await.unwrap(); + } + #[tokio::test] async fn a_batch_retries_from_its_first_retryable_fact() { let (mut queue, feed) = fact_channel(64, policy(3)); diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index c0430baa5..2c203402c 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -111,9 +111,9 @@ pub use entity::{Entities, Entity, EntityCollection, EntityId, Project}; #[cfg(feature = "runtime")] pub use facts::spawn_fact_worker; pub use facts::{ - fact_channel, fact_key, BoxFuture, EmitError, Fact, FactCommand, FactError, FactFeed, - FactOutcome, FactQueue, FactReceipt, FactSettlement, FactSink, MemoryFactSink, RetryPolicy, - SettlementScript, + fact_channel, fact_key, BoxFuture, EmitError, Fact, FactBatchSink, FactCommand, FactError, + FactFeed, FactOutcome, FactQueue, FactReceipt, FactSettlement, FactSink, MemoryFactSink, + RetryPolicy, SettlementScript, }; pub use handoff::{HandoffError, HandoffRequest, Transfer}; pub use host::{AdvanceReport, Ctx, FrameStats, HostConfig, JoinError, Joined, Outbound, ZoneHost}; From 7b9bef2fed4672f20cbba77c06bd484e3c4a86b8 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:24:44 -0500 Subject: [PATCH 08/22] test(game): time Mob Pit room recovery after a celld crash mobpit-recovery wakes N rooms with one bot each, waits for (or runs) the kill, probes every room with real joins and reports when each answered (p50/p90/max, per 10 s), then each room's ticks over a hold. Bots record when their Welcome came (joined_after). Rooms report a wake trace in /info: when the object was built, its first call, and the awaited storage spans of its first wake and join. [[celld-fleet-recovery-hops]] --- distributed_game/src/bots.rs | 4 + distributed_game/src/celld_zone/object.rs | 74 +++ .../game/server/examples/mobpit-recovery.rs | 529 ++++++++++++++++++ 3 files changed, 607 insertions(+) create mode 100644 examples/game/server/examples/mobpit-recovery.rs diff --git a/distributed_game/src/bots.rs b/distributed_game/src/bots.rs index cff54e092..04f4ebe3a 100644 --- a/distributed_game/src/bots.rs +++ b/distributed_game/src/bots.rs @@ -173,6 +173,8 @@ pub struct BotStats { pub index: usize, /// Welcome received. pub joined: bool, + /// Time from the start of the connect to the Welcome. + pub joined_after: Option, /// Why it stopped early. pub error: Option, /// Deltas applied. @@ -371,6 +373,7 @@ async fn drive( extensions: Default::default(), wake: false, }); + let started = Instant::now(); let url = format!( "{}/{}/{}", config.url.trim_end_matches('/'), @@ -432,6 +435,7 @@ async fn drive( ServerMessage::Welcome(_) => { stats.joined = true; let now = Instant::now(); + stats.joined_after = Some(now - started); joined_at = Some(now); next_input = now; next_ping = now; diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index 14158850d..b695ab6b3 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -589,6 +589,31 @@ struct Snapshot { state: String, } +/// When this activation woke and where its first join spent its time +/// (`Date.now()` ms; celld advances it at I/O boundaries, so the spans are +/// the awaited storage calls). Debug info for recovery timing. +#[derive(Default, Clone, Copy)] +struct WakeTrace { + /// The object was constructed (celld claimed and opened the cell). + constructed_at: f64, + /// First request of any kind (fetch, socket message, alarm). + first_call_at: Option, + /// What woke it: 1 fetch, 2 socket message, 3 alarm. + first_call: u8, + /// [`ZoneObject::start_facts`] (zone id write + outbox listing). + facts_ms: Option, + /// Facts resent from the outbox on wake. + outbox_resent: usize, + /// Snapshot read. + snapshot_read_ms: Option, + /// Restore + the first status write. + restore_ms: Option, + /// The join's own status write and start of the loop, after the wake. + join_ms: Option, + /// First Welcome handed to celld (the output gate may hold it). + welcome_at: Option, +} + #[derive(Default)] struct TickStats { late: Window, @@ -652,6 +677,7 @@ struct Inner { /// [`CelldZone::refresh`] calls of this activation, and the last one. refresh_round: u64, last_refresh_ms: f64, + trace: WakeTrace, } impl Inner { @@ -976,6 +1002,10 @@ impl ZoneObject { world_due: HashMap::new(), refresh_round: 0, last_refresh_ms: 0.0, + trace: WakeTrace { + constructed_at: now_ms(), + ..WakeTrace::default() + }, }; let inner = Rc::new(RefCell::new(inner)); let weak = Rc::downgrade(&inner); @@ -998,6 +1028,7 @@ impl ZoneObject { /// Durable Object `fetch`. pub async fn fetch(&self, req: Request) -> Result { + self.first_call(1); if let Some(zone) = req.headers().get(ZONE_HEADER)? { let mut inner = self.inner.borrow_mut(); if inner.zone_id.is_empty() { @@ -1035,6 +1066,7 @@ impl ZoneObject { ws: WebSocket, message: WebSocketIncomingMessage, ) -> Result<()> { + self.first_call(2); let WebSocketIncomingMessage::Binary(bytes) = message else { let _ = ws.close(Some(1003), Some("binary frames only")); return Ok(()); @@ -1160,6 +1192,7 @@ impl ZoneObject { /// Durable Object `alarm`: hibernate a room that stayed empty, resend /// unsettled facts, and after a crash mark the room asleep. pub async fn alarm(&self) -> Result { + self.first_call(3); if self.inner.borrow().options.is_none() { return Response::ok("unconfigured"); } @@ -1206,6 +1239,14 @@ impl ZoneObject { // -------------------------------------------------------------- internals + fn first_call(&self, kind: u8) { + let mut inner = self.inner.borrow_mut(); + if inner.trace.first_call_at.is_none() { + inner.trace.first_call_at = Some(now_ms()); + inner.trace.first_call = kind; + } + } + /// Sockets accepted before an eviction reach a new instance without a /// zone id from a request; the id is kept in storage. async fn load_zone_id(&self) { @@ -1243,6 +1284,7 @@ impl ZoneObject { ); return; }; + let started = now_ms(); let host = CelldObjectHost::new(self.env.env().clone(), secret, &service); let sink = match ObjectFactSink::new(host, service, zone_principal(&zone_id)) { Ok(sink) => { @@ -1282,7 +1324,9 @@ impl ZoneObject { .await; let mut inner = self.inner.borrow_mut(); inner.outbox_loaded = true; + inner.trace.facts_ms.get_or_insert(now_ms() - started); if let Ok(map) = stored { + let mut resent = 0; let Some(queue) = inner.activity.as_mut() else { return; }; @@ -1295,10 +1339,12 @@ impl ZoneObject { }; if let Some(fact) = stored.into_fact() { let _ = queue.try_send(fact); + resent += 1; } } // Already stored. queue.take_journal(); + inner.trace.outbox_resent += resent; } } @@ -1308,7 +1354,9 @@ impl ZoneObject { if self.inner.borrow().host.is_some() { return Ok(()); } + let read_at = now_ms(); let stored: Option = self.state.storage().get(SNAPSHOT_KEY).await.ok().flatten(); + let restore_at = now_ms(); let host = { let inner = self.inner.borrow(); if inner.host.is_some() { @@ -1402,6 +1450,12 @@ impl ZoneObject { inner.report(ACTIVITY_ACTIVE); } self.store_status().await; + { + let mut inner = self.inner.borrow_mut(); + let trace = &mut inner.trace; + trace.snapshot_read_ms.get_or_insert(restore_at - read_at); + trace.restore_ms.get_or_insert(now_ms() - restore_at); + } self.find_node(); // Hydrate every watched cell, off the join path: a wake (often a // doorway into a sleeping zone, with the player's inputs waiting) @@ -1790,8 +1844,17 @@ impl ZoneObject { if observer { return Ok(()); } + let joined_at = now_ms(); self.store_status().await; self.start_loop(); + { + let mut inner = self.inner.borrow_mut(); + let trace = &mut inner.trace; + if trace.welcome_at.is_none() { + trace.welcome_at = Some(joined_at); + trace.join_ms = Some(now_ms() - joined_at); + } + } if let Some(principal) = principal { self.after_join(principal); } @@ -2182,6 +2245,17 @@ impl ZoneObject { "freeze": format!("{:?}", h.freeze_state()).to_lowercase(), })), "workMs": { "p50": stats.work.percentile(0.5), "p99": stats.work.percentile(0.99) }, + "wake": { + "constructedAt": inner.trace.constructed_at, + "firstCallAt": inner.trace.first_call_at, + "firstCall": inner.trace.first_call, + "factsMs": inner.trace.facts_ms, + "outboxResent": inner.trace.outbox_resent, + "snapshotReadMs": inner.trace.snapshot_read_ms, + "restoreMs": inner.trace.restore_ms, + "joinMs": inner.trace.join_ms, + "welcomeAt": inner.trace.welcome_at, + }, }) } } diff --git a/examples/game/server/examples/mobpit-recovery.rs b/examples/game/server/examples/mobpit-recovery.rs new file mode 100644 index 000000000..ac1d2cd78 --- /dev/null +++ b/examples/game/server/examples/mobpit-recovery.rs @@ -0,0 +1,529 @@ +//! Crash recovery on a celld fleet: how long until every Mob Pit room takes +//! a player again after celld dies, and how the rooms tick afterwards. +//! +//! 1. Wake: one moving bot per room (`--rooms`, a grid from +//! `pit.{x0}.{y0}`) plays for `--wake-secs`. +//! 2. Kill: runs `--kill-cmd`, or (bots in a cluster, the kill from outside) +//! prints `READY` and waits for the bots to lose their sockets. The first +//! lost socket is t0. +//! 3. Recover: from t0 every room is probed with a real join (Hello with a +//! ticket, until Welcome; retried every 500 ms). A room's recovery time +//! is t0 → its first Welcome; then a bot stays in it (players come back). +//! celld's health URL is polled alongside (`--health-url`). +//! 4. Hold: once every room answered (or `--max-secs`), each room's ticks +//! over `--hold-secs` (dropped %, late p99) from `/zones/{room}/info`. +//! +//! Prints the per-room distribution (p50/p90/max), the order rooms came +//! back in (per 10 s), and with `--json` every room's numbers. +//! +//! ```sh +//! mobpit-recovery --url ws://celld.mobpit-celld.svc.cluster.local:8080 \ +//! --secret "$GAME_TICKET_SECRET" --rooms 64 --wake-secs 60 +//! ``` + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use clap::Parser; +use distributed_game::authoring::GameInput; +use distributed_game::bots::{run_bot, BotConfig, Circle}; +use distributed_game::net::TicketKey; +use game_demos::mobpit::encode_bot_input; +use serde_json::{json, Value}; +use tokio::time::Instant; + +#[derive(Parser, Debug)] +#[command(about = "Time Mob Pit rooms' recovery after celld dies")] +struct Args { + /// celld public listener (ws://…). + #[arg(long)] + url: String, + /// Room info base (http://…); default: `--url` with http. + #[arg(long)] + info_url: Option, + /// Ticket secret of the worker. + #[arg(long, env = "GAME_TICKET_SECRET")] + secret: String, + /// Rooms (one bot each). + #[arg(long, default_value_t = 64)] + rooms: usize, + /// Grid origin: rooms are `pit.{x0 + i % side}.{y0 + i / side}`. + #[arg(long, default_value_t = 0, allow_hyphen_values = true)] + x0: i32, + #[arg(long, default_value_t = 0, allow_hyphen_values = true)] + y0: i32, + /// Seconds the bots play before the kill. + #[arg(long, default_value_t = 60)] + wake_secs: u64, + /// Seconds over which the wake bots join. + #[arg(long, default_value_t = 10)] + ramp_secs: u64, + /// Shell command that kills celld (else wait for an outside kill). + #[arg(long)] + kill_cmd: Option, + /// How long to wait for an outside kill. + #[arg(long, default_value_t = 900)] + kill_wait_secs: u64, + /// Give up on rooms that have not answered this long after t0. + #[arg(long, default_value_t = 900)] + max_secs: u64, + /// Tick window once the rooms are back. + #[arg(long, default_value_t = 30)] + hold_secs: u64, + /// Polled during recovery (first 200 after t0 is reported). + #[arg(long)] + health_url: Option, + /// Move without holding attack. + #[arg(long)] + no_attack: bool, + /// Hello content version (the game's; a room on another one reads the + /// world's content rollout before it admits the player). + #[arg(long, default_value = "emberfall")] + content_version: String, + /// Write the result as JSON here. + #[arg(long)] + json: Option, +} + +#[derive(Default, Clone)] +struct Room { + zone: String, + /// t0 → Welcome. + recovered: Option, + attempts: u32, + last_error: Option, + /// Join latency of the successful attempt (connect → Welcome). + join: Option, +} + +fn unix_secs() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()) +} + +fn grid(n: usize, x0: i32, y0: i32) -> Vec { + let side = (1..).find(|s| s * s >= n).unwrap_or(1); + (0..n) + .map(|i| format!("pit.{}.{}", x0 + (i % side) as i32, y0 + (i / side) as i32)) + .collect() +} + +fn config(args: &Args, zone: &str, prefix: String, seed: u64, duration: Duration) -> BotConfig { + let mut config = BotConfig::new( + args.url.clone(), + zone.to_owned(), + TicketKey::new(args.secret.as_bytes()), + ); + config.path = "/zones".into(); + config.bots = 1; + config.input_rate = 30.0; + config.duration = duration; + config.principal_prefix = prefix; + config.seed = seed; + config.content_version = args.content_version.clone(); + config.encoder = if args.no_attack { + Arc::new(encode_move) + } else { + Arc::new(encode_bot_input) + }; + config +} + +fn encode_move(dir: [f32; 2]) -> Vec { + let axis = |v: f32| (v.clamp(-1.0, 1.0) * 127.0).round() as i8; + game_sim::mobpit::PitInput { + mx: axis(dir[0]), + my: axis(dir[1]), + ..Default::default() + } + .to_payload() +} + +fn circle(index: usize) -> Box { + Box::new(Circle { + period: Duration::from_millis(3000 + 250 * (index % 8) as u64), + phase: index as f32 * 0.7, + }) +} + +const FOREVER: Duration = Duration::from_secs(24 * 3600); + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<(), String> { + let args = Arc::new(Args::parse()); + let info_base = args.info_url.clone().unwrap_or_else(|| { + args.url + .replacen("ws://", "http://", 1) + .replacen("wss://", "https://", 1) + }); + let zones = grid(args.rooms, args.x0, args.y0); + let run = unix_secs() as u64; + + // 1. Wake. + println!( + "waking {} rooms ({} … {}) for {} s", + zones.len(), + zones[0], + zones[zones.len() - 1], + args.wake_secs + ); + let (lost_tx, mut lost_rx) = tokio::sync::mpsc::unbounded_channel::<(usize, Instant)>(); + let mut wake = Vec::new(); + for (i, zone) in zones.iter().enumerate() { + let delay = args.ramp_secs as f64 * i as f64 / zones.len().max(1) as f64; + let config = config(&args, zone, format!("rw{run}z{i}"), i as u64 + 1, FOREVER); + let lost = lost_tx.clone(); + wake.push(tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs_f64(delay)).await; + let stats = run_bot(&config, 0, circle(i)).await; + let _ = lost.send((i, Instant::now())); + stats + })); + } + tokio::time::sleep(Duration::from_secs(args.ramp_secs + 3)).await; + let infos = get_all(&info_base, &zones).await; + let players: u64 = infos + .iter() + .map(|v| v["players"].as_u64().unwrap_or(0)) + .sum(); + let restored = infos.iter().filter(|v| v["restored"] == true).count(); + println!( + "{players} players in {} rooms ({restored} restored a snapshot)", + zones.len() + ); + tokio::time::sleep(Duration::from_secs(args.wake_secs.saturating_sub(3))).await; + // A bot that already left (a room refused it) is not the kill. + let mut early = 0; + while let Ok((i, _)) = lost_rx.try_recv() { + early += 1; + eprintln!("{} lost its bot before the kill", zones[i]); + } + if early > 0 { + eprintln!("{early} rooms lost their bot before the kill"); + } + + // 2. Kill. + let kill_unix; + if let Some(cmd) = &args.kill_cmd { + kill_unix = unix_secs(); + let out = tokio::process::Command::new("sh") + .arg("-c") + .arg(cmd) + .output() + .await + .map_err(|e| format!("kill: {e}"))?; + println!("kill: {}", String::from_utf8_lossy(&out.stdout).trim()); + } else { + kill_unix = 0.0; + println!("READY {:.3}", unix_secs()); + } + let wait = Duration::from_secs(args.kill_wait_secs); + let (first, t0) = tokio::time::timeout(wait, lost_rx.recv()) + .await + .map_err(|_| "no bot lost its socket: celld was not killed")? + .ok_or("bots gone")?; + let t0_unix = unix_secs(); + println!( + "t0 {t0_unix:.3}: {} lost its socket{}", + zones[first], + if kill_unix > 0.0 { + format!(" ({:.1} s after the kill command)", t0_unix - kill_unix) + } else { + String::new() + } + ); + for task in &wake { + task.abort(); + } + + // 3. Recover. + let rooms: Arc>> = Arc::new(Mutex::new( + zones + .iter() + .map(|z| Room { + zone: z.clone(), + ..Room::default() + }) + .collect(), + )); + let health: Arc>> = Arc::new(Mutex::new(None)); + let health_task = args.health_url.clone().map(|url| { + let health = health.clone(); + tokio::spawn(async move { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .expect("http client"); + loop { + let ok = client + .get(&url) + .send() + .await + .is_ok_and(|r| r.status().is_success()); + if ok { + *health.lock().unwrap() = Some(t0.elapsed()); + return; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + }); + let deadline = t0 + Duration::from_secs(args.max_secs); + let (back_tx, mut back_rx) = tokio::sync::mpsc::unbounded_channel::(); + // Rooms whose player was dropped again after it came back. + let relost: Arc>> = Arc::new(Mutex::new(Vec::new())); + let mut stay = Vec::new(); + for (i, zone) in zones.iter().enumerate() { + let rooms = rooms.clone(); + let args = args.clone(); + let zone = zone.clone(); + let back = back_tx.clone(); + let relost = relost.clone(); + stay.push(tokio::spawn(async move { + let mut attempt = 0u32; + loop { + if Instant::now() > deadline { + return; + } + attempt += 1; + let started = Instant::now(); + let probe = config( + &args, + &zone, + format!("rp{run}z{i}a{attempt}"), + i as u64 + 1, + Duration::ZERO, + ); + let stats = + tokio::time::timeout(Duration::from_secs(15), run_bot(&probe, 0, circle(i))) + .await; + let ok = match stats { + Ok(stats) if stats.joined => { + let join = stats.joined_after.unwrap_or_default(); + let mut rooms = rooms.lock().unwrap(); + let room = &mut rooms[i]; + room.recovered = Some((started + join) - t0); + room.join = Some(join); + room.attempts = attempt; + true + } + Ok(stats) => { + let mut rooms = rooms.lock().unwrap(); + rooms[i].last_error = stats.error; + rooms[i].attempts = attempt; + false + } + Err(_) => { + let mut rooms = rooms.lock().unwrap(); + rooms[i].last_error = Some("no Welcome in 15 s".into()); + rooms[i].attempts = attempt; + false + } + }; + if ok { + let _ = back.send(i); + break; + } + let next = started + Duration::from_millis(500); + tokio::time::sleep_until(next).await; + } + // Back: a player stays (until the hold ends). + let config = config(&args, &zone, format!("rs{run}z{i}"), i as u64 + 7, FOREVER); + let stats = run_bot(&config, 0, circle(i)).await; + relost.lock().unwrap().push(( + i, + t0.elapsed().as_secs_f64(), + stats.error.unwrap_or_else(|| "no Welcome".into()), + )); + })); + } + drop(back_tx); + let mut back = 0; + let mut last_print = Instant::now(); + while back < zones.len() { + match tokio::time::timeout_at(deadline, back_rx.recv()).await { + Ok(Some(_)) => back += 1, + Ok(None) | Err(_) => break, + } + if last_print.elapsed() > Duration::from_secs(10) { + last_print = Instant::now(); + println!( + " {:>5.0} s: {back}/{} rooms back", + t0.elapsed().as_secs_f64(), + zones.len() + ); + } + } + let all_back = t0.elapsed(); + if let Some(task) = health_task { + task.abort(); + } + println!( + "{back}/{} rooms back after {:.1} s", + zones.len(), + all_back.as_secs_f64() + ); + + // 4. Hold. + tokio::time::sleep(Duration::from_secs(5)).await; + let start = get_all(&info_base, &zones).await; + tokio::time::sleep(Duration::from_secs(args.hold_secs)).await; + let end = get_all(&info_base, &zones).await; + for task in &stay { + task.abort(); + } + + let rooms = rooms.lock().unwrap().clone(); + let relost = relost.lock().unwrap().clone(); + if !relost.is_empty() { + let first = relost.iter().map(|r| r.1).fold(f64::MAX, f64::min); + println!( + "{} rooms lost their player again after recovering (first at {first:.1} s: {} — {})", + relost.len(), + zones[relost[0].0], + relost[0].2 + ); + } + let mut times: Vec = rooms + .iter() + .filter_map(|r| r.recovered.map(|d| d.as_secs_f64())) + .collect(); + times.sort_by(f64::total_cmp); + let q = |q: f64| -> Option { + let i = ((times.len().checked_sub(1)?) as f64 * q).round() as usize; + times.get(i).copied() + }; + let fmt = |v: Option| v.map_or("—".to_owned(), |v| format!("{v:.1}")); + let health = *health.lock().unwrap(); + println!( + "recovery s: min {} p10 {} p50 {} p90 {} max {} · celld health {}", + fmt(q(0.0)), + fmt(q(0.1)), + fmt(q(0.5)), + fmt(q(0.9)), + fmt(q(1.0)), + fmt(health.map(|d| d.as_secs_f64())) + ); + // Rooms back per 10 s. + let mut buckets: Vec = Vec::new(); + for t in × { + let b = (*t / 10.0) as usize; + if buckets.len() <= b { + buckets.resize(b + 1, 0); + } + buckets[b] += 1; + } + let histogram: Vec = buckets + .iter() + .enumerate() + .filter(|(_, n)| **n > 0) + .map(|(b, n)| format!("{}-{}s:{n}", b * 10, b * 10 + 10)) + .collect(); + println!("rooms back per 10 s: {}", histogram.join(" ")); + let missing: Vec<&Room> = rooms.iter().filter(|r| r.recovered.is_none()).collect(); + for r in missing.iter().take(10) { + println!( + " {} never answered ({} attempts; last: {})", + r.zone, + r.attempts, + r.last_error.as_deref().unwrap_or("?") + ); + } + let attempts: u32 = rooms.iter().map(|r| r.attempts).sum(); + let mut errors: std::collections::BTreeMap = Default::default(); + for r in &rooms { + if let Some(e) = &r.last_error { + *errors.entry(short(e)).or_default() += 1; + } + } + println!("{attempts} join attempts; last failure per room: {errors:?}"); + + // Ticks over the hold. + let num = |v: &Value, key: &str| v[key].as_f64().unwrap_or(0.0); + let mut drops = Vec::new(); + let mut lates = Vec::new(); + for (s, e) in start.iter().zip(&end) { + let same = s["activation"] == e["activation"] && !e["activation"].is_null(); + let iterations = num(e, "iterations") - num(s, "iterations"); + if same && iterations > 0.0 { + drops.push((num(e, "droppedTicks") - num(s, "droppedTicks")) / iterations * 100.0); + } + lates.push(e["lateMs"]["p99"].as_f64().unwrap_or(0.0)); + } + drops.sort_by(f64::total_cmp); + lates.sort_by(f64::total_cmp); + let mid = |v: &[f64]| v.get(v.len() / 2).copied(); + println!( + "hold {} s ({} rooms measured): dropped % median {} worst {} · late p99 ms median {} worst {}", + args.hold_secs, + drops.len(), + fmt(mid(&drops).map(|d| d * 10.0).map(|d| d / 10.0)), + fmt(drops.last().copied()), + fmt(mid(&lates)), + fmt(lates.last().copied()) + ); + let restored = end.iter().filter(|v| v["restored"] == true).count(); + println!("{restored}/{} rooms restored a snapshot", zones.len()); + + if let Some(path) = &args.json { + let out = json!({ + "rooms": zones.len(), + "t0": t0_unix, + "killUnix": kill_unix, + "allBackSecs": all_back.as_secs_f64(), + "healthSecs": health.map(|d| d.as_secs_f64()), + "recoveredSecs": rooms.iter().map(|r| json!({ + "zone": r.zone, + "secs": r.recovered.map(|d| d.as_secs_f64()), + "joinMs": r.join.map(|d| d.as_secs_f64() * 1000.0), + "attempts": r.attempts, + "lastError": r.last_error, + })).collect::>(), + "holdDroppedPct": drops, + "holdLateP99Ms": lates, + "relost": relost.iter().map(|r| json!({"zone": zones[r.0], "secs": r.1, "error": r.2})).collect::>(), + "infoEnd": end, + }); + std::fs::write(path, serde_json::to_string_pretty(&out).unwrap_or_default()) + .map_err(|e| format!("{}: {e}", path.display()))?; + } + Ok(()) +} + +/// An error without its ids (for counting kinds). +fn short(e: &str) -> String { + e.chars() + .take_while(|c| *c != '(' && *c != ':') + .collect::() + .trim() + .to_owned() + + &e.split_once(':') + .map(|(_, rest)| format!(":{}", rest.chars().take(40).collect::())) + .unwrap_or_default() +} + +async fn get_all(base: &str, zones: &[String]) -> Vec { + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("http client"); + let tasks: Vec<_> = zones + .iter() + .map(|zone| { + let client = client.clone(); + let url = format!("{base}/zones/{zone}/info"); + tokio::spawn(async move { + match client.get(&url).send().await { + Ok(r) => r.json::().await.unwrap_or(Value::Null), + Err(_) => Value::Null, + } + }) + }) + .collect(); + let mut out = Vec::with_capacity(tasks.len()); + for task in tasks { + out.push(task.await.unwrap_or(Value::Null)); + } + out +} From a0d20519b1ef748c0c851756f1e2ac01b46f0b03 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:27:36 -0500 Subject: [PATCH 09/22] feat(cell): run a fact batch under one command guard `AggregateCell::dispatch_idempotent_batch` runs commands in order, each exactly as `dispatch_idempotent` (own ledger entry, identity, session), under one command guard and one alarm reschedule instead of one per command. A rejected command leaves the earlier ones committed and the later ones run; the first retryable result (in progress, 408, 429, 5xx) ends the batch; one result per command attempted. The `__batch` wire moves to `distributed_game::cell::run_fact_batch` (host-testable); `serve_cell` parses, calls it and drains the outbox once. An entry that does not parse is answered in its place. Each entry is still its own transaction: a handler runs between its reservation and its commit, outside the synchronous cell SQL transaction, and must see the entries before it committed. Tests: a batch runs in order and replays as a whole; a rejected entry leaves the others committed; a replayed `__batch` body answers only replays and credits once. [[zone-fact-batches]] --- distributed_game/src/cell.rs | 108 ++++++++++++++++++++++- distributed_game/src/celld_zone/cells.rs | 74 +++------------- distributed_game/tests/cell_sink/main.rs | 79 ++++++++++++++++- src/microsvc/cell_host/cell.rs | 65 ++++++++++++++ src/microsvc/cell_host/mod.rs | 2 +- src/microsvc/cell_host/tests.rs | 96 +++++++++++++++++++- 6 files changed, 353 insertions(+), 71 deletions(-) diff --git a/distributed_game/src/cell.rs b/distributed_game/src/cell.rs index d12ba304e..9d17e21f8 100644 --- a/distributed_game/src/cell.rs +++ b/distributed_game/src/cell.rs @@ -20,12 +20,13 @@ #[cfg(not(target_arch = "wasm32"))] use std::sync::Arc; -#[cfg(not(target_arch = "wasm32"))] -use distributed::cell_host::{AggregateCell, CellCommandIdentity, CellDispatchError}; +use distributed::cell_host::{ + AggregateCell, CellBatchCommand, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, +}; use distributed::microsvc::Session; -#[cfg(not(target_arch = "wasm32"))] use distributed::Aggregate; use distributed::USER_ID_KEY; +use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use crate::facts::Fact; @@ -125,6 +126,107 @@ pub fn session_stamp(session: &Session) -> Option { crate::FactStamp::from_session_vars(session.variables()) } +/// Run a fact batch (`POST /{kind}/{shard}/__batch`, the celld object +/// host's batch request) against `cell`: `{ entries: [{ command, +/// commandId, input, partition, session }] }` → `{ results: [{ status, +/// body }] }`, each body as the entry's own wait-path response. Every entry +/// gets its own ledger id (`service` + `partition` + `commandId`) and its +/// session from its `session` variables through `sessions`, and runs in +/// order ([`AggregateCell::dispatch_idempotent_batch`]); the results end at +/// the first retryable one. Call it only after the request authenticated +/// with `DISTRIBUTED_INTERNAL_SECRET`. +pub async fn run_fact_batch( + cell: &AggregateCell, + service: &str, + body: &Value, + sessions: impl Fn(&dyn Fn(&str) -> Option) -> Option, +) -> Result +where + A: Aggregate + Send + Sync + 'static, +{ + let entries = body + .get("entries") + .and_then(Value::as_array) + .ok_or_else(|| CellDispatchError::BadRequest("`entries` is required".into()))?; + // An entry that does not parse never reaches the cell; it is answered + // in its place. + let mut refused = Vec::with_capacity(entries.len()); + let mut commands = Vec::with_capacity(entries.len()); + for entry in entries { + match batch_command(service, entry, &sessions) { + Ok(command) => { + commands.push(command); + refused.push(None); + } + Err(error) => refused.push(Some(error)), + } + } + let mut dispatched = cell.dispatch_idempotent_batch(commands).await.into_iter(); + let mut results = Vec::with_capacity(entries.len()); + for refused in refused { + let result = match refused { + Some(error) => Err(error), + None => match dispatched.next() { + Some(result) => result.map(|dispatch| dispatch.wait_path_wire()), + // Not attempted: an earlier entry ended the batch. + None => break, + }, + }; + let (status, body) = match result { + Ok(body) => (200, body), + Err(error) => ( + error.status_code(), + json!({ "error": error.client_message(), "code": error.code() }), + ), + }; + let retryable = status >= 500 + || matches!(status, 408 | 429) + || body.get("code").and_then(Value::as_str) == Some("COMMAND_IN_PROGRESS"); + results.push(json!({ "status": status, "body": body })); + if retryable { + break; + } + } + Ok(json!({ "results": results })) +} + +/// One `__batch` entry as its own wait-path command. +fn batch_command( + service: &str, + entry: &Value, + sessions: &impl Fn(&dyn Fn(&str) -> Option) -> Option, +) -> Result { + let field = |name: &str| entry.get(name).and_then(Value::as_str); + let (Some(command), Some(partition)) = (field("command"), field("partition")) else { + return Err(CellDispatchError::BadRequest( + "`command` and `partition` are required".into(), + )); + }; + let parsed = CellWaitPathRequest::parse(json!({ + "commandId": entry.get("commandId"), + "input": entry.get("input"), + })) + .map_err(CellDispatchError::BadRequest)?; + let identity = CellCommandIdentity::new( + service.to_string(), + partition.to_string(), + &parsed.command_id, + )?; + let vars = entry.get("session").and_then(Value::as_object); + let header = |name: &str| { + vars.and_then(|vars| vars.get(name)) + .and_then(Value::as_str) + .map(str::to_string) + }; + let session = sessions(&header).ok_or(CellDispatchError::Forbidden)?; + Ok(CellBatchCommand { + command: command.to_string(), + identity, + input: parsed.input, + session, + }) +} + #[cfg(not(target_arch = "wasm32"))] fn classify(error: CellDispatchError) -> FactError { match error { diff --git a/distributed_game/src/celld_zone/cells.rs b/distributed_game/src/celld_zone/cells.rs index 1da7663ec..8d53cc8f2 100644 --- a/distributed_game/src/celld_zone/cells.rs +++ b/distributed_game/src/celld_zone/cells.rs @@ -14,7 +14,7 @@ use distributed::Aggregate; use serde_json::{json, Value}; use worker::{Env, Headers, Method, Request, RequestInit, Response, Result, State}; -use crate::cell::fact_session_from_headers; +use crate::cell::{fact_session_from_headers, run_fact_batch}; use crate::world::content_rollout::{mount_content_rollout, ContentRollout}; use crate::world::watch::{ parse_watcher_address, topic, watch_query, WatchUpdate, Watchers, PUSH_CONCURRENCY, PUSH_ROUTE, @@ -80,11 +80,10 @@ pub const BATCH_ROUTE: &str = "__batch"; /// from the service-id and principal-partition headers, the session from /// `sessions`), then a best-effort outbox drain. `POST /// /{kind}/{shard}/__batch` ([`BATCH_ROUTE`]) runs a zone's consecutive -/// facts for this cell in one request: `{ entries: [{ command, commandId, -/// input, partition, session }] }` → `{ results: [{ status, body }] }`, -/// each entry exactly as its own wait-path command (its own ledger id and -/// session), in order, stopping after the first retryable result, then one -/// outbox drain. Every request must carry the internal secret. +/// facts for this cell in one request ([`run_fact_batch`]: each entry as +/// its own wait-path command, with its own ledger id and session, in order, +/// stopping after the first retryable result), then one outbox drain. +/// Every request must carry the internal secret. pub async fn serve_cell( env: &Env, cell: &AggregateCell, @@ -173,67 +172,14 @@ where Ok(body) => body, Err(_) => return error_response(CellDispatchError::BadRequest("invalid JSON".into())), }; - let Some(entries) = body.get("entries").and_then(Value::as_array) else { - return error_response(CellDispatchError::BadRequest( - "`entries` is required".into(), - )); - }; - let mut results = Vec::with_capacity(entries.len()); - for entry in entries { - let (status, body) = match batch_entry(cell, &service, entry, sessions).await { - Ok(body) => (200, body), - Err(error) => (error.status_code(), error_body(&error)), - }; - let retryable = status >= 500 - || matches!(status, 408 | 429) - || body.get("code").and_then(Value::as_str) == Some("COMMAND_IN_PROGRESS"); - results.push(json!({ "status": status, "body": body })); - if retryable { - break; - } - } + let results = run_fact_batch(cell, &service, &body, sessions).await; if let Err(error) = cell.drain_outbox(env).await { worker::console_error!("post-commit drain deferred: {error}"); } - json_status(json!({ "results": results }), 200) -} - -/// One `__batch` entry as its own wait-path command. -async fn batch_entry( - cell: &AggregateCell, - service: &str, - entry: &Value, - sessions: Sessions, -) -> std::result::Result -where - A: Aggregate + Send + Sync + 'static, -{ - let field = |name: &str| entry.get(name).and_then(Value::as_str); - let (Some(command), Some(partition)) = (field("command"), field("partition")) else { - return Err(CellDispatchError::BadRequest( - "`command` and `partition` are required".into(), - )); - }; - let parsed = CellWaitPathRequest::parse(json!({ - "commandId": entry.get("commandId"), - "input": entry.get("input"), - })) - .map_err(CellDispatchError::BadRequest)?; - let identity = CellCommandIdentity::new( - service.to_string(), - partition.to_string(), - &parsed.command_id, - )?; - let vars = entry.get("session").and_then(Value::as_object); - let header = |name: &str| { - vars.and_then(|vars| vars.get(name)) - .and_then(Value::as_str) - .map(str::to_string) - }; - let session = sessions(&header).ok_or(CellDispatchError::Forbidden)?; - cell.dispatch_idempotent(command, &identity, parsed.input, session) - .await - .map(|dispatch| dispatch.wait_path_wire()) + match results { + Ok(results) => json_status(results, 200), + Err(error) => error_response(error), + } } /// A watched world cell's watchers ([`crate::world::watch`]): zones that diff --git a/distributed_game/tests/cell_sink/main.rs b/distributed_game/tests/cell_sink/main.rs index 2bb062127..ed730e5cf 100644 --- a/distributed_game/tests/cell_sink/main.rs +++ b/distributed_game/tests/cell_sink/main.rs @@ -12,7 +12,9 @@ use distributed::microsvc::{ use distributed::{sourced, Entity}; use std::collections::HashMap; -use distributed_game::cell::{command_id_for, session_stamp, CellFactSink}; +use distributed_game::cell::{ + command_id_for, fact_session_from_headers, run_fact_batch, session_stamp, CellFactSink, +}; use distributed_game::{ fact_channel, spawn_fact_worker, stamp_metadata, Fact, FactOutcome, FactStamp, RetryPolicy, }; @@ -178,6 +180,81 @@ async fn facts_dispatch_idempotently_with_trusted_principal() { worker.await.unwrap(); } +/// A `__batch` body of `amounts` for alice, one entry per amount; `None` +/// is an entry without a partition. +fn batch_body(amounts: &[Option]) -> serde_json::Value { + let entries: Vec<_> = amounts + .iter() + .enumerate() + .map(|(i, amount)| { + let mut session = serde_json::Map::new(); + session.insert("x-user-id".into(), json!("alice")); + let stamp = FactStamp { + zone_id: "z1".into(), + tick: i as u64, + ..FactStamp::default() + }; + for (name, value) in stamp.session_vars() { + session.insert(name, json!(value)); + } + let mut entry = json!({ + "command": "hero.credit_xp", + "commandId": command_id_for("game-svc", &format!("kill:z1:{i}")), + "input": { "amount": amount.unwrap_or(1) }, + "session": session, + }); + if amount.is_some() { + entry["partition"] = json!("alice"); + } + entry + }) + .collect(); + json!({ "entries": entries }) +} + +#[tokio::test] +async fn a_replayed_batch_answers_only_replays() { + let cell = AggregateCell::::new("alice").unwrap().mount(CreditXp); + let sessions = |header: &dyn Fn(&str) -> Option| fact_session_from_headers(header); + let body = batch_body(&[Some(10), Some(20), None, Some(30)]); + + let first = run_fact_batch(&cell, "game-svc", &body, sessions) + .await + .unwrap(); + let first = first["results"].as_array().unwrap().clone(); + let statuses: Vec<_> = first + .iter() + .map(|r| r["status"].as_u64().unwrap()) + .collect(); + // The entry without a partition is refused in its place; the rest commit. + assert_eq!(statuses, [200, 200, 400, 200]); + let xp: Vec<_> = first + .iter() + .map(|r| r["body"]["payload"]["xp"].clone()) + .collect(); + assert_eq!(xp, [json!(10), json!(30), json!(null), json!(60)]); + assert!(first + .iter() + .all(|r| r["body"]["receipt"]["replayed"] != json!(true))); + + // The whole batch again, same command ids: replays only, nothing new. + let replay = run_fact_batch(&cell, "game-svc", &body, sessions) + .await + .unwrap(); + let replay = replay["results"].as_array().unwrap().clone(); + assert_eq!(replay.len(), 4); + for (i, (replayed, original)) in replay.iter().zip(&first).enumerate() { + assert_eq!(replayed["status"], original["status"]); + if i == 2 { + continue; + } + assert_eq!(replayed["body"]["receipt"]["replayed"], json!(true)); + assert_eq!(replayed["body"]["payload"], original["body"]["payload"]); + } + assert_eq!(cell.load().await.unwrap().unwrap().xp, 60, "credited once"); + assert_eq!(cell.history().await.unwrap().len(), 3); +} + async fn wait_one(queue: &mut distributed_game::FactQueue) -> distributed_game::FactSettlement { for _ in 0..500 { if let Some(s) = queue.drain().pop() { diff --git a/src/microsvc/cell_host/cell.rs b/src/microsvc/cell_host/cell.rs index 9fcbb54dc..159f260f2 100644 --- a/src/microsvc/cell_host/cell.rs +++ b/src/microsvc/cell_host/cell.rs @@ -252,6 +252,51 @@ where result } + /// Dispatch `commands` in order, each exactly as + /// [`dispatch_idempotent`](Self::dispatch_idempotent) (its own ledger + /// entry, identity and session), under one command guard and one alarm + /// reschedule. A rejected command leaves the earlier ones committed and + /// the later ones still run; the first retryable result (in progress, + /// 408, 429, 5xx) ends the batch. One result per command attempted. + /// + /// Each command is still its own transaction: a handler runs between + /// its ledger reservation and its commit, outside the synchronous cell + /// SQL transaction, and must see the commands before it committed. + pub async fn dispatch_idempotent_batch( + &self, + commands: Vec, + ) -> Vec> { + if commands.is_empty() { + return Vec::new(); + } + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + let _command = match self.begin_command().await { + Ok(guard) => guard, + Err(error) => return vec![Err(CellDispatchError::Internal(error.to_string()))], + }; + let mut results = Vec::with_capacity(commands.len()); + for command in commands { + let result = self + .routes + .dispatch_cell_causal( + &command.command, + &command.identity, + command.input, + command.session, + &self.shard, + ) + .await; + let stop = result.as_ref().is_err_and(ends_batch); + results.push(result); + if stop { + break; + } + } + #[cfg(all(feature = "workers-rs", target_arch = "wasm32"))] + self.reschedule_after_command().await; + results + } + /// Armed alarms of this cell, earliest first. pub fn armed_schedules(&self) -> Result, RepositoryError> { self.routes.repo().repo().armed_schedules() @@ -686,6 +731,26 @@ where } } +/// One command of [`AggregateCell::dispatch_idempotent_batch`]. +#[derive(Debug, Clone)] +pub struct CellBatchCommand { + /// Mounted command name. + pub command: String, + /// Its ledger identity (service, principal partition, command id). + pub identity: CellCommandIdentity, + /// Command input. + pub input: Value, + /// Its trusted session. + pub session: Session, +} + +/// A batch stops at a result worth retrying: the command is in progress, +/// timed out, throttled or failed inside the cell. +fn ends_batch(error: &CellDispatchError) -> bool { + let status = error.status_code(); + status >= 500 || matches!(status, 408 | 429) || error.code() == "COMMAND_IN_PROGRESS" +} + /// Worker-side namespace: `getByName(format!("{}:{}", type, shard))`. pub struct CellNamespace where diff --git a/src/microsvc/cell_host/mod.rs b/src/microsvc/cell_host/mod.rs index d178d2432..e5ccac1b3 100644 --- a/src/microsvc/cell_host/mod.rs +++ b/src/microsvc/cell_host/mod.rs @@ -45,7 +45,7 @@ pub use causal::{ CellCommandIdentity, CellDispatchError, CellDispatchResult, CELL_CAUSATION_ID_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, }; -pub use cell::{instance_name, parent_cell_name, AggregateCell, CellNamespace}; +pub use cell::{instance_name, parent_cell_name, AggregateCell, CellBatchCommand, CellNamespace}; #[cfg(feature = "workers-rs")] pub use celld_outbox::{ CelldAlarmOutcome, CelldOutbox, CelldOutboxDrainOutcome, CELLD_OUTBOX_DEFAULT_BATCH_SIZE, diff --git a/src/microsvc/cell_host/tests.rs b/src/microsvc/cell_host/tests.rs index 389367969..fdad0542d 100644 --- a/src/microsvc/cell_host/tests.rs +++ b/src/microsvc/cell_host/tests.rs @@ -1,6 +1,6 @@ use super::{ - instance_name, parent_cell_name, AggregateCell, CellCommandIdentity, CellDispatchError, - CellNamespace, CellStreamStore, DURABLE_AGGREGATE_CELL_STATE_VERSION, + instance_name, parent_cell_name, AggregateCell, CellBatchCommand, CellCommandIdentity, + CellDispatchError, CellNamespace, CellStreamStore, DURABLE_AGGREGATE_CELL_STATE_VERSION, }; use crate::aggregate::{Aggregate, AggregateRepository}; use crate::command::{typed_command, PreparedCommand, Succeeded}; @@ -455,6 +455,98 @@ async fn cell_wait_path_replays_the_same_command_without_new_domain_effects() { ); } +fn batch_command(command: &str, id: &str, input: serde_json::Value) -> CellBatchCommand { + CellBatchCommand { + command: command.into(), + identity: CellCommandIdentity::new("cell-test-service", "principal-alice", id).unwrap(), + input, + session: owner_session(), + } +} + +fn event_count(cell: &AggregateCell) -> usize { + cell.durable_events() + .unwrap() + .iter() + .map(|stream| stream.events.len()) + .sum() +} + +#[tokio::test] +async fn a_batch_runs_in_order_and_replays_as_a_whole() { + let cell = AggregateCell::::new("item-batch") + .unwrap() + .mount(Create) + .mount(Complete); + let batch = || { + vec![ + batch_command( + "cell_item.create", + "0190a000-0000-7000-8000-000000000410", + json!({ "id": "item-batch", "title": "once" }), + ), + // Sees the create before it. + batch_command( + "cell_item.complete", + "0190a000-0000-7000-8000-000000000411", + json!({ "id": "item-batch" }), + ), + ] + }; + let first = cell.dispatch_idempotent_batch(batch()).await; + assert_eq!(first.len(), 2); + assert!(first.iter().all(|r| !r.as_ref().unwrap().replayed())); + assert_eq!(first[1].as_ref().unwrap().payload()["done"], json!(true)); + assert_eq!(event_count(&cell), 2); + + let replay = cell.dispatch_idempotent_batch(batch()).await; + assert_eq!(replay.len(), 2); + for (replayed, original) in replay.iter().zip(&first) { + let (replayed, original) = (replayed.as_ref().unwrap(), original.as_ref().unwrap()); + assert!(replayed.replayed()); + assert_eq!(replayed.payload(), original.payload()); + assert_eq!(replayed.causation_id(), original.causation_id()); + } + assert_eq!(event_count(&cell), 2, "a replayed batch appends nothing"); +} + +#[tokio::test] +async fn a_rejected_batch_entry_leaves_the_others_committed() { + let cell = AggregateCell::::new("item-batch-reject") + .unwrap() + .mount(Create) + .mount(Complete); + let create = json!({ "id": "item-batch-reject", "title": "once" }); + let results = cell + .dispatch_idempotent_batch(vec![ + batch_command( + "cell_item.create", + "0190a000-0000-7000-8000-000000000420", + create.clone(), + ), + // Already exists: rejected by the handler. + batch_command( + "cell_item.create", + "0190a000-0000-7000-8000-000000000421", + create, + ), + batch_command( + "cell_item.complete", + "0190a000-0000-7000-8000-000000000422", + json!({ "id": "item-batch-reject" }), + ), + ]) + .await; + assert_eq!(results.len(), 3); + assert!(results[0].is_ok()); + assert!(matches!( + results[1], + Err(CellDispatchError::Rejected { .. }) + )); + assert!(results[2].is_ok()); + assert_eq!(event_count(&cell), 2); +} + #[tokio::test] async fn cell_wait_path_rejects_command_id_reuse_with_different_input() { let cell = AggregateCell::::new("item-conflict") From d909a0a992aba1b96e2b90bb2403b7f723e0080e Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:27:37 -0500 Subject: [PATCH 10/22] fix(game): never lose a fact to a cell without __batch Mob Pit's crawler and presence cells used their own router, which took `__batch` for a command name and answered 400. The object host only fell back to one-by-one on 404, so it read that 400 as the first fact's permanent refusal: a batch of kills for one crawler dropped its first kill. The cells now use `serve_cell`, and the object host sends the facts one by one whenever the batch request as a whole gets a 4xx other than 408/429, so each fact gets its own answer. [[zone-fact-batches]] --- distributed_game/README.md | 7 +- distributed_game/src/celld_zone/object.rs | 9 ++- examples/game/cells/src/lib.rs | 95 ++--------------------- 3 files changed, 18 insertions(+), 93 deletions(-) diff --git a/distributed_game/README.md b/distributed_game/README.md index 658eb4161..a26ec82d4 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -1188,7 +1188,12 @@ order through the same idempotent wait path and drains the cell outbox once). A batch stops at its first retryable failure and is retried from there; attempts count per fact, so order, exactly-once and `RetryPolicy` mean what they mean for single facts. A cell without the -batch route (404) gets the facts one by one. +batch route, or one refusing the batch request as a whole (a 4xx other +than 408/429), gets the facts one by one. Serve fact cells with +`serve_cell` (or `CellWatchers::serve`), whose `__batch` runs the batch +under one command guard and one alarm reschedule +(`AggregateCell::dispatch_idempotent_batch`); each entry is still its own +ledger reservation and commit. ## Netcode and operations diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index f792cce62..17f7c47f2 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -294,8 +294,8 @@ impl CelldObjectHost { /// `calls` (one aggregate) in one `POST /{kind}/{shard}/__batch` /// ([`super::cells::serve_cell`]); each entry carries its own command - /// id, principal and stamp. A cell without the route gets them one by - /// one. + /// id, principal and stamp. A cell without the route, or one refusing + /// the batch request as a whole (4xx), gets them one by one. async fn post_batch( &self, calls: Vec>, @@ -341,7 +341,10 @@ impl CelldObjectHost { ) .await; match sent { - Ok((404, _)) => { + // No batch route (404; a cell serving only commands answers + // `__batch` 400), or the batch itself refused: each fact gets + // its own answer, never the first fact the batch's. + Ok((status, _)) if (400..500).contains(&status) && !matches!(status, 408 | 429) => { let mut results = Vec::with_capacity(calls.len()); for call in &calls { let result = self.post(call).await; diff --git a/examples/game/cells/src/lib.rs b/examples/game/cells/src/lib.rs index 3e93a96ad..ed855d99a 100644 --- a/examples/game/cells/src/lib.rs +++ b/examples/game/cells/src/lib.rs @@ -49,9 +49,8 @@ use distributed::bus::{CelldQueueEnvelope, CelldQueueHttpPublisher, CelldQueueRelay}; use distributed::cell_host::{ - AggregateCell, CellCommandIdentity, CellDispatchError, CellWaitPathRequest, CelldOutbox, - InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, - CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, + AggregateCell, CellDispatchError, CelldOutbox, InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, + CELL_INTERNAL_SECRET_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, }; use distributed::command::{typed_command, PreparedCommand, Succeeded}; use distributed::microsvc::{ @@ -60,7 +59,8 @@ use distributed::microsvc::{ use distributed::{sourced, Aggregate, Entity}; use distributed_game::cell::{fact_session_from_headers, session_stamp}; use distributed_game::celld_zone::{ - forward_to_zone, CellWatchers, ContentRolloutCell, ZoneActivityCell, ZoneDevSettings, + forward_to_zone, serve_cell, CellWatchers, ContentRolloutCell, ZoneActivityCell, + ZoneDevSettings, }; use distributed_game::lifecycle::{ Lifecycle, CONNECTION_LOST, PLAYER_DISCONNECTED, PLAYER_JOINED, PLAYER_LEFT, PLAYER_RECONNECTED, @@ -247,89 +247,6 @@ fn fact_session(header: &dyn Fn(&str) -> Option) -> Option { fact_session_from_headers(header) } -type Sessions = fn(&dyn Fn(&str) -> Option) -> Option; - -async fn dispatch( - env: &Env, - cell: &AggregateCell, - command: &str, - req: &mut Request, - sessions: Sessions, -) -> Result -where - A: Aggregate + Send + Sync + 'static, -{ - let body: Value = match req.json().await { - Ok(body) => body, - Err(_) => return error_response(CellDispatchError::BadRequest("invalid JSON".into())), - }; - let parsed = match CellWaitPathRequest::parse(body) { - Ok(parsed) => parsed, - Err(error) => return error_response(CellDispatchError::BadRequest(error)), - }; - let header = |name: &str| req.headers().get(name).ok().flatten(); - let identity = match ( - header(CELL_SERVICE_ID_HEADER), - header(CELL_PRINCIPAL_PARTITION_HEADER), - ) { - (Some(service), Some(partition)) => { - match CellCommandIdentity::new(service, partition, &parsed.command_id) { - Ok(identity) => identity, - Err(error) => return error_response(error), - } - } - _ => return error_response(CellDispatchError::Unauthorized), - }; - let Some(session) = sessions(&header) else { - return error_response(CellDispatchError::Forbidden); - }; - let result = cell - .dispatch_idempotent(command, &identity, parsed.input, session) - .await; - if let Err(error) = cell.drain_outbox(env).await { - worker::console_error!("post-commit drain deferred: {error}"); - } - match result { - Ok(dispatch) => json_status(dispatch.wait_path_wire(), 200), - Err(error) => error_response(error), - } -} - -/// `GET /{kind}/{shard}` → `read(cell)`; `POST /{kind}/{shard}/{command}`. -async fn route( - env: &Env, - cell: &AggregateCell, - mut req: Request, - read: F, - sessions: Sessions, -) -> Result -where - A: Aggregate + Send + Sync + 'static, - F: AsyncFnOnce(&AggregateCell) -> std::result::Result, -{ - if let Err(error) = authenticate(&req, env) { - return error_response(error); - } - let url = req.url()?; - let parts: Vec = url - .path() - .split('/') - .filter(|part| !part.is_empty()) - .map(str::to_string) - .collect(); - match (req.method(), parts.get(2)) { - (Method::Get, None) => match read(cell).await { - Ok(body) => json_status(body, 200), - Err(error) => error_response(CellDispatchError::Internal(error)), - }, - (Method::Post, Some(command)) => { - let command = command.clone(); - dispatch(env, cell, &command, &mut req, sessions).await - } - _ => json_status(json!({ "error": "not found" }), 404), - } -} - async fn read_crawler(cell: &AggregateCell) -> std::result::Result { let state = cell.load().await.map_err(|error| error.to_string())?; let mut kills = Vec::new(); @@ -383,7 +300,7 @@ impl DurableObject for CrawlerCell { } async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_crawler, fact_session).await + serve_cell(&self.env, &self.cell, req, read_crawler, fact_session).await } async fn alarm(&self) -> Result { @@ -412,7 +329,7 @@ impl DurableObject for PresenceCell { } async fn fetch(&self, req: Request) -> Result { - route(&self.env, &self.cell, req, read_presence, fact_session).await + serve_cell(&self.env, &self.cell, req, read_presence, fact_session).await } async fn alarm(&self) -> Result { From 1a27ddac87ce2b715f1f1816d034a2ceaf15c123 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:36:04 -0500 Subject: [PATCH 11/22] feat(game): work counters for replication, facts and watches Plain integer counts that do not move with machine load, so A/B runs of one build compare exactly: WorkCounters (records encoded per patch, creates/updates sent, updates held by rates, Delta bytes, fact commands, cell and __batch requests, outbox deletes, world reads and pushes) in GET /zones/{zone}/info as `work`, with the switches in effect; watched cells answer GET /{kind}/{shard}/__watchers with WatcherStats. New A/B switches: ZONE_UPDATE_RATES=0, ZONE_WORLD_PUSH=0. The zone-side reconcile schedule moves into world::watch::WorldSync, with a fake-clock test that a reconcile repairs missed pushes within one period. [[framework-work-counters]] [[zone-world-views-pushed]] [[replication-encode-once-lod]] [[zone-fact-batches]] --- distributed_game/README.md | 23 +++ distributed_game/src/celld_zone/cells.rs | 41 +++- distributed_game/src/celld_zone/mod.rs | 13 +- distributed_game/src/celld_zone/object.rs | 127 +++++++----- distributed_game/src/celld_zone/plan.rs | 26 +++ distributed_game/src/host.rs | 5 + distributed_game/src/lib.rs | 5 +- distributed_game/src/replication.rs | 21 ++ distributed_game/src/work.rs | 202 +++++++++++++++++++ distributed_game/src/world/watch.rs | 229 ++++++++++++++++++++++ 10 files changed, 641 insertions(+), 51 deletions(-) create mode 100644 distributed_game/src/work.rs diff --git a/distributed_game/README.md b/distributed_game/README.md index 71cd9b877..712608405 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -969,6 +969,11 @@ unlimited rewind. inputs by outcome, bytes per Delta, facts pending and settled. - **Load bots**: `game-bots` connects N WebSocket clients and reports round-trip time and bandwidth. +- **Work counters**: `WorkCounters` counts what a zone did (records + encoded, updates sent and held back, Delta bytes, fact commands and + cell requests, outbox deletes, world reads and pushes). Counts do not + move with machine load, so they compare runs exactly where timings + cannot. - **Network simulation**: `NetSim` adds latency, jitter and loss in development, and can be changed at runtime. @@ -1249,6 +1254,24 @@ batch route (404) gets the facts one by one. Only the first `ServerConfig::metrics_zone_labels` (default 64) live zones get a `zone` label; the rest aggregate into `zone="_other"`. `AdvanceReport` carries the raw numbers for custom drivers. +- **Work counters** (`work::WorkCounters`, always on, plain integers): + `record_advance(&report)` adds a patch's records encoded + (`AdvanceReport::encodes`, shared by every client), frames, Delta bytes, + creates, updates, updates held back by update rates + (`DeltaStats::held`) and budget deferrals. A celld zone object keeps one + per object and adds its fact commands, cell requests and `__batch` + requests, outbox deletes, watched-cell reads and pushes received; `GET + /zones/{zone}/info` reports it as `work`, with the A/B switches in effect + (`switches`). A watched cell answers `GET /{kind}/{shard}/__watchers` + (internal secret) with its live watchers and `WatcherStats` (watching + reads, views published, views skipped as unchanged, pushes sent). + Switches (worker vars, default on): `ZONE_FACT_BATCH=1` (one fact per + request), `ZONE_UPDATE_RATES=0` (every visible change every patch), + `ZONE_WORLD_PUSH=0` (zones poll watched cells every second instead of + taking pushes). Mob Pit's `mobpit-load` prints the counters per step, + normalised per room patch, room-second or room-minute, and runs + interleaved A/B (`--ab ZONE_UPDATE_RATES=0 --pairs 3`: A,B then B,A …, + medians per side and of the per-pair change). - **Bots** (`bots`): `run_bots(BotConfig, |i| behaviour)` connects N WebSocket clients with signed tickets, sends inputs (random walk, circle, or your own `Behaviour`), measures RTT and Delta bytes/s: diff --git a/distributed_game/src/celld_zone/cells.rs b/distributed_game/src/celld_zone/cells.rs index 1da7663ec..89b056b76 100644 --- a/distributed_game/src/celld_zone/cells.rs +++ b/distributed_game/src/celld_zone/cells.rs @@ -18,6 +18,7 @@ use crate::cell::fact_session_from_headers; use crate::world::content_rollout::{mount_content_rollout, ContentRollout}; use crate::world::watch::{ parse_watcher_address, topic, watch_query, WatchUpdate, Watchers, PUSH_CONCURRENCY, PUSH_ROUTE, + STATS_ROUTE, }; use crate::world::zone_activity::{mount_zone_activity, ZoneActivity}; @@ -267,7 +268,9 @@ impl CellWatchers { /// [`serve_cell`], plus: a read with `?watch=` leaves a lease and its /// body carries the cell's `version`; a command that changed the view - /// is pushed to the live watchers. + /// is pushed to the live watchers; `GET /{kind}/{shard}/__watchers` + /// ([`STATS_ROUTE`]) answers the live watchers and + /// [`WatcherStats`](crate::world::watch::WatcherStats). pub async fn serve( &self, env: &Env, @@ -285,6 +288,9 @@ impl CellWatchers { return error_response(error); } let url = req.url()?; + if url.path().ends_with(&format!("/{STATS_ROUTE}")) { + return json_status(self.stats(), 200); + } if let Some((watcher, lease)) = url.query().and_then(watch_query) { self.watchers.borrow_mut().watch(&watcher, now_ms(), lease); } @@ -304,6 +310,11 @@ impl CellWatchers { response } + /// The live watchers and what the cell did for them. + fn stats(&self) -> Value { + stats_body(&self.topic, &mut self.watchers.borrow_mut()) + } + /// [`cell_alarm`], then a push if a schedule moved the version. pub async fn alarm(&self, env: &Env, cell: &AggregateCell, read: F) -> Result where @@ -331,8 +342,12 @@ impl CellWatchers { return; }; let version = crate::world::watch::view_version(&view); - if !self.watchers.borrow_mut().publish(version, &view) { - return; + { + let mut watchers = self.watchers.borrow_mut(); + if !watchers.publish(version, &view) { + return; + } + watchers.pushed(live.len()); } let update = WatchUpdate { topic: self.topic.clone(), @@ -343,6 +358,16 @@ impl CellWatchers { } } +/// `{topic, live, ...WatcherStats}`. +fn stats_body(topic: &str, watchers: &mut Watchers) -> Value { + let mut body = serde_json::to_value(watchers.stats()).unwrap_or_else(|_| json!({})); + if let Value::Object(fields) = &mut body { + fields.insert("topic".into(), json!(topic)); + fields.insert("live".into(), json!(watchers.live(now_ms()).len())); + } + body +} + /// `read(cell)` with the aggregate's version in its `version` field. async fn versioned(cell: &AggregateCell, read: &F) -> std::result::Result where @@ -509,7 +534,14 @@ impl ZoneDevSettings { const VERSION: &str = "version"; match req.method() { Method::Get => { - if let Some((watcher, lease)) = req.url()?.query().and_then(watch_query) { + let url = req.url()?; + if url.path().ends_with(&format!("/{STATS_ROUTE}")) { + return json_status( + stats_body(&self.topic, &mut self.watchers.borrow_mut()), + 200, + ); + } + if let Some((watcher, lease)) = url.query().and_then(watch_query) { self.watchers.borrow_mut().watch(&watcher, now_ms(), lease); } let stored: Option = self.state.storage().get(KEY).await.ok().flatten(); @@ -553,6 +585,7 @@ impl ZoneDevSettings { fields.insert(VERSION.into(), json!(version)); } if !live.is_empty() && self.watchers.borrow_mut().publish(version, &view) { + self.watchers.borrow_mut().pushed(live.len()); push_all( self.env.clone(), live, diff --git a/distributed_game/src/celld_zone/mod.rs b/distributed_game/src/celld_zone/mod.rs index 16ad4fb91..acacb909b 100644 --- a/distributed_game/src/celld_zone/mod.rs +++ b/distributed_game/src/celld_zone/mod.rs @@ -109,6 +109,16 @@ //! rollout (each write is pushed); a dev route writes it (one setting for //! every zone, like the native server's `set_net_sim`). //! +//! **Work counters.** `GET …/info` carries the object's +//! [`WorkCounters`](crate::work::WorkCounters) as `work` (records encoded, +//! updates sent and held, Delta bytes, fact commands and `__batch` +//! requests, outbox deletes, world reads and pushes) and the switches in +//! effect; a watched cell answers `GET /{kind}/{shard}/__watchers` with its +//! [`WatcherStats`](crate::world::watch::WatcherStats). A/B switches (worker +//! vars, default on): `ZONE_FACT_BATCH` (1 = one fact per request), +//! `ZONE_UPDATE_RATES=0` (every change every patch), `ZONE_WORLD_PUSH=0` +//! (poll watched cells every second instead of taking pushes). +//! //! Debug only: celld does not tell application code which node runs an //! object. With `CELLD_ZONE_NODES` set (operator listeners) the object asks //! each node whether it holds it; `CELLD_ZONE_NODE` names a fixed node. @@ -123,7 +133,8 @@ mod object; pub use plan::{ cell_binding, ObjectOptions, RoomState, StoredFact, TickSchedule, CLOSE_RESTARTED, - CONTENT_ROLLOUT_VAR, DEV_SETTINGS_VAR, NETSIM_SHARD, ZONE_DEV_KIND, ZONE_HEADER, + CONTENT_ROLLOUT_VAR, DEV_SETTINGS_VAR, NETSIM_SHARD, UPDATE_RATES_VAR, WORLD_PUSH_VAR, + ZONE_DEV_KIND, ZONE_HEADER, }; #[cfg(target_arch = "wasm32")] diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index 14158850d..25498b299 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -36,10 +36,11 @@ use crate::netsim::{DelayLine, NetSim, NetSimJson}; use crate::protocol::{error, ClientMessage, Notice, Role, ServerMessage, PROTOCOL_VERSION}; use crate::stamp::FactStamp; use crate::ticket::{NonceCache, TicketKey, VerifiedTicket}; +use crate::work::WorkCounters; use crate::world::content_rollout::{zone_step, ContentState, ZoneStep, CONTENT_ROLLOUT_KIND}; use crate::world::watch::{ - next_reconcile_ms, topic, view_version, watch_query_string, watcher_address, TopicVersions, - WatchUpdate, Watches, PUSH_ROUTE, WATCH_LEASE_MS, + topic, view_version, watch_query_string, watcher_address, WatchUpdate, Watches, WorldSync, + PUSH_ROUTE, WATCH_LEASE_MS, }; use crate::world::zone_activity::{ zone_principal, RecordActivity, ACTIVITY_ACTIVE, ACTIVITY_DORMANT, ACTIVITY_HIBERNATED, @@ -66,9 +67,6 @@ const FACT_CAPACITY: usize = 512; const OUTBOX_RETRY_MS: f64 = 10_000.0; /// Shortest crash watchdog period (each re-arm is a durable alarm write). const WATCHDOG_MIN_MS: f64 = 60_000.0; -/// Re-read watched cells this often when the object cannot take pushes -/// (no class binding: built with [`ZoneObject::new`] and no `ZONE_BINDING`). -const POLL_MS: f64 = 1_000.0; /// Server → client, through the socket's simulated network. enum OutItem { @@ -255,6 +253,7 @@ pub struct CelldObjectHost { env: SendWrapper, secret: String, service_id: String, + work: SendWrapper>>, } impl CelldObjectHost { @@ -264,9 +263,16 @@ impl CelldObjectHost { env: SendWrapper::new(env), secret: secret.into(), service_id: service_id.into(), + work: SendWrapper::new(Rc::default()), } } + /// Count the facts and requests it sends into `work`. + fn counting(mut self, work: Rc>) -> Self { + self.work = SendWrapper::new(work); + self + } + async fn post(&self, call: &FactCall<'_>) -> std::result::Result { let headers = Headers::new(); let set = |name: &str, value: &str| { @@ -285,6 +291,11 @@ impl CelldObjectHost { } } let body = json!({ "commandId": call.command_id, "input": call.input }); + { + let mut work = self.work.borrow_mut(); + work.fact_commands += 1; + work.fact_requests += 1; + } let (status, body) = self .send(call.kind, &call.shard, call.command, headers, body) .await?; @@ -330,6 +341,12 @@ impl CelldObjectHost { { return vec![Err(FactError::Permanent("batch headers".into()))]; } + { + let mut work = self.work.borrow_mut(); + work.fact_commands += calls.len() as u64; + work.fact_requests += 1; + work.fact_batches += 1; + } let sent = self .send( kind, @@ -443,6 +460,7 @@ impl FactHost for CelldObjectHost { struct DurableFacts { sink: ObjectFactSink, storage: SendWrapper>, + work: SendWrapper>>, } impl FactSink for DurableFacts { @@ -462,6 +480,7 @@ impl FactSink for DurableFacts { Ok(_) => {} } if !matches!(result, Err(FactError::Retryable(_))) { + self.work.borrow_mut().outbox_deletes += 1; let _ = self .storage .delete(&StoredFact::storage_key(fact.key())) @@ -497,6 +516,7 @@ impl FactSink for DurableFacts { settled.push(StoredFact::storage_key(fact.key())); } if !settled.is_empty() { + self.work.borrow_mut().outbox_deletes += 1; let _ = self.storage.delete_multiple(settled).await; } results @@ -645,10 +665,11 @@ struct Inner { /// Watched world cells ([`CelldZone::watch`] plus the dev settings and /// content rollout), built on the first wake. watches: Option>>, - /// Highest version applied per watched topic (this activation). - world_versions: TopicVersions, - /// When each watched topic is read next (ms). - world_due: HashMap, + /// Highest version applied per watched topic (this activation) and + /// when each is read next. + world: WorldSync, + /// Work done by this object (all its activations). + work: Rc>, /// [`CelldZone::refresh`] calls of this activation, and the last one. refresh_round: u64, last_refresh_ms: f64, @@ -972,8 +993,8 @@ impl ZoneObject { drain_at: None, drainer: None, watches: None, - world_versions: TopicVersions::new(), - world_due: HashMap::new(), + world: WorldSync::new(), + work: Rc::default(), refresh_round: 0, last_refresh_ms: 0.0, }; @@ -1243,7 +1264,9 @@ impl ZoneObject { ); return; }; - let host = CelldObjectHost::new(self.env.env().clone(), secret, &service); + let work = self.inner.borrow().work.clone(); + let host = + CelldObjectHost::new(self.env.env().clone(), secret, &service).counting(work.clone()); let sink = match ObjectFactSink::new(host, service, zone_principal(&zone_id)) { Ok(sink) => { Z::fact_routes(sink.route(FactRoute::by_zone::(ZONE_ACTIVITY_KIND))) @@ -1256,6 +1279,7 @@ impl ZoneObject { let durable = DurableFacts { sink, storage: SendWrapper::new(Rc::new(self.state.storage())), + work: SendWrapper::new(work), }; let (queue, feed) = fact_channel(FACT_CAPACITY, retry_policy()); let queue = queue.with_journal(); @@ -1327,6 +1351,9 @@ impl ZoneObject { host_config.idle_tick_rate = None; host_config.reconnect_grace = options.reconnect_grace; host_config.boot_epoch = (now_ms() * 1000.0) as u64; + if !options.update_rates { + host_config.interest.update_rates = crate::UpdateRates::every_patch(); + } // The content version the zone slept on (a rollout it missed is // caught up by its first content rollout read). if let Some(snapshot) = &stored { @@ -1409,7 +1436,7 @@ impl ZoneObject { // content version; the rollout read catches up with one it slept // through (a Hello on another version waits for its own read, see // `hello`). - self.inner.borrow_mut().world_versions.clear(); + self.inner.borrow_mut().world.clear(); self.reconcile_world(true); self.refresh(); Ok(()) @@ -1485,32 +1512,35 @@ impl ZoneObject { topics } + /// Whether watched cells push to this object: it has a class binding + /// and `ZONE_WORLD_PUSH` is not `0`. + fn pushed(&self) -> bool { + !self.binding.is_empty() + && self + .inner + .borrow() + .options + .as_ref() + .is_some_and(|o| o.world_push) + } + /// Read the watched topics that are due (all of them with `all`, on - /// wake) off the tick. With a class binding each read renews the - /// zone's watch and the next comes ~45 s later (pushes carry the - /// changes); without one the cells are polled every [`POLL_MS`]. + /// wake) off the tick ([`WorldSync::due`]). When pushed, each read + /// renews the zone's watch and the next comes ~45 s later (pushes + /// carry the changes); otherwise the cells are polled every second. fn reconcile_world(&self, all: bool) { - let now = now_ms(); - let pushed = !self.binding.is_empty(); - for topic in self.world_topics() { - let due = { - let mut inner = self.inner.borrow_mut(); - let next = inner.world_due.get(&topic).copied().unwrap_or(0.0); - if !all && now < next { - continue; - } - let after = if pushed { - next_reconcile_ms(now as u64, js_sys::Math::random()) as f64 - } else { - now + POLL_MS - }; - inner.world_due.insert(topic.clone(), after); - true - }; - if due { - let this = self.handle(); - wasm_bindgen_futures::spawn_local(async move { this.reconcile(&topic).await }); - } + let topics = self.world_topics(); + let pushed = self.pushed(); + let due = self.inner.borrow_mut().world.due( + topics, + now_ms() as u64, + all, + pushed, + js_sys::Math::random, + ); + for topic in due { + let this = self.handle(); + wasm_bindgen_futures::spawn_local(async move { this.reconcile(&topic).await }); } } @@ -1520,10 +1550,11 @@ impl ZoneObject { let Some((kind, shard)) = topic.split_once('/') else { return; }; - let watcher = (!self.binding.is_empty()).then(|| { + let watcher = self.pushed().then(|| { let zone_id = self.inner.borrow().zone_id.clone(); watcher_address(&self.binding, &zone_id) }); + self.inner.borrow().work.borrow_mut().world_reads += 1; match self .env .read_cell_watching(kind, shard, watcher.as_deref()) @@ -1544,7 +1575,11 @@ impl ZoneObject { let Ok(update) = req.json::().await else { return Response::error("a world update is required", 400); }; - let awake = self.inner.borrow().host.is_some(); + let awake = { + let inner = self.inner.borrow(); + inner.work.borrow_mut().world_pushes += 1; + inner.host.is_some() + }; if awake && self.world_topics().contains(&update.topic) { self.apply_view(&update.topic, update.view); } @@ -1554,12 +1589,7 @@ impl ZoneObject { /// Apply `view` of `topic` if it is newer than what the zone has. fn apply_view(&self, topic: &str, view: Value) { let version = view_version(&view); - if !self - .inner - .borrow_mut() - .world_versions - .accept(topic, version) - { + if !self.inner.borrow_mut().world.accept(topic, version) { return; } let (kind, shard) = topic.split_once('/').unwrap_or((topic, "")); @@ -1905,6 +1935,7 @@ impl ZoneObject { }; let report = host.advance(); let players = host.player_count(); + inner.work.borrow_mut().record_advance(&report); if players < inner.players_seen { inner.persist_soon = true; } @@ -2182,6 +2213,12 @@ impl ZoneObject { "freeze": format!("{:?}", h.freeze_state()).to_lowercase(), })), "workMs": { "p50": stats.work.percentile(0.5), "p99": stats.work.percentile(0.99) }, + "work": *inner.work.borrow(), + "switches": inner.options.as_ref().map(|o| json!({ + "factBatch": o.fact_batch, + "updateRates": o.update_rates, + "worldPush": o.world_push && !self.binding.is_empty(), + })), }) } } diff --git a/distributed_game/src/celld_zone/plan.rs b/distributed_game/src/celld_zone/plan.rs index 7f4d9991f..18bacb322 100644 --- a/distributed_game/src/celld_zone/plan.rs +++ b/distributed_game/src/celld_zone/plan.rs @@ -55,6 +55,14 @@ pub const DEV_SETTINGS_VAR: &str = "ZONE_DEV_SETTINGS"; /// the worker's `content-rollout` cell, e.g. `world`; unset: no live /// content updates). pub const CONTENT_ROLLOUT_VAR: &str = "ZONE_CONTENT_ROLLOUT"; +/// Env var (A/B switch): `0` makes zones poll their watched cells every +/// second instead of taking pushes (default `1`: pushed, reconciled every +/// ~45 s). +pub const WORLD_PUSH_VAR: &str = "ZONE_WORLD_PUSH"; +/// Env var (A/B switch): `0` turns the distance-based update rates off +/// (every visible change every patch, [`UpdateRates::every_patch`](crate::UpdateRates::every_patch)); +/// default `1`: the zone's [`HostConfig`](crate::HostConfig) rates. +pub const UPDATE_RATES_VAR: &str = "ZONE_UPDATE_RATES"; pub use crate::netsim::{NETSIM_SHARD, ZONE_DEV_KIND}; /// Request header naming the zone (set by [`super::forward_to_zone`]; the @@ -97,6 +105,15 @@ pub struct ObjectOptions { /// The world (`content-rollout` shard) whose live content version and /// rollout the zone follows. pub content_rollout: Option, + /// Watched cells push to the zone ([`WORLD_PUSH_VAR`]). + pub world_push: bool, + /// Distance-based update rates on ([`UPDATE_RATES_VAR`]). + pub update_rates: bool, +} + +/// A switch var: on unless set to `0`, `false` or `no`. +fn on_unless_off(get: &impl Fn(&str) -> Option, name: &str) -> bool { + get(name).is_none_or(|v| !matches!(v.trim(), "0" | "false" | "no")) } fn parse( @@ -157,6 +174,8 @@ impl ObjectOptions { content_rollout: get(CONTENT_ROLLOUT_VAR) .map(|world| world.trim().to_string()) .filter(|world| !world.is_empty()), + world_push: on_unless_off(&get, WORLD_PUSH_VAR), + update_rates: on_unless_off(&get, UPDATE_RATES_VAR), }) } } @@ -393,14 +412,21 @@ mod tests { assert_eq!(options.hibernate_after, Duration::from_secs(10)); assert!(options.nodes.is_empty(), "node lookup is off by default"); assert_eq!(options.fact_service, "celld-zone"); + assert!( + options.world_push && options.update_rates, + "switches default on" + ); let options = ObjectOptions::from_vars(vars(&[ (TICKET_SECRET_VAR, "0123456789abcdef"), (TICK_HZ_VAR, "10"), (PATCH_HZ_VAR, "20"), (NODES_VAR, "node-a=127.0.0.1:1, bad ,node-b=127.0.0.1:2"), ("DISTRIBUTED_INTERNAL_SECRET", "s"), + (WORLD_PUSH_VAR, "0"), + (UPDATE_RATES_VAR, "false"), ])) .unwrap(); + assert!(!options.world_push && !options.update_rates); assert_eq!(options.patch_hz, 10, "patches never outpace ticks"); assert_eq!(options.nodes.len(), 2); assert_eq!(options.internal_secret.as_deref(), Some("s")); diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index 9acbb921c..d5a0dda1b 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -174,6 +174,9 @@ pub struct AdvanceReport { pub entities: Option, /// One entry per Delta frame built. pub frames: Vec, + /// Entity records encoded by the patches + /// ([`EncodedWorld::encodes`]: shared across clients). + pub encodes: usize, /// Settlements delivered to the zone, with the tick they were delivered on. pub settled: Vec<(u64, FactSettlement)>, /// `(tick, state hash)` after each tick, when @@ -2034,6 +2037,7 @@ impl ZoneHost { }); } core.outbound.extend(frames); + self.pending.encodes += world.encodes(); self.last_world = Some(world); } @@ -2123,6 +2127,7 @@ impl ZoneHost { report.tick_durations.extend(pending.tick_durations); report.entities = pending.entities; report.frames.extend(pending.frames); + report.encodes += pending.encodes; report.settled.extend(pending.settled); report.tick_hashes.extend(pending.tick_hashes); report diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index c0430baa5..453e8ded6 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -44,7 +44,8 @@ //! barrier / safe-room policy. //! //! # Extension seams (later phases) -//! Metrics hook [`AdvanceReport`] and [`replication::DeltaStats`]. +//! Metrics hook [`AdvanceReport`] and [`replication::DeltaStats`]; plain +//! work counts in [`WorkCounters`] ([`work`]). #![warn(missing_docs)] @@ -96,6 +97,7 @@ pub mod surface; pub mod ticket; pub mod timers; pub mod timestep; +pub mod work; pub mod world; pub mod zone; @@ -132,6 +134,7 @@ pub use stamp::{stamp_metadata, FactStamp}; pub use surface::GameSurface; pub use timers::TimerId; pub use timestep::FixedTimestep; +pub use work::WorkCounters; pub use world::{BarrierMode, ChildEvent, FreezeState, StalePolicy}; pub use zone::{ Admission, ClientId, JoinRequest, LeaveReason, ObserverId, ObserverInfo, PlayerId, PlayerInfo, diff --git a/distributed_game/src/replication.rs b/distributed_game/src/replication.rs index 08c69ceb5..911930d01 100644 --- a/distributed_game/src/replication.rs +++ b/distributed_game/src/replication.rs @@ -243,6 +243,9 @@ pub struct EncodedWorld { grid: SpatialGrid, globals: Vec, patch: u32, + /// Records encoded building this world (changed entities' payloads and + /// shared updates). + encoded: usize, cache: Mutex, } @@ -254,6 +257,7 @@ impl Clone for EncodedWorld { grid: self.grid.clone(), globals: self.globals.clone(), patch: self.patch, + encoded: self.encoded, cache: Mutex::default(), } } @@ -286,6 +290,7 @@ impl EncodedWorld { grid: SpatialGrid::new(interest.cell_size), globals: Vec::new(), patch: previous.map_or(0, |p| p.patch.wrapping_add(1)), + encoded: 0, cache: Mutex::default(), }; let zero = FieldValue::U(0); @@ -313,6 +318,7 @@ impl EncodedWorld { } _ => { let payload = Arc::new(Payload::new(e.id, e.kind, data, offsets)); + world.encoded += 1 + usize::from(before.is_some()); let update = before.map(|p| { let mask = payload.changed_since(&p.payload, kind.fields.len()); let mut w = Writer::new(); @@ -356,6 +362,15 @@ impl EncodedWorld { pub fn is_empty(&self) -> bool { self.entities.is_empty() } + + /// Entity records encoded for this patch so far: one payload (with its + /// create record) and one shared update per changed entity, plus the + /// update records for older baselines built for clients. Encoding per + /// client would cost one per record sent or deferred instead. + pub fn encodes(&self) -> usize { + let cache = self.cache.lock().unwrap_or_else(PoisonError::into_inner); + self.encoded + cache.records.len() + } } /// Counters from one Delta build. @@ -369,6 +384,8 @@ pub struct DeltaStats { pub removes: usize, /// Candidates deferred by the budget. pub deferred: usize, + /// Changed known entities held back by the update rates this patch. + pub held: usize, /// Create+update section bytes (what the budget limits). pub budget_bytes: usize, } @@ -440,6 +457,7 @@ impl ClientReplica { // Visible entities (indices are unique) become candidates with // accumulated priority. let mut visible = 0usize; + let mut held = 0usize; let mut candidates: Vec<(usize, Candidate, f32)> = Vec::new(); for index in hits { let index = index as usize; @@ -491,6 +509,7 @@ impl ClientReplica { let every = interest.update_rates.interval(d, radius); if every > 1 && !world.patch.wrapping_add(e.id).is_multiple_of(every) { // Held: keeps its accumulated priority. + held += 1; continue; } } @@ -555,6 +574,7 @@ impl ClientReplica { let mut stats = DeltaStats { removes: removes.len(), + held, ..DeltaStats::default() }; let mut creates = Writer::new(); @@ -752,6 +772,7 @@ mod tests { let mut stats = DeltaStats { removes: removes.len(), + held: held.len(), ..DeltaStats::default() }; let mut creates = Writer::new(); diff --git a/distributed_game/src/work.rs b/distributed_game/src/work.rs new file mode 100644 index 000000000..5a1c89a78 --- /dev/null +++ b/distributed_game/src/work.rs @@ -0,0 +1,202 @@ +//! Work counters: how much work a zone did, as plain integers. +//! +//! Timings on a loaded machine move with everything else running on it; +//! counts do not. Two runs of the same build with a switch flipped (fact +//! batches, update rates, pushed world views) compare exactly on these: +//! records encoded per patch, updates sent or held back, cell requests per +//! fact, world reads per room-second. +//! +//! The celld zone host keeps one [`WorkCounters`] per object (they survive +//! the zone's sleeps, not an eviction) and reports it as `work` in `GET +//! /zones/{zone}/info`; a watched cell reports its side +//! ([`WatcherStats`](crate::world::watch::WatcherStats)) at `GET +//! /{kind}/{shard}/__watchers`. `mobpit-load` reads both around each step. + +use serde::{Deserialize, Serialize}; + +use crate::host::AdvanceReport; + +/// Monotonic work counts of one zone (camelCase JSON). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct WorkCounters { + /// Patches (Delta rounds). + pub patches: u64, + /// Entity records encoded, shared by every client + /// ([`EncodedWorld::encodes`](crate::replication::EncodedWorld::encodes)). + pub encodes: u64, + /// Delta frames built (one per client per patch). + pub frames: u64, + /// Delta bytes, all clients. + pub delta_bytes: u64, + /// Entity creates sent. + pub creates: u64, + /// Entity updates sent. + pub updates: u64, + /// Entity updates held back by the update rates (a held entity counts + /// once per client per patch). + pub held: u64, + /// Creates and updates deferred by the byte budget. + pub deferred: u64, + /// Fact commands sent to cells (each entry of a batch counts). + pub fact_commands: u64, + /// Requests that carried them (a `__batch` counts once). + pub fact_requests: u64, + /// Of those, `__batch` requests. + pub fact_batches: u64, + /// Outbox delete calls for settled facts (one per batch when batched). + pub outbox_deletes: u64, + /// Reads of watched world cells (wake, reconcile, or polls when the + /// zone takes no pushes). + pub world_reads: u64, + /// World view pushes received. + pub world_pushes: u64, +} + +impl WorkCounters { + /// Add one [`ZoneHost::advance`](crate::ZoneHost::advance). + pub fn record_advance(&mut self, report: &AdvanceReport) { + self.patches += u64::from(report.patched); + self.encodes += report.encodes as u64; + for frame in &report.frames { + self.frames += 1; + self.delta_bytes += frame.bytes as u64; + self.creates += frame.stats.creates as u64; + self.updates += frame.stats.updates as u64; + self.held += frame.stats.held as u64; + self.deferred += frame.stats.deferred as u64; + } + } + + /// The work done since `earlier` (`None` when a count went backwards: + /// a different object instance). + pub fn since(&self, earlier: &Self) -> Option { + let d = |now: u64, then: u64| now.checked_sub(then); + Some(Self { + patches: d(self.patches, earlier.patches)?, + encodes: d(self.encodes, earlier.encodes)?, + frames: d(self.frames, earlier.frames)?, + delta_bytes: d(self.delta_bytes, earlier.delta_bytes)?, + creates: d(self.creates, earlier.creates)?, + updates: d(self.updates, earlier.updates)?, + held: d(self.held, earlier.held)?, + deferred: d(self.deferred, earlier.deferred)?, + fact_commands: d(self.fact_commands, earlier.fact_commands)?, + fact_requests: d(self.fact_requests, earlier.fact_requests)?, + fact_batches: d(self.fact_batches, earlier.fact_batches)?, + outbox_deletes: d(self.outbox_deletes, earlier.outbox_deletes)?, + world_reads: d(self.world_reads, earlier.world_reads)?, + world_pushes: d(self.world_pushes, earlier.world_pushes)?, + }) + } + + /// Element-wise sum (several zones). + pub fn add(&mut self, other: &Self) { + self.patches += other.patches; + self.encodes += other.encodes; + self.frames += other.frames; + self.delta_bytes += other.delta_bytes; + self.creates += other.creates; + self.updates += other.updates; + self.held += other.held; + self.deferred += other.deferred; + self.fact_commands += other.fact_commands; + self.fact_requests += other.fact_requests; + self.fact_batches += other.fact_batches; + self.outbox_deletes += other.outbox_deletes; + self.world_reads += other.world_reads; + self.world_pushes += other.world_pushes; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + EntitySchema, EntityState, FieldType, FieldValue, KindDef, ManualClock, Role, TicketClaims, + WorldView, Zone, ZoneHost, + }; + use std::sync::Arc; + use std::time::Duration; + + /// Eight dots that all move every tick. + struct Dots(u32); + + impl Zone for Dots { + type Config = (); + type Input = u8; + type Timer = (); + type Message = distributed_game::NoMessages; + + fn schema() -> EntitySchema { + EntitySchema::new().kind(KindDef::new(1, "dot").field("v", FieldType::U16)) + } + + fn on_create(_ctx: &mut crate::Ctx<'_, Self>, _: ()) -> Self { + Dots(0) + } + + fn decode_input(&self, _p: crate::PlayerId, b: &[u8]) -> Option { + b.first().copied() + } + + fn on_tick(&mut self, _ctx: &mut crate::Ctx<'_, Self>) { + self.0 += 1; + } + + fn replicate(&self, out: &mut WorldView) { + for i in 0..8 { + out.push(EntityState::new( + i, + 1, + i as f32, + 0.0, + vec![FieldValue::U(self.0 + i)], + )); + } + } + } + + fn claims(principal: &str) -> TicketClaims { + TicketClaims { + principal: principal.into(), + zone: "z".into(), + expires_at: u64::MAX, + nonce: principal.into(), + role: Role::Player, + name: None, + extensions: Default::default(), + wake: false, + } + } + + #[test] + fn counts_shared_encodes_and_per_client_records() { + let clock = ManualClock::new(); + let mut host: ZoneHost = + ZoneHost::new("z", Default::default(), Arc::new(clock.clone()), ()); + for p in ["a", "b", "c"] { + host.join(&claims(p), 0, "").unwrap(); + } + let mut work = WorkCounters::default(); + for _ in 0..10 { + clock.advance(Duration::from_millis(50)); + work.record_advance(&host.advance()); + } + assert!(work.patches >= 5, "{work:?}"); + assert_eq!(work.frames, work.patches * 3); + assert_eq!(work.creates, 8 * 3, "each client gets each dot once"); + // Every dot changes every patch and every client is current: one + // payload and one shared update per dot and patch (the first patch + // has no updates), however many clients. + assert_eq!(work.encodes, 8 + (work.patches - 1) * 16, "{work:?}"); + assert_eq!(work.updates + work.held, (work.patches - 1) * 8 * 3); + assert!(work.delta_bytes > 0); + let later = WorkCounters { + world_reads: 2, + ..work + }; + assert_eq!(later.since(&work).map(|d| d.world_reads), Some(2)); + assert_eq!(work.since(&later), None, "went backwards"); + } +} diff --git a/distributed_game/src/world/watch.rs b/distributed_game/src/world/watch.rs index cc77fdf46..4bcce7c56 100644 --- a/distributed_game/src/world/watch.rs +++ b/distributed_game/src/world/watch.rs @@ -36,6 +36,9 @@ pub const RECONCILE_MS: u64 = 45_000; /// Spread of [`RECONCILE_MS`] (± ms), so zones that woke together do not /// read together. pub const RECONCILE_JITTER_MS: u64 = 15_000; +/// How often a zone that takes no pushes (no class binding, or +/// `ZONE_WORLD_PUSH=0`) polls each watched topic (ms). +pub const POLL_MS: u64 = 1_000; /// Watchers pushed to at once by one cell. pub const PUSH_CONCURRENCY: usize = 16; /// Query parameter naming the watcher (`/`). @@ -44,6 +47,9 @@ pub const WATCH_PARAM: &str = "watch"; pub const LEASE_PARAM: &str = "lease"; /// Path segment of the zone route that takes pushes (`POST …/__world`). pub const PUSH_ROUTE: &str = "__world"; +/// Path segment of a watched cell's counters (`GET /{kind}/{shard}/__watchers`, +/// internal secret): its live watchers and [`WatcherStats`]. +pub const STATS_ROUTE: &str = "__watchers"; /// The topic of cell `kind/shard`. pub fn topic(kind: &str, shard: &str) -> String { @@ -93,6 +99,22 @@ pub struct Watchers { leases: BTreeMap, published: Option, last_view: Option, + stats: WatcherStats, +} + +/// What a watched cell did for its watchers since it started (camelCase +/// JSON; the cell's `GET …/__watchers`). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct WatcherStats { + /// Watching reads (each leaves or renews a lease). + pub reads: u64, + /// Views published to the live watchers. + pub published: u64, + /// Views not published: unchanged apart from `version`, or not newer. + pub skipped: u64, + /// Pushes sent (one per live watcher of each published view). + pub pushes: u64, } impl Watchers { @@ -104,6 +126,7 @@ impl Watchers { /// `watcher` watches until `now_ms + lease_ms` (capped at /// [`WATCH_LEASE_MS`]); a zero lease ends its watch. pub fn watch(&mut self, watcher: &str, now_ms: u64, lease_ms: u64) { + self.stats.reads += 1; if lease_ms == 0 { self.leases.remove(watcher); return; @@ -123,6 +146,16 @@ impl Watchers { /// (a command that moves the aggregate but not its view, like a /// heartbeat, pushes nothing). Records it when it is. pub fn publish(&mut self, version: u64, view: &Value) -> bool { + let news = self.news(version, view); + if news { + self.stats.published += 1; + } else { + self.stats.skipped += 1; + } + news + } + + fn news(&mut self, version: u64, view: &Value) -> bool { if self.published.is_some_and(|last| version <= last) { return false; } @@ -141,6 +174,16 @@ impl Watchers { self.last_view = Some(content); true } + + /// `n` pushes were sent. + pub fn pushed(&mut self, n: usize) { + self.stats.pushes += n as u64; + } + + /// Counts so far. + pub fn stats(&self) -> WatcherStats { + self.stats + } } /// `(watcher, lease ms)` from a read's query string, if it watches. @@ -201,12 +244,81 @@ impl TopicVersions { } } + /// The highest version seen of `topic`. + pub fn get(&self, topic: &str) -> Option { + self.seen.get(topic).copied() + } + /// Forget everything (a new activation re-hydrates). pub fn clear(&mut self) { self.seen.clear(); } } +/// A zone's side of its watches: the highest version applied per topic +/// ([`TopicVersions`]) and when each topic is read next. A zone that takes +/// pushes reads each topic on wake and then every [`RECONCILE_MS`] ± +/// [`RECONCILE_JITTER_MS`] (renewing its lease, repairing a lost push); +/// one that does not polls every [`POLL_MS`]. +#[derive(Debug, Default)] +pub struct WorldSync { + versions: TopicVersions, + due: BTreeMap, +} + +impl WorldSync { + /// Nothing applied, every topic due. + pub fn new() -> Self { + Self::default() + } + + /// The `topics` to read at `now_ms` (every one with `all`, on wake), + /// each scheduled for its next read: a reconcile when the zone takes + /// `pushed` updates (`unit()` in `[0, 1)` choosing where in the + /// spread), else a poll. + pub fn due( + &mut self, + topics: impl IntoIterator, + now_ms: u64, + all: bool, + pushed: bool, + mut unit: impl FnMut() -> f64, + ) -> Vec { + let mut reads = Vec::new(); + for topic in topics { + let next = self.due.get(&topic).copied().unwrap_or(0); + if !all && now_ms < next { + continue; + } + let after = if pushed { + next_reconcile_ms(now_ms, unit()) + } else { + now_ms + POLL_MS + }; + self.due.insert(topic.clone(), after); + reads.push(topic); + } + reads + } + + /// Whether `version` of `topic` (read or pushed) should be applied + /// ([`TopicVersions::accept`]). + pub fn accept(&mut self, topic: &str, version: u64) -> bool { + self.versions.accept(topic, version) + } + + /// The highest version applied of `topic`. + pub fn version(&self, topic: &str) -> Option { + self.versions.get(topic) + } + + /// Forget everything (a new activation re-hydrates). + pub fn clear(&mut self) { + self.versions.clear(); + self.due.clear(); + } +} + /// The next reconcile of a topic: [`RECONCILE_MS`] ± [`RECONCILE_JITTER_MS`] /// after `now_ms`, `unit` in `[0, 1)` choosing where in the spread. pub fn next_reconcile_ms(now_ms: u64, unit: f64) -> u64 { @@ -383,6 +495,123 @@ mod tests { assert!(watchers.publish(3, &json!({ "version": 3, "tier": 3 }))); } + #[test] + fn watchers_count_their_work() { + let mut watchers = Watchers::new(); + watchers.watch("PIT_ROOM/pit.0.0", 0, 1_000); + watchers.watch("PIT_ROOM/pit.0.0", 500, 1_000); + assert!(watchers.publish(1, &json!({ "version": 1, "tier": 1 }))); + let live = watchers.live(600).len(); + watchers.pushed(live); + assert!(!watchers.publish(2, &json!({ "version": 2, "tier": 1 }))); + assert_eq!( + watchers.stats(), + WatcherStats { + reads: 2, + published: 1, + skipped: 1, + pushes: 1, + } + ); + } + + /// A deterministic `[0, 1)` sequence (xorshift). + fn units(seed: u64) -> impl FnMut() -> f64 { + let mut x = seed.max(1); + move || { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 11) as f64 / (1u64 << 53) as f64 + } + } + + /// One zone watching one cell on a fake clock: the cell changes now and + /// then, and pushes are lost with `loss` (0..=1). Returns the longest + /// time (ms) the zone was behind the cell. + fn longest_lag(seed: u64, loss: f64) -> u64 { + const ZONE: &str = "PIT_ROOM/pit.0.0"; + const STEP_MS: u64 = 100; + let hood = topic("hood", "0.0"); + let view = |v: u64| json!({ "version": v, "tier": v }); + let mut cell = Watchers::new(); + let mut cell_version = 1u64; + let mut zone = WorldSync::new(); + let mut unit = units(seed); + let mut chance = units(seed ^ 0xfeed); + let (mut behind_since, mut longest) = (None::, 0u64); + let mut now = 0u64; + let mut all = true; + while now < 30 * 60_000 { + // The cell changes about every 20 s; its push may be lost. + if now > 0 && chance() < STEP_MS as f64 / 20_000.0 { + cell_version += 1; + if cell.publish(cell_version, &view(cell_version)) { + let live = cell.live(now); + cell.pushed(live.len()); + if live.iter().any(|w| w == ZONE) && chance() >= loss { + zone.accept(&hood, cell_version); + } + } + } + // The zone's tick: due reads renew its lease and apply the view. + for topic in zone.due([hood.clone()], now, all, true, &mut unit) { + cell.watch(ZONE, now, WATCH_LEASE_MS); + zone.accept(&topic, cell_version); + } + all = false; + if zone.version(&hood) == Some(cell_version) { + behind_since = None; + } else { + let since = *behind_since.get_or_insert(now); + longest = longest.max(now - since); + } + now += STEP_MS; + } + assert!(cell.live(now).contains(&ZONE.to_string()), "lease renewed"); + longest + } + + #[test] + fn a_reconcile_repairs_a_missed_push() { + let period = RECONCILE_MS + RECONCILE_JITTER_MS; + for seed in 1..=8 { + // Every push lost: the next reconcile catches the zone up. + let lag = longest_lag(seed, 1.0); + assert!(lag > 0 && lag <= period, "seed {seed}: behind for {lag} ms"); + // Some pushes lost. + let lag = longest_lag(seed, 0.3); + assert!(lag <= period, "seed {seed}: behind for {lag} ms"); + } + // No push lost: the zone is never behind. + assert_eq!(longest_lag(3, 0.0), 0); + } + + #[test] + fn a_zone_without_pushes_polls() { + let mut zone = WorldSync::new(); + let topics = || [topic("hood", "0.0"), topic("city", "city")]; + assert_eq!(zone.due(topics(), 0, true, false, || 0.5).len(), 2); + assert!(zone + .due(topics(), POLL_MS - 1, false, false, || 0.5) + .is_empty()); + assert_eq!(zone.due(topics(), POLL_MS, false, false, || 0.5).len(), 2); + // Pushed: the next read is a reconcile period away. + assert_eq!( + zone.due(topics(), 2 * POLL_MS, false, true, || 0.5).len(), + 2 + ); + assert!(zone + .due( + topics(), + 2 * POLL_MS + RECONCILE_MS - 1, + false, + true, + || 0.5 + ) + .is_empty()); + } + #[test] fn zones_never_roll_back() { let mut seen = TopicVersions::new(); From a61a9e0a2e3bff04fe953e9a60ca9247d5984be9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:36:05 -0500 Subject: [PATCH 12/22] feat(game): mobpit-load work lines and interleaved A/B runs Each step prints the rooms' and watched cells' work counters normalised per room patch, room-second or room-minute; --ab VAR=VAL with --pairs N alternates fresh-celld runs (A,B then B,A) and prints medians per side and of the per-pair change. Skill and README "Checking your work". [[framework-work-counters]] [[mobpit-zone-load-test]] --- .../skills/distributed-game/SKILL.md | 16 +- examples/game/server/examples/mobpit-load.rs | 393 +++++++++++++++++- 2 files changed, 387 insertions(+), 22 deletions(-) diff --git a/distributed_cli/skills/distributed-game/SKILL.md b/distributed_cli/skills/distributed-game/SKILL.md index 6d606ae9c..e58d7e90f 100644 --- a/distributed_cli/skills/distributed-game/SKILL.md +++ b/distributed_cli/skills/distributed-game/SKILL.md @@ -135,6 +135,16 @@ How it behaves: - `--steps 32,64` for one room; - `--steps 64x1,128x1` for many rooms; - `--no-attack` for movement only; - - `--var ZONE_FACT_BATCH=1` to A/B fact batching. - - Compare runs only on a quiet machine. + - `--var KEY=VALUE` for a worker var on every run. +- Compare work, not wall clock. Each step prints a `work` line from the + rooms' `WorkCounters` (`GET /zones/{zone}/info` → `work`) and the + watched cells' `__watchers` counts: records encoded and sent per patch, + updates held by rates, bytes per frame, fact commands, cell requests and + outbox deletes per room-second, world reads and pushes per room-minute. + Counts do not depend on machine load; timings do. +- A/B one switch with interleaved runs of the same build: + `--ab VAR=VAL --pairs 3` runs A,B then B,A … (fresh celld each) and + prints the median of each side and of the per-pair change. Switches: + `ZONE_FACT_BATCH=1` (fact batches off), `ZONE_UPDATE_RATES=0` (update + rates off), `ZONE_WORLD_PUSH=0` (watched cells polled every second + instead of pushed). Read timings only as these interleaved medians. diff --git a/examples/game/server/examples/mobpit-load.rs b/examples/game/server/examples/mobpit-load.rs index 937fcbbcf..2a6567492 100644 --- a/examples/game/server/examples/mobpit-load.rs +++ b/examples/game/server/examples/mobpit-load.rs @@ -18,12 +18,34 @@ //! A row is marked `DEGRADED` past 1 % dropped ticks, late p99 > 50 ms, RTT //! p99 > 150 ms or a bot that did not join. //! +//! Under each row, the work the rooms did over the hold (their `work` +//! counters, `distributed_game::WorkCounters`, and the watched cells' +//! `__watchers` counts) — exact, whatever the machine's load: +//! +//! - `enc/patch` records encoded per room patch (shared by all clients), +//! `sent/patch` creates + updates sent, `held` the share of updates held +//! back by update rates, `B/frame` Delta bytes per client frame; +//! - `facts/room-s`, `req/room-s` (cell requests; a `__batch` is one), +//! `batches`, `deletes/room-s` (outbox deletes); +//! - `reads/room-min` (watched-cell reads by the rooms), `pushes/room-min` +//! (received), and per minute the cells' `pushed` / `skipped` (unchanged) +//! views and watching `reads`. +//! +//! Interleaved A/B (local only; same build, fresh celld per run): `--ab +//! VAR=VAL` (repeatable) is the B side's extra worker vars; `--pairs N` +//! runs A,B then B,A … and prints per step the median of each side and the +//! median of the per-pair change. Switches: `ZONE_FACT_BATCH=1` (one fact +//! per request; default 16), `ZONE_UPDATE_RATES=0` (every change every +//! patch), `ZONE_WORLD_PUSH=0` (rooms poll watched cells every second +//! instead of taking pushes). +//! //! Local (starts its own `celld dev` with the cells worker; build it first //! with `./run.sh build`): //! //! ```sh //! cargo run --release --example mobpit-load -- --steps 8,16,32,64,96,128 //! cargo run --release --example mobpit-load -- --steps 16x1,64x1,64x8 +//! cargo run --release --example mobpit-load -- --steps 32 --ab ZONE_UPDATE_RATES=0 --pairs 3 //! ``` //! //! Against a running fleet (celld's public listener, the worker's ticket @@ -38,12 +60,18 @@ use std::sync::Arc; use std::time::Duration; use clap::Parser; +use distributed::cell_host::CELL_INTERNAL_SECRET_HEADER; use distributed_game::authoring::GameInput; use distributed_game::bots::{run_bots, Behaviour, BotConfig, Circle, RandomWalk, Summary}; use distributed_game::net::TicketKey; +use distributed_game::netsim::{NETSIM_SHARD, ZONE_DEV_KIND}; +use distributed_game::world::content_rollout::CONTENT_ROLLOUT_KIND; +use distributed_game::world::watch::{WatcherStats, STATS_ROUTE}; +use distributed_game::WorkCounters; use game_demos::devcelld::DevCelld; use game_demos::mobpit::encode_bot_input; -use game_sim::mobpit::PitInput; +use game_sim::mobpit::{borough_of, hood_of, pair_shard, parse_room, PitInput}; +use mobpit_tiers::{BOROUGH_KIND, CITY_KIND, CITY_SHARD, HOOD_KIND}; use serde_json::{json, Value}; #[derive(Parser, Debug)] @@ -95,11 +123,73 @@ struct Args { /// Write every step as JSON lines here. #[arg(long)] json: Option, + /// Internal secret for the watched cells' counters (with `--url`; + /// local runs know it). + #[arg(long, env = "DISTRIBUTED_INTERNAL_SECRET")] + internal_secret: Option, + /// A/B: the B side's extra worker var, `KEY=VALUE` (repeatable); the A + /// side runs with `--var` only. Local only. + #[arg(long)] + ab: Vec, + /// A/B pairs (A,B then B,A, …). + #[arg(long, default_value_t = 3)] + pairs: usize, } #[tokio::main(flavor = "multi_thread")] async fn main() -> Result<(), String> { let args = Args::parse(); + let mut json_out = match &args.json { + Some(path) => { + Some(std::fs::File::create(path).map_err(|e| format!("{}: {e}", path.display()))?) + } + None => None, + }; + if args.ab.is_empty() { + run(&args, &[], "", 0, &mut json_out).await?; + return Ok(()); + } + if args.url.is_some() { + return Err("--ab starts its own celld per run (local only)".into()); + } + let b_vars: Vec = args.ab.clone(); + println!("A/B: B = A + {} · {} pairs", b_vars.join(" "), args.pairs); + let (mut a_runs, mut b_runs) = (Vec::new(), Vec::new()); + for pair in 0..args.pairs.max(1) { + // Alternate the order so drift on the machine hits both sides. + let order = if pair % 2 == 0 { + [false, true] + } else { + [true, false] + }; + for b in order { + tokio::time::sleep(Duration::from_secs(args.rest_secs)).await; + let (vars, label) = if b { + (&b_vars[..], "B") + } else { + (&[][..], "A") + }; + let rows = run(&args, vars, label, pair, &mut json_out).await?; + if b { + b_runs.push(rows); + } else { + a_runs.push(rows); + } + } + } + print_ab(&args.steps, &a_runs, &b_runs); + Ok(()) +} + +/// One run over every step (a fresh local celld with `extra` vars, unless +/// `--url`); prints each row as it finishes. +async fn run( + args: &Args, + extra: &[String], + label: &str, + pair: usize, + json_out: &mut Option, +) -> Result, String> { let largest = args.steps.iter().map(|s| s.1).max().unwrap_or(32); // Local: our own fleet, the room cap raised to the largest step. let cap = largest.to_string(); @@ -107,9 +197,19 @@ async fn main() -> Result<(), String> { Some(_) => None, None => { let mut vars = vec![("ZONE_MAX_PLAYERS", cap.as_str())]; - vars.extend(args.var.iter().filter_map(|v| v.split_once('='))); + vars.extend( + args.var + .iter() + .chain(extra) + .filter_map(|v| v.split_once('=')), + ); Some(match &args.celld_dir { - Some(dir) => DevCelld::in_project_with(dir, None, &vars)?, + Some(dir) if label.is_empty() => DevCelld::in_project_with(dir, None, &vars)?, + Some(dir) => DevCelld::in_project_with( + &dir.join(format!("{}{pair}", label.to_lowercase())), + None, + &vars, + )?, None => DevCelld::start_with(None, &vars)?, }) } @@ -124,21 +224,30 @@ async fn main() -> Result<(), String> { ), (None, None) => unreachable!(), }; + let internal = local + .as_ref() + .map(|celld| celld.secret().to_owned()) + .or_else(|| args.internal_secret.clone()); let info_base = args.info_url.clone().unwrap_or_else(|| { url.replacen("ws://", "http://", 1) .replacen("wss://", "https://", 1) }); let celld_pid = local.as_ref().and_then(DevCelld::pid); - let mut json_out = match &args.json { - Some(path) => { - Some(std::fs::File::create(path).map_err(|e| format!("{}: {e}", path.display()))?) - } - None => None, - }; + let tag = if label.is_empty() { + String::new() + } else { + format!("[{label}{pair}] ") + }; println!( - "{url} · hold {} s · {} inputs/s per bot · worst room's ticks", - args.hold_secs, args.rate + "{tag}{url} · hold {} s · {} inputs/s per bot · worst room's ticks{}", + args.hold_secs, + args.rate, + if extra.is_empty() { + String::new() + } else { + format!(" · {}", extra.join(" ")) + } ); println!( "{:>9} {:>6} {:>6} {:>6} {:>8} {:>9} {:>8} {:>8} {:>9} {:>6} {:>7} stats", @@ -154,6 +263,7 @@ async fn main() -> Result<(), String> { "cores", "rss MB" ); + let mut rows = Vec::with_capacity(args.steps.len()); for (step, &(zones, n)) in args.steps.iter().enumerate() { if step > 0 { tokio::time::sleep(Duration::from_secs(args.rest_secs)).await; @@ -185,11 +295,19 @@ async fn main() -> Result<(), String> { .iter() .map(|room| format!("{info_base}/zones/{room}/info")) .collect(); + let cell_urls: Vec = match &internal { + Some(_) => watched_cells(&rooms) + .iter() + .map(|cell| format!("{info_base}/{cell}/{STATS_ROUTE}")) + .collect(), + None => Vec::new(), + }; // Window: from shortly after the last join to shortly before the // first leave. tokio::time::sleep(Duration::from_secs(args.ramp_secs + 2)).await; let start = get_all(&info_urls).await; + let cells_start = get_cells(&cell_urls, internal.as_deref()).await; let window_start = unix_secs(); let cpu_start = (celld_pid.and_then(tree_usage), std::time::Instant::now()); tokio::time::sleep(Duration::from_secs(args.hold_secs.saturating_sub(5))).await; @@ -198,6 +316,7 @@ async fn main() -> Result<(), String> { None => String::new(), }; let end = get_all(&info_urls).await; + let cells_end = get_cells(&cell_urls, internal.as_deref()).await; let window = (window_start, unix_secs()); let usage = match (cpu_start, celld_pid.and_then(tree_usage)) { ((Some(before), at), Some(after)) => { @@ -221,14 +340,132 @@ async fn main() -> Result<(), String> { summary.elapsed = summary.elapsed.max(part.elapsed); } - let row = Row::new((zones, n), &start, &end, &summary, usage, stats, window); - println!("{}", row.line()); + let mut row = Row::new((zones, n), &start, &end, &summary, usage, stats, window); + row.cells = cells_since(&cells_start, &cells_end); + println!("{tag}{}", row.line()); + println!("{tag}{:>9} {}", "", row.work_line()); if let Some(out) = json_out.as_mut() { use std::io::Write; - writeln!(out, "{}", row.json()).map_err(|e| e.to_string())?; + let mut json = row.json(); + json["run"] = json!(format!("{label}{pair}")); + json["vars"] = json!(extra); + writeln!(out, "{json}").map_err(|e| e.to_string())?; + } + rows.push(row); + } + Ok(rows) +} + +/// The cells the rooms watch (`kind/shard`): each room's hood and +/// borough, the city, the content rollout and the dev settings. +fn watched_cells(rooms: &[String]) -> Vec { + let mut cells = std::collections::BTreeSet::new(); + for room in rooms { + let Some(at) = parse_room(room) else { + continue; + }; + let hood = hood_of(at); + cells.insert(format!("{HOOD_KIND}/{}", pair_shard(hood))); + cells.insert(format!("{BOROUGH_KIND}/{}", pair_shard(borough_of(hood)))); + } + cells.insert(format!("{CITY_KIND}/{CITY_SHARD}")); + cells.insert(format!("{CONTENT_ROLLOUT_KIND}/world")); + cells.insert(format!("{ZONE_DEV_KIND}/{NETSIM_SHARD}")); + cells.into_iter().collect() +} + +/// Each cell's [`WatcherStats`] (`None`: unreadable). +async fn get_cells(urls: &[String], secret: Option<&str>) -> Vec> { + let Some(secret) = secret else { + return Vec::new(); + }; + let mut out = Vec::with_capacity(urls.len()); + for url in urls { + let stats = client() + .get(url) + .header(CELL_INTERNAL_SECRET_HEADER, secret) + .send() + .await + .ok() + .filter(|r| r.status().is_success()); + out.push(match stats { + Some(response) => response.json().await.ok(), + None => None, + }); + } + out +} + +/// The cells' work over the window, summed (`None` without readings). +fn cells_since( + start: &[Option], + end: &[Option], +) -> Option { + let mut sum = WatcherStats::default(); + let mut any = false; + for (start, end) in start.iter().zip(end) { + // A cell that started in the window counts from zero. + let (start, Some(end)) = (start.unwrap_or_default(), end) else { + continue; + }; + if end.reads < start.reads { + continue; + } + any = true; + sum.reads += end.reads - start.reads; + sum.published += end.published - start.published; + sum.skipped += end.skipped - start.skipped; + sum.pushes += end.pushes - start.pushes; + } + any.then_some(sum) +} + +fn median(mut values: Vec) -> Option { + values.retain(|v| v.is_finite()); + values.sort_by(f64::total_cmp); + let n = values.len(); + match n { + 0 => None, + _ if n % 2 == 1 => Some(values[n / 2]), + _ => Some((values[n / 2 - 1] + values[n / 2]) / 2.0), + } +} + +/// Per step and metric: A's median, B's median and the median of the +/// per-pair change (B vs A of the same pair). +fn print_ab(steps: &[(usize, usize)], a: &[Vec], b: &[Vec]) { + let fmt = |v: Option| match v { + None => "—".to_owned(), + Some(v) if v.abs() >= 100.0 => format!("{v:.0}"), + Some(v) => format!("{v:.2}"), + }; + for (i, &(zones, n)) in steps.iter().enumerate() { + println!("\nA/B {zones}x{n}: median A · median B · median pair change"); + let names: Vec<&str> = Row::metric_names().to_vec(); + for (m, name) in names.iter().enumerate() { + let side = |runs: &[Vec]| -> Vec> { + runs.iter() + .map(|rows| rows.get(i).and_then(|row| row.metrics()[m])) + .collect() + }; + let (sa, sb) = (side(a), side(b)); + let changes: Vec = sa + .iter() + .zip(&sb) + .filter_map(|(a, b)| match (a, b) { + (Some(a), Some(b)) if *a != 0.0 => Some((b - a) / a * 100.0), + _ => None, + }) + .collect(); + let change = median(changes).map_or("—".to_owned(), |c| format!("{c:+.0} %")); + println!( + " {name:>16} {:>10} {:>10} {:>8}", + fmt(median(sa.into_iter().flatten().collect())), + fmt(median(sb.into_iter().flatten().collect())), + change + ); } } - Ok(()) } /// Random walkers cross the room; circlers stay near where they spawn. @@ -308,15 +545,18 @@ fn encode_bot_move(dir: [f32; 2]) -> Vec { .to_payload() } -async fn get(url: &str) -> Value { +fn client() -> &'static reqwest::Client { static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); - let client = CLIENT.get_or_init(|| { + CLIENT.get_or_init(|| { reqwest::Client::builder() .timeout(Duration::from_secs(5)) .build() .expect("http client") - }); - match client.get(url).send().await { + }) +} + +async fn get(url: &str) -> Value { + match client().get(url).send().await { Ok(response) => response.json().await.unwrap_or(Value::Null), Err(_) => Value::Null, } @@ -396,6 +636,11 @@ struct Row { stats: String, /// Measured window, Unix seconds (to line up external samples). window: (f64, f64), + /// The rooms' work over the window, summed, and how many rooms it + /// covers (rooms whose counters restarted are left out). + work: Option<(WorkCounters, usize)>, + /// The watched cells' work over the window, summed. + cells: Option, } impl Row { @@ -422,6 +667,16 @@ impl Row { .collect::>>() .map(|all| all.into_iter().fold(0.0, f64::max)); let ms = |d: Option| d.map(|d| d.as_secs_f64() * 1000.0); + let counters = |v: &Value| serde_json::from_value::(v["work"].clone()).ok(); + let mut work: Option<(WorkCounters, usize)> = None; + for (start, end) in start.iter().zip(end) { + let Some(done) = counters(end).and_then(|end| end.since(&counters(start)?)) else { + continue; + }; + let (sum, rooms) = work.get_or_insert_with(Default::default); + sum.add(&done); + *rooms += 1; + } Self { zones, per_zone, @@ -441,9 +696,101 @@ impl Row { usage, stats, window, + work, + cells: None, } } + /// Names of [`metrics`](Self::metrics), in order. + fn metric_names() -> [&'static str; 16] { + [ + "dropped %", + "late p99 ms", + "rtt p99 ms", + "B/s/bot", + "celld cores", + "enc/patch", + "sent/patch", + "held %", + "B/frame", + "facts/room-s", + "req/room-s", + "deletes/room-s", + "reads/room-min", + "pushes/room-min", + "cell pushed/min", + "cell skipped/min", + ] + } + + /// The numbers A/B compares (see [`metric_names`](Self::metric_names)). + fn metrics(&self) -> [Option; 16] { + let secs = (self.window.1 - self.window.0).max(1e-3); + let w = self.work.map(|(w, rooms)| (w, rooms as f64 * secs)); + let per = |num: fn(&WorkCounters) -> u64, den: fn(&WorkCounters) -> u64| { + w.and_then(|(w, _)| (den(&w) > 0).then(|| num(&w) as f64 / den(&w) as f64)) + }; + let per_room_s = + |num: fn(&WorkCounters) -> u64| w.map(|(w, room_s)| num(&w) as f64 / room_s); + let cell_per_min = + |num: fn(&WatcherStats) -> u64| self.cells.map(|c| num(&c) as f64 * 60.0 / secs); + [ + self.dropped_pct, + Some(self.late_p99), + self.rtt_p99, + Some(self.bytes_per_bot), + self.usage.map(|u| u.0), + per(|w| w.encodes, |w| w.patches), + per(|w| w.creates + w.updates, |w| w.patches), + per(|w| w.held * 100, |w| w.held + w.updates), + per(|w| w.delta_bytes, |w| w.frames), + per_room_s(|w| w.fact_commands), + per_room_s(|w| w.fact_requests), + per_room_s(|w| w.outbox_deletes), + per_room_s(|w| w.world_reads * 60), + per_room_s(|w| w.world_pushes * 60), + cell_per_min(|c| c.pushes), + cell_per_min(|c| c.skipped), + ] + } + + /// The work line under the row. + fn work_line(&self) -> String { + let Some((w, rooms)) = self.work else { + return "work: — (no counters: an older worker build?)".into(); + }; + let m = self.metrics(); + let opt = + |v: Option, digits: usize| v.map_or("—".to_owned(), |v| format!("{v:.digits$}")); + let mut line = format!( + "work ({rooms} rooms): enc/patch {} · sent/patch {} · held {}% · B/frame {} · \ + facts/room-s {} · req/room-s {} ({} batches) · deletes/room-s {} · \ + reads/room-min {} · pushes/room-min {}", + opt(m[5], 1), + opt(m[6], 1), + opt(m[7], 0), + opt(m[8], 0), + opt(m[9], 2), + opt(m[10], 2), + w.fact_batches, + opt(m[11], 2), + opt(m[12], 1), + opt(m[13], 1), + ); + if let Some(c) = self.cells { + line.push_str(&format!( + " · cells/min: pushed {} skipped {} reads {}", + opt(m[14], 0), + opt(m[15], 0), + opt( + Some(c.reads as f64 * 60.0 / (self.window.1 - self.window.0).max(1e-3)), + 0 + ), + )); + } + line + } + fn degraded(&self) -> bool { self.joined < self.bots || self.dropped_pct.is_none_or(|d| d > 1.0) @@ -498,6 +845,14 @@ impl Row { "windowStart": self.window.0, "windowEnd": self.window.1, "degraded": self.degraded(), + "work": self.work.map(|(w, _)| w), + "workRooms": self.work.map(|(_, rooms)| rooms), + "cells": self.cells, + "metrics": Row::metric_names() + .iter() + .zip(self.metrics()) + .map(|(name, value)| (name.to_string(), json!(value))) + .collect::>(), }) } } From 7c89cfecd3d30dc63e1f0cf2296ab448ca806302 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:52:52 -0500 Subject: [PATCH 13/22] feat(game): mobpit-load reports facts per request and deletes per fact Fact rates follow the fighting, which moves with machine load; the ratios are what a fact-batch A/B compares. [[framework-work-counters]] [[zone-fact-batches]] --- examples/game/server/examples/mobpit-load.rs | 22 +++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/examples/game/server/examples/mobpit-load.rs b/examples/game/server/examples/mobpit-load.rs index 2a6567492..11c418fda 100644 --- a/examples/game/server/examples/mobpit-load.rs +++ b/examples/game/server/examples/mobpit-load.rs @@ -26,7 +26,8 @@ //! `sent/patch` creates + updates sent, `held` the share of updates held //! back by update rates, `B/frame` Delta bytes per client frame; //! - `facts/room-s`, `req/room-s` (cell requests; a `__batch` is one), -//! `batches`, `deletes/room-s` (outbox deletes); +//! `batches`, `facts/req`, `deletes/room-s` (outbox deletes) and +//! `deletes/fact` (rates depend on the fighting; the ratios do not); //! - `reads/room-min` (watched-cell reads by the rooms), `pushes/room-min` //! (received), and per minute the cells' `pushed` / `skipped` (unchanged) //! views and watching `reads`. @@ -702,7 +703,7 @@ impl Row { } /// Names of [`metrics`](Self::metrics), in order. - fn metric_names() -> [&'static str; 16] { + fn metric_names() -> [&'static str; 18] { [ "dropped %", "late p99 ms", @@ -716,6 +717,8 @@ impl Row { "facts/room-s", "req/room-s", "deletes/room-s", + "facts/req", + "deletes/fact", "reads/room-min", "pushes/room-min", "cell pushed/min", @@ -724,7 +727,7 @@ impl Row { } /// The numbers A/B compares (see [`metric_names`](Self::metric_names)). - fn metrics(&self) -> [Option; 16] { + fn metrics(&self) -> [Option; 18] { let secs = (self.window.1 - self.window.0).max(1e-3); let w = self.work.map(|(w, rooms)| (w, rooms as f64 * secs)); let per = |num: fn(&WorkCounters) -> u64, den: fn(&WorkCounters) -> u64| { @@ -747,6 +750,8 @@ impl Row { per_room_s(|w| w.fact_commands), per_room_s(|w| w.fact_requests), per_room_s(|w| w.outbox_deletes), + per(|w| w.fact_commands, |w| w.fact_requests), + per(|w| w.outbox_deletes, |w| w.fact_commands), per_room_s(|w| w.world_reads * 60), per_room_s(|w| w.world_pushes * 60), cell_per_min(|c| c.pushes), @@ -764,7 +769,7 @@ impl Row { |v: Option, digits: usize| v.map_or("—".to_owned(), |v| format!("{v:.digits$}")); let mut line = format!( "work ({rooms} rooms): enc/patch {} · sent/patch {} · held {}% · B/frame {} · \ - facts/room-s {} · req/room-s {} ({} batches) · deletes/room-s {} · \ + facts/room-s {} · req/room-s {} ({} batches, {} facts/req) · deletes/room-s {} · \ reads/room-min {} · pushes/room-min {}", opt(m[5], 1), opt(m[6], 1), @@ -773,15 +778,16 @@ impl Row { opt(m[9], 2), opt(m[10], 2), w.fact_batches, + opt(m[12], 2), opt(m[11], 2), - opt(m[12], 1), - opt(m[13], 1), + opt(m[14], 1), + opt(m[15], 1), ); if let Some(c) = self.cells { line.push_str(&format!( " · cells/min: pushed {} skipped {} reads {}", - opt(m[14], 0), - opt(m[15], 0), + opt(m[16], 0), + opt(m[17], 0), opt( Some(c.reads as f64 * 60.0 / (self.window.1 - self.window.0).max(1e-3)), 0 From 682d93ff9578038140a812d8d70a0eef74d85708 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:58:42 -0500 Subject: [PATCH 14/22] feat(game): negotiate protocol v2 with features and origin Hello offered only version 1 and decoders reject trailing bytes, so Welcome could not grow and a new client message (Focus) made an old celld zone close the connection: servers had to deploy first. Servers (native net, celld zone object) now answer v1 or v2 in the client's version (ZoneHost::speaking) and refuse others with VERSION_MISMATCH "protocol ..." before the ticket is touched. Welcome v2 appends runtime features (bit 0 FOCUS) and the zone's world origin (Zone::origin). The JS SDK offers 2, resends the same Hello once with 1 to a v1-only server (remembered per URL, not surfaced as an error) and sends Focus only when FOCUS is announced. Golden fixtures cover v1 and v2; PROTOCOL.md gains a negotiation section. Game surfaces keep protocolVersion 1 (payload encodings are unchanged). [[zone-protocol-v2-negotiation]] --- distributed_game/PROTOCOL.md | 105 +++++++-- distributed_game/README.md | 5 +- distributed_game/src/celld_zone/object.rs | 21 +- distributed_game/src/host.rs | 55 ++++- distributed_game/src/lib.rs | 2 +- distributed_game/src/net/mod.rs | 27 ++- distributed_game/src/protocol/messages.rs | 157 +++++++++++-- distributed_game/src/surface.rs | 12 +- distributed_game/src/zone.rs | 13 +- distributed_game/tests/celld_zone/main.rs | 56 ++++- distributed_game/tests/fixtures/messages.json | 212 ++++++++++++++++++ distributed_game/tests/golden/main.rs | 154 +++++++++++-- distributed_game/tests/net_e2e/main.rs | 61 +++++ distributed_game/tests/replication/main.rs | 3 + distributed_game/tests/zone_runtime/arena.rs | 6 + distributed_game/tests/zone_runtime/main.rs | 37 +++ js/src/game/connection.ts | 49 +++- js/src/game/index.ts | 2 + js/src/game/messages.ts | 60 ++++- js/tests/fixtures/game-harness.mjs | 6 +- js/tests/game-codec.test.mjs | 39 +++- js/tests/game-connection.test.mjs | 96 ++++++++ 22 files changed, 1068 insertions(+), 110 deletions(-) diff --git a/distributed_game/PROTOCOL.md b/distributed_game/PROTOCOL.md index b9c76fb94..f98bceeb7 100644 --- a/distributed_game/PROTOCOL.md +++ b/distributed_game/PROTOCOL.md @@ -1,4 +1,4 @@ -# distributed_game wire protocol — version 1 +# distributed_game wire protocol — version 2 (and 1) Normative specification of the binary protocol spoken between a `distributed_game` zone server and its clients. The Rust implementation @@ -8,6 +8,11 @@ independent implementations live in `tests/fixtures/*.json` (§9). Key words MUST / MUST NOT / SHOULD / MAY are used as in RFC 2119. +Two versions are specified: 2 (current) and 1. They differ only in the +handshake: Hello's `protocol_version` is negotiated (§6.0) and Welcome v2 +appends the server's runtime `features` and the zone's world `origin` +(§6.3). Everything else is the same in both. + --- ## 1. Transport and framing @@ -186,6 +191,30 @@ replication priorities are not transmitted. ## 6. Messages +### 6.0 Version negotiation + +- A client offers the newest version it speaks in Hello + `protocol_version`. +- A server that speaks the offered version answers in it: a v1 Hello gets + a v1 Welcome, a v2 Hello a v2 Welcome. Every later frame is the same in + both versions. +- A server that does not speak it MUST answer `Error(VERSION_MISMATCH)` + whose message starts with `protocol` (e.g. `protocol 3 unsupported`) + and close, **before** verifying the ticket or reconnect token, so + neither is consumed. (Content mismatches use the same code with a + message starting `content version`.) Servers built before version 2 + already behave this way: the native server sends `protocol 2 != 1` and + closes the socket normally, a celld zone sends `protocol version` and + closes with 1008. +- On that refusal a client MAY send the same Hello (same ticket or token) + once more with an older version. The JS SDK falls back from 2 to 1 once + per host and remembers it for later connections to the same URL; it + surfaces neither the Error nor the close. +- The Welcome is decoded in the version the client offered: v2 bytes are + the v1 bytes plus a tail, and trailing bytes stay a decode error. +- A client MUST NOT send a message gated by a runtime feature (`Focus`) + unless the Welcome announced it: a v1 Welcome announces none. + ### 6.1 Type bytes | Byte | Direction | Message | @@ -226,7 +255,7 @@ Focus := 0x07, f32 x, f32 y ``` - **Hello** MUST be the first message, sent within the server's hello - timeout (default 5 s). `protocol_version` MUST be `1`. + timeout (default 5 s). `protocol_version` is `2` or `1` (§6.0). `content_version` MUST equal the server's when the server pins one. `capabilities` are game-defined bits, opaque to the runtime. For a fresh join `ticket` is a signed admission ticket (§8) and `reconnect_token` is @@ -263,15 +292,18 @@ Focus := 0x07, f32 x, f32 y `Error(WAKE_DENIED)` (the connection stays open). Servers that keep observed zones running anyway (the native `net` server) ignore it; players' are ignored. -- **Focus**: where an observer is looking, in the zone's local - coordinates (e.g. its player next door, seen from this zone). The zone - decides what it means; by default the observer's view centre moves - there and keeps its radius. It only steers what that observer is sent. - Clients SHOULD throttle it (the JS SDK: at most every 200 ms, after a - move of 0.25 or more) and send it again after each Welcome. Players' - are ignored; non-finite values are ignored. - -Worked example — Hello (`version 1, content "c1", caps 5, ticket "t.k.n"`): +- **Focus** (feature `FOCUS`; only to servers whose Welcome announces + it): where an observer is looking, in the zone's local coordinates + (e.g. its player next door, seen from this zone: world position minus + the zone's Welcome `origin`). The zone decides what it means; by + default the observer's view centre moves there and keeps its radius. + It only steers what that observer is sent. Clients SHOULD throttle it (the JS + SDK: at most every 200 ms, after a move of 0.25 or more) and send it + again after each Welcome. Players' are ignored; non-finite values are + ignored. + +Worked example — Hello (`version 1, content "c1", caps 5, ticket "t.k.n"`; +`hello_v2` in the fixtures is the same with `02` as the second byte): ``` 01 type Hello @@ -293,6 +325,8 @@ Welcome := 0x80, string session_id, string zone_id, u8 role, varuint tick_rate, varuint patch_rate, varuint server_tick, f64 server_time, string reconnect_token, string content_version, schema + [v2: varuint features, bool placed, + [placed: f64 origin_x, f64 origin_y]] Delta := 0x81, see §7 Event := 0x82, varuint type_id, bytes payload Pong := 0x83, f64 client_time, f64 server_time @@ -309,6 +343,13 @@ Redirect := 0x89, string zone_id, string url rotated on every Welcome (empty for observers). A client MUST discard all mirrored entities on Welcome; the server resends creates afterwards. `tick_rate`/`patch_rate` are Hz. + v2 only: `features` are runtime feature bits (defined by the runtime, + unlike Hello's game-defined `capabilities`): bit 0 `FOCUS` = the server + handles `Focus`; unknown bits MUST be ignored. `origin` (when `placed`) + is where the zone sits in the world: world = origin + zone-local, for + zones whose positions are local to one place in a continuous world + (`Zone::origin`). A client that knows its player's world position + focuses each placed zone it observes on `world - origin` by itself. - **Event**: reliable, ordered, game-defined (`type_id`, `payload`). Events are queued when emitted and may arrive before the Delta of the tick that produced them. @@ -349,7 +390,7 @@ Redirect := 0x89, string zone_id, string url grant it). Errors during the handshake are followed by close; `BAD_MESSAGE`, `RATE_LIMITED` and `WAKE_DENIED` after joining are not. -Worked example — Welcome (session "s1", zone "z1", player, 30 Hz ticks, +Worked example — Welcome v1 (session "s1", zone "z1", player, 30 Hz ticks, 20 Hz patches, tick 42, time 1000.25 ms, token "rt", content "c1", schema of `fixtures/messages.json`): @@ -380,6 +421,18 @@ schema of `fixtures/messages.json`): 01 78 06 00000000 00008042 0c "x" quant_f32 min 0 max 64 bits 12 ``` +Welcome v2 (`welcome_v2`: the same fields with role observer, then the +v2 tail): + +``` +01 features FOCUS +01 placed +00 00 00 00 00 00 48 c0 origin_x -48 +00 00 00 00 81 84 2e 41 origin_y 1000000.5 +``` + +`welcome_v2_unplaced` (player) ends `01 00`: features `FOCUS`, not placed. + Redirect (`fixtures/messages.json` → `redirect`): `89 0a 7265616c6d2e77657374 23 77733a2f2f31302e302e302e323a323536372f7a6f6e65732f7265616c6d2e77657374` = zone `"realm.west"`, url `"ws://10.0.0.2:2567/zones/realm.west"`. @@ -579,7 +632,10 @@ distributed_game --test golden`; a drift fails the test): `q` the quantized integer, `hex` its varuint bytes. - `messages.json` — `{messages: [{name, direction, hex, decoded, known_entities?, schema?}]}`. Delta entries include the schema and the - `known_entities` (`[{id, kind}]`) needed to decode updates. + `known_entities` (`[{id, kind}]`) needed to decode updates. Welcome + entries carry `decoded.protocol_version` (the version to decode them + in); v2 ones add `features` and `origin` (`{x, y}` or `null`). Hellos + cover both versions (`hello` v1, `hello_v2`). - `authoring.json` (`--test authoring`) — a sample game surface and input / event payloads encoded with the typed payload codec (§11), with their expected TypeScript values; generated clients are round-tripped @@ -597,9 +653,22 @@ JSON conventions in `decoded`: ## 10. Versioning -`protocol_version` changes on any incompatible wire change. Message type -bytes `0x07..0x7f` and `0x8a..0xff`, and field type tags `10..255`, are -reserved for future versions. +`protocol_version` changes on any incompatible wire change, and is +negotiated (§6.0): servers keep answering older versions they still +speak, so clients and servers may be deployed in either order. Message +type bytes `0x08..0x7f` and `0x8a..0xff`, and field type tags `10..255`, +are reserved for future versions. + +Version 2 (negotiated; v1 peers keep working unchanged): + +- Welcome appends `features` and the optional `origin` (§6.3). +- `0x07 Focus` is gated on the `FOCUS` feature: a client sends it only + to servers that announce it, so a server built before it (which + answers `Error(BAD_MESSAGE)`; a celld zone also closes the connection) + never receives it. +- Payload encodings (fields, inputs, events, §11) are unchanged, so a + game surface keeps `protocolVersion: 1` and generated clients work + with both versions. Additive changes within version 1 (clients SHOULD ignore unknown kick and error codes and treat them as "session over" / "request failed"): @@ -610,9 +679,7 @@ error codes and treat them as "session over" / "request failed"): - Error code `10` ASLEEP. - Error code `11` WAKE_DENIED. - `0x06 Near` (client → server) and the ticket claim `wake`. -- `0x07 Focus` (client → server). A server built before it answers - `Error(BAD_MESSAGE)` (the native server keeps the connection; a celld - zone closes it), so deploy servers before clients that send it. +- `0x07 Focus` (client → server), gated since version 2 (above). Not additive (made before the first release, so the version stays 1): Hello's trailing `bool near` — clients built before it send one byte diff --git a/distributed_game/README.md b/distributed_game/README.md index 71cd9b877..066df5aad 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -1014,7 +1014,10 @@ unlimited rewind. every float bit and costs about a microsecond per step. - **Binary protocol with the schema sent on connect.** Small, diff-friendly frames, and any client can decode them without generated - code. + code. The version is negotiated in the handshake (`PROTOCOL.md` §6.0): + servers answer v1 and v2 clients, the JS SDK falls back to v1 against + an old server and sends feature-gated messages (`Focus`) only to + servers that announce them, so servers and clients deploy in any order. - **Reliable, ordered WebSockets.** Deltas are computed against what each client last received, which only works because nothing is lost or reordered. Loss shows up as delay, which the adaptive interpolation diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index 14158850d..a58bb1f11 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -33,7 +33,7 @@ use crate::facts::{ use crate::handoff::{HandoffRequest, Transfer}; use crate::inbox::ZoneInbound; use crate::netsim::{DelayLine, NetSim, NetSimJson}; -use crate::protocol::{error, ClientMessage, Notice, Role, ServerMessage, PROTOCOL_VERSION}; +use crate::protocol::{error, supports_protocol, ClientMessage, Notice, Role, ServerMessage}; use crate::stamp::FactStamp; use crate::ticket::{NonceCache, TicketKey, VerifiedTicket}; use crate::world::content_rollout::{zone_step, ContentState, ZoneStep, CONTENT_ROLLOUT_KIND}; @@ -1629,8 +1629,11 @@ impl ZoneObject { } async fn hello(&self, ws: &WebSocket, hello: crate::protocol::Hello) -> Result<()> { - if hello.protocol_version != PROTOCOL_VERSION { - reject(ws, error::VERSION_MISMATCH, "protocol version"); + // Answer in the client's version; refuse others before the ticket + // is touched, so the client can retry with an older one (§6.0). + if !supports_protocol(hello.protocol_version) { + let message = format!("protocol {} unsupported", hello.protocol_version); + reject(ws, error::VERSION_MISMATCH, &message); return Ok(()); } let cold = self.inner.borrow().host.is_none(); @@ -1733,6 +1736,7 @@ impl ZoneObject { reject(ws, error::ZONE_UNAVAILABLE, "zone asleep"); return Ok(()); }; + let host = host.speaking(hello.protocol_version); match &transfer { Some(transfer) => { host.join_transfer(transfer, hello.capabilities, &hello.content_version) @@ -1746,11 +1750,12 @@ impl ZoneObject { reject(ws, error::RECONNECT_FAILED, "zone restarted; rejoin"); return Ok(()); }; - host.reconnect_with_version( - &hello.reconnect_token, - hello.capabilities, - &hello.content_version, - ) + host.speaking(hello.protocol_version) + .reconnect_with_version( + &hello.reconnect_token, + hello.capabilities, + &hello.content_version, + ) }; match joined { Ok(joined) => { diff --git a/distributed_game/src/host.rs b/distributed_game/src/host.rs index 9acbb921c..780e9be9a 100644 --- a/distributed_game/src/host.rs +++ b/distributed_game/src/host.rs @@ -41,7 +41,9 @@ use crate::lifecycle::{ ConnectionLost, JoinVia, LeftReason, LostCause, PlayerDisconnected, PlayerJoined, PlayerLeft, PlayerReconnected, }; -use crate::protocol::{error, kick, Notice, Role, ServerMessage, Welcome}; +use crate::protocol::{ + error, feature, kick, Notice, Role, ServerMessage, Welcome, PROTOCOL_VERSION, +}; use crate::replication::{ClientReplica, DeltaStats, EncodedWorld, WorldView}; use crate::rewind::{Rewind, RewindConfig, RewindFrame}; use crate::rng::ZoneRng; @@ -852,6 +854,8 @@ pub struct ZoneHost { /// The last patch's world: the next one shares its unchanged entities /// and encodes each change once for every client. last_world: Option, + /// Protocol version of the next admission's client ([`ZoneHost::speaking`]). + next_protocol: Option, } macro_rules! hook { @@ -929,6 +933,7 @@ impl ZoneHost { pending: AdvanceReport::default(), hash_ticks: false, last_world: None, + next_protocol: None, }; host.process_kicks(); host @@ -1103,6 +1108,22 @@ impl ZoneHost { std::mem::take(&mut self.core.outbound) } + /// The protocol version of the client the next + /// [`join`](Self::join) / [`join_transfer`](Self::join_transfer) / + /// [`reconnect`](Self::reconnect) admits (its Hello's + /// `protocol_version`, already checked with + /// [`supports_protocol`](crate::protocol::supports_protocol)): its + /// Welcome is encoded in it. Applies to that one call; without it the + /// client speaks [`PROTOCOL_VERSION`]. + /// + /// ```ignore + /// host.speaking(hello.protocol_version).join(&claims, hello.capabilities, &hello.content_version) + /// ``` + pub fn speaking(&mut self, protocol_version: u32) -> &mut Self { + self.next_protocol = Some(protocol_version); + self + } + /// Admit a verified ticket holder. pub fn join( &mut self, @@ -1151,6 +1172,7 @@ impl ZoneHost { content_version: &str, handoff: Option<(&str, Vec)>, ) -> Result { + let protocol = self.next_protocol.take().unwrap_or(PROTOCOL_VERSION); if self.disposed { return Err(JoinError::new(error::NOT_FOUND, "zone disposed")); } @@ -1213,7 +1235,7 @@ impl ZoneHost { } slot.info.extensions = claims.extensions.clone(); } - self.bind_player(player, client, capabilities); + self.bind_player(player, client, capabilities, protocol); let zone = self.core.zone_id.clone(); if let Some((from, carried)) = handoff.take() { self.core.emit_lifecycle( @@ -1266,7 +1288,7 @@ impl ZoneHost { transitions: 0, }, ); - self.bind_player(player, client, capabilities); + self.bind_player(player, client, capabilities, protocol); let zone = self.core.zone_id.clone(); match handoff.take() { Some((from, carried)) => { @@ -1315,7 +1337,7 @@ impl ZoneHost { }, ); self.attach_client(client, Viewer::Observer(observer)); - self.welcome(client, session_id, String::new(), Role::Observer); + self.welcome(client, session_id, String::new(), Role::Observer, protocol); hook!(self.on_observe(observer)); Joined { client, @@ -1360,6 +1382,7 @@ impl ZoneHost { capabilities: u32, content_version: Option<&str>, ) -> Result { + let protocol = self.next_protocol.take().unwrap_or(PROTOCOL_VERSION); let player = (!token.is_empty() && !self.disposed) .then(|| { self.core @@ -1385,7 +1408,7 @@ impl ZoneHost { } self.wake(); let client = ClientId(self.core.next_id()); - self.bind_player(player, client, capabilities); + self.bind_player(player, client, capabilities, protocol); if let Some(version) = content_version { if let Some(slot) = self.core.players.get_mut(&player) { slot.content_version = version.to_string(); @@ -1544,7 +1567,13 @@ impl ZoneHost { ); } - fn bind_player(&mut self, player: PlayerId, client: ClientId, capabilities: u32) { + fn bind_player( + &mut self, + player: PlayerId, + client: ClientId, + capabilities: u32, + protocol: u32, + ) { let token = (self.token_source)(); let slot = self.core.players.get_mut(&player).expect("player exists"); let previous = slot.client.replace(client); @@ -1568,11 +1597,19 @@ impl ZoneHost { self.core.outbound.push(Outbound::Close { client: old }); } self.attach_client(client, Viewer::Player(player)); - self.welcome(client, session_id, token, Role::Player); + self.welcome(client, session_id, token, Role::Player, protocol); } - fn welcome(&mut self, client: ClientId, session_id: String, token: String, role: Role) { + fn welcome( + &mut self, + client: ClientId, + session_id: String, + token: String, + role: Role, + protocol_version: u32, + ) { let welcome = ServerMessage::Welcome(Welcome { + protocol_version, session_id, zone_id: self.core.zone_id.clone(), role, @@ -1583,6 +1620,8 @@ impl ZoneHost { reconnect_token: token, content_version: self.core.config.content_version.clone(), schema: self.schema.clone(), + features: feature::ALL, + origin: self.zone.origin(), }); self.core.server_message(client, &welcome); } diff --git a/distributed_game/src/lib.rs b/distributed_game/src/lib.rs index c0430baa5..d8dd45a32 100644 --- a/distributed_game/src/lib.rs +++ b/distributed_game/src/lib.rs @@ -121,7 +121,7 @@ pub use inbox::{Delivered, NoMessages, ZoneInbound}; pub use input::{ConsumePolicy, InputBuffer, InputCredits, PushOutcome, SequencedInput}; pub use interest::{InterestConfig, UpdateRates, View}; pub use prediction::{PredictContext, PredictStep}; -pub use protocol::{ClientMessage, ClientMirror, Role, ServerMessage, PROTOCOL_VERSION}; +pub use protocol::{ClientMessage, ClientMirror, Origin, Role, ServerMessage, PROTOCOL_VERSION}; pub use pure::GamePure; pub use replay::{replay, Recorder, Recording}; pub use replication::{EntityState, WorldView}; diff --git a/distributed_game/src/net/mod.rs b/distributed_game/src/net/mod.rs index 3a9a1a545..cdc55c6d8 100644 --- a/distributed_game/src/net/mod.rs +++ b/distributed_game/src/net/mod.rs @@ -83,7 +83,10 @@ use crate::handoff::{HandoffRequest, Transfer}; use crate::host::{HostConfig, JoinError, Joined, Outbound, ZoneHost}; use crate::metrics::GameMetrics; use crate::protocol::Role; -use crate::protocol::{error, kick, ClientMessage, Notice, ServerMessage, PROTOCOL_VERSION}; +use crate::protocol::{ + error, kick, supports_protocol, ClientMessage, Notice, ServerMessage, MIN_PROTOCOL_VERSION, + PROTOCOL_VERSION, +}; use crate::rng::ZoneRng; use crate::world::{ BarrierMode, ChildEvent, DottedTopology, FreezeState, HostAddr, LocalOwnership, OwnershipError, @@ -230,6 +233,7 @@ enum ZoneCmd { Join { claims: TicketClaims, transfer: Option, + protocol: u32, capabilities: u32, content_version: String, out: mpsc::Sender, @@ -237,6 +241,7 @@ enum ZoneCmd { }, Reconnect { token: String, + protocol: u32, capabilities: u32, content_version: String, out: mpsc::Sender, @@ -1410,11 +1415,13 @@ fn handle_cmd( ZoneCmd::Join { claims, transfer, + protocol, capabilities, content_version, out, reply, } => { + let host = host.speaking(protocol); let result = match &transfer { Some(transfer) => host.join_transfer(transfer, capabilities, &content_version), None => host.join(&claims, capabilities, &content_version), @@ -1426,12 +1433,17 @@ fn handle_cmd( } ZoneCmd::Reconnect { token, + protocol, capabilities, content_version, out, reply, } => { - let result = host.reconnect_with_version(&token, capabilities, &content_version); + let result = host.speaking(protocol).reconnect_with_version( + &token, + capabilities, + &content_version, + ); if let Ok(joined) = &result { conns.insert(joined.client, out); } @@ -1660,11 +1672,16 @@ where Ok(Some(Ok(_))) => return reject(socket, error::BAD_MESSAGE, "binary frames only").await, _ => return, }; - if hello.protocol_version != PROTOCOL_VERSION { + // Answer in the client's version; refuse others before the ticket is + // touched, so the client can retry with an older one (§6.0). + if !supports_protocol(hello.protocol_version) { return reject( socket, error::VERSION_MISMATCH, - format!("protocol {} != {PROTOCOL_VERSION}", hello.protocol_version), + format!( + "protocol {} unsupported ({MIN_PROTOCOL_VERSION}..={PROTOCOL_VERSION})", + hello.protocol_version + ), ) .await; } @@ -1756,6 +1773,7 @@ where ZoneCmd::Join { claims, transfer, + protocol: hello.protocol_version, capabilities: hello.capabilities, content_version: hello.content_version, out: out_tx, @@ -1764,6 +1782,7 @@ where } else { ZoneCmd::Reconnect { token: hello.reconnect_token, + protocol: hello.protocol_version, capabilities: hello.capabilities, content_version: hello.content_version, out: out_tx, diff --git a/distributed_game/src/protocol/messages.rs b/distributed_game/src/protocol/messages.rs index 3def9af5f..71354f824 100644 --- a/distributed_game/src/protocol/messages.rs +++ b/distributed_game/src/protocol/messages.rs @@ -5,8 +5,28 @@ use std::collections::BTreeMap; use super::wire::{DecodeError, Reader, Writer}; use crate::schema::{EntitySchema, FieldValue}; -/// Wire protocol version spoken by this crate. -pub const PROTOCOL_VERSION: u32 = 1; +/// Newest wire protocol version spoken by this crate (what its clients +/// offer in Hello). +pub const PROTOCOL_VERSION: u32 = 2; + +/// Oldest protocol version this crate's servers still answer +/// (`PROTOCOL.md` §6.0): a v1 client gets a v1 Welcome. +pub const MIN_PROTOCOL_VERSION: u32 = 1; + +/// Whether this crate's servers answer a Hello offering `version`. +pub fn supports_protocol(version: u32) -> bool { + (MIN_PROTOCOL_VERSION..=PROTOCOL_VERSION).contains(&version) +} + +/// Runtime feature bits a server announces in Welcome v2 (`features`). +/// Unlike Hello's game-defined `capabilities`, the runtime defines these: a +/// client sends a message a feature covers only when the bit is set. +pub mod feature { + /// The server handles `Focus` (observers steer their view). + pub const FOCUS: u32 = 1 << 0; + /// Every feature this crate's runtime implements. + pub const ALL: u32 = FOCUS; +} /// Message type bytes. pub mod tag { @@ -99,7 +119,10 @@ pub mod error { /// Client handshake. #[derive(Debug, Clone, PartialEq, Default)] pub struct Hello { - /// Must equal [`PROTOCOL_VERSION`]. + /// The newest protocol version the client speaks; the server answers + /// in it or refuses with `Error(VERSION_MISMATCH)` (message starting + /// `protocol`) before touching the ticket, and the client may retry + /// with an older one. See [`supports_protocol`]. pub protocol_version: u32, /// Game content version the client was built against. pub content_version: String, @@ -252,9 +275,41 @@ pub enum Role { Observer, } +/// Where a zone sits in the world: its origin in world coordinates, so +/// world = origin + zone-local (`Zone::origin`). Clients use it to say +/// where their player is in a neighbour's coordinates (`Focus`). +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct Origin { + /// World x of the zone's local (0, 0). + pub x: f64, + /// World y of the zone's local (0, 0). + pub y: f64, +} + +impl Origin { + /// Origin at world `(x, y)`. + pub fn new(x: f64, y: f64) -> Self { + Self { x, y } + } + + /// A world point in this zone's local coordinates. + pub fn to_local(self, x: f64, y: f64) -> (f32, f32) { + ((x - self.x) as f32, (y - self.y) as f32) + } + + /// A zone-local point in world coordinates. + pub fn to_world(self, x: f32, y: f32) -> (f64, f64) { + (self.x + f64::from(x), self.y + f64::from(y)) + } +} + /// Server handshake reply. #[derive(Debug, Clone, PartialEq)] pub struct Welcome { + /// Protocol version this Welcome is encoded in: the client's Hello + /// version (not itself on the wire; decoders take it from the mirror, + /// [`EntityKinds::protocol_version`]). + pub protocol_version: u32, /// Opaque session id (stable across reconnects of the same player). pub session_id: String, /// Zone joined. @@ -275,6 +330,11 @@ pub struct Welcome { pub content_version: String, /// Replicated entity schema. pub schema: EntitySchema, + /// v2: runtime features of the server ([`feature`]); 0 in v1. + pub features: u32, + /// v2: where the zone sits in the world, when it says + /// (`Zone::origin`); always `None` in v1. + pub origin: Option, } /// Full entity state for a create. @@ -381,8 +441,14 @@ pub enum ServerMessage { }, } -/// What a Delta decoder/encoder needs to know about the client's view. +/// What a decoder/encoder needs to know about the client's view: the +/// negotiated protocol version (Welcome), the schema and the known ids +/// (Deltas). pub trait EntityKinds { + /// Protocol version the server answers in (default: the newest). + fn protocol_version(&self) -> u32 { + PROTOCOL_VERSION + } /// Schema from the Welcome. fn schema(&self) -> Option<&EntitySchema>; /// Kind of a known entity id. @@ -429,6 +495,12 @@ impl ServerMessage { .string(&m.reconnect_token) .string(&m.content_version); m.schema.encode(&mut w); + if m.protocol_version >= 2 { + w.varuint(m.features).bool(m.origin.is_some()); + if let Some(origin) = m.origin { + w.f64(origin.x).f64(origin.y); + } + } } Self::Delta(d) => encode_delta(&mut w, d, kinds)?, Self::Event { type_id, payload } => { @@ -476,22 +548,7 @@ impl ServerMessage { pub fn decode(bytes: &[u8], kinds: &dyn EntityKinds) -> Result { let mut r = Reader::new(bytes); let message = match r.u8()? { - tag::WELCOME => Self::Welcome(Welcome { - session_id: r.string()?, - zone_id: r.string()?, - role: match r.u8()? { - 0 => Role::Player, - 1 => Role::Observer, - _ => return Err(DecodeError::Invalid("role")), - }, - tick_rate: r.varuint()?, - patch_rate: r.varuint()?, - server_tick: r.varuint()?, - server_time: r.f64()?, - reconnect_token: r.string()?, - content_version: r.string()?, - schema: EntitySchema::decode(&mut r)?, - }), + tag::WELCOME => Self::Welcome(decode_welcome(&mut r, kinds.protocol_version())?), tag::DELTA => Self::Delta(decode_delta(&mut r, kinds)?), tag::EVENT => Self::Event { type_id: r.varuint()?, @@ -536,6 +593,38 @@ impl ServerMessage { } } +fn decode_welcome(r: &mut Reader<'_>, protocol_version: u32) -> Result { + let mut welcome = Welcome { + protocol_version, + session_id: r.string()?, + zone_id: r.string()?, + role: match r.u8()? { + 0 => Role::Player, + 1 => Role::Observer, + _ => return Err(DecodeError::Invalid("role")), + }, + tick_rate: r.varuint()?, + patch_rate: r.varuint()?, + server_tick: r.varuint()?, + server_time: r.f64()?, + reconnect_token: r.string()?, + content_version: r.string()?, + schema: EntitySchema::decode(r)?, + features: 0, + origin: None, + }; + if protocol_version >= 2 { + welcome.features = r.varuint()?; + if r.bool()? { + welcome.origin = Some(Origin { + x: r.f64()?, + y: r.f64()?, + }); + } + } + Ok(welcome) +} + fn kind_lookup( id: u32, created: &BTreeMap, @@ -651,8 +740,11 @@ pub struct MirrorEntity { /// /// This is the normative behaviour a client implementation (e.g. JS) must /// match; it is also what the crate's own tests use. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct ClientMirror { + /// Protocol version the client offered in Hello (Welcomes decode in + /// it); [`PROTOCOL_VERSION`] by default. + pub protocol_version: u32, /// Last Welcome. pub welcome: Option, /// Known entities. @@ -663,7 +755,22 @@ pub struct ClientMirror { pub ack_seq: u32, } +impl Default for ClientMirror { + fn default() -> Self { + Self { + protocol_version: PROTOCOL_VERSION, + welcome: None, + entities: BTreeMap::new(), + server_tick: 0, + ack_seq: 0, + } + } +} + impl EntityKinds for ClientMirror { + fn protocol_version(&self) -> u32 { + self.protocol_version + } fn schema(&self) -> Option<&EntitySchema> { self.welcome.as_ref().map(|w| &w.schema) } @@ -678,6 +785,14 @@ impl ClientMirror { Self::default() } + /// Empty mirror of a client that offered `protocol_version` in Hello. + pub fn speaking(protocol_version: u32) -> Self { + Self { + protocol_version, + ..Self::default() + } + } + /// Decode a frame and apply it. Welcome resets all entities. pub fn apply(&mut self, frame: &[u8]) -> Result { let message = ServerMessage::decode(frame, self)?; diff --git a/distributed_game/src/surface.rs b/distributed_game/src/surface.rs index 3b787da0b..b310aa3ae 100644 --- a/distributed_game/src/surface.rs +++ b/distributed_game/src/surface.rs @@ -26,11 +26,16 @@ use serde::{Deserialize, Serialize}; use crate::codec::WireType; use crate::prediction::{PredictContext, PredictStep}; -use crate::protocol::PROTOCOL_VERSION; use crate::pure::GamePure; use crate::schema::{Correction, EntitySchema}; use crate::{GameEvent, GameInput, Wire, Zone}; +/// Wire protocol version whose payload encodings (fields, inputs, events) +/// a surface describes. They have not changed since 1: protocol 2 only +/// added handshake negotiation (Welcome `features` and `origin`), which the +/// SDK handles, so generated clients stay valid for both. +pub const SURFACE_PROTOCOL_VERSION: u32 = 1; + /// Version of the [`GameSurface`] JSON shape. pub const GAME_SURFACE_SCHEMA_VERSION: u32 = 3; @@ -42,7 +47,8 @@ pub struct GameSurface { pub schema_version: u32, /// Surface name (generated TypeScript prefixes exports with it). pub name: String, - /// Zone wire protocol version (`PROTOCOL_VERSION`). + /// Zone wire protocol version of the payload encodings + /// ([`SURFACE_PROTOCOL_VERSION`]). pub protocol_version: u32, /// Replicated kinds in schema order. pub kinds: Vec, @@ -238,7 +244,7 @@ impl GameSurface { Self { schema_version: GAME_SURFACE_SCHEMA_VERSION, name: name.into(), - protocol_version: PROTOCOL_VERSION, + protocol_version: SURFACE_PROTOCOL_VERSION, kinds: Vec::new(), input: None, events: Vec::new(), diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index 69bda36b3..95190e550 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -14,7 +14,7 @@ use crate::claims::TicketExtensions; use crate::facts::FactSettlement; use crate::host::Ctx; use crate::interest::View; -use crate::protocol::Role; +use crate::protocol::{Origin, Role}; use crate::replication::WorldView; use crate::rewind::RewindFrame; use crate::schema::EntitySchema; @@ -297,6 +297,17 @@ pub trait Zone: Sized + Send + 'static { } } + /// Where this zone sits in the world: the world position of its local + /// (0, 0), for zones whose positions are local to a place in one + /// continuous world (a grid of rooms, map tiles). Sent in Welcome, so a + /// client that knows where its player is focuses every neighbour it + /// watches by itself. Derive it from the + /// zone's config (e.g. the room parsed from its zone id). `None` + /// (default): the zone is not placed and clients focus it only by hand. + fn origin(&self) -> Option { + None + } + /// Decode raw input bytes from the wire; `None` drops the input. fn decode_input(&self, player: PlayerId, payload: &[u8]) -> Option; diff --git a/distributed_game/tests/celld_zone/main.rs b/distributed_game/tests/celld_zone/main.rs index 9dd6094bc..f602061f4 100644 --- a/distributed_game/tests/celld_zone/main.rs +++ b/distributed_game/tests/celld_zone/main.rs @@ -34,7 +34,9 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; -use distributed_game::protocol::{error, ClientMessage, Hello, ServerMessage, PROTOCOL_VERSION}; +use distributed_game::protocol::{ + error, feature, ClientMessage, ClientMirror, Hello, ServerMessage, Welcome, PROTOCOL_VERSION, +}; use distributed_game::ticket::TicketKey; use distributed_game::{Role, TicketClaims}; use futures_util::{SinkExt, StreamExt}; @@ -118,6 +120,8 @@ enum Got { struct Player { ws: Ws, seq: u32, + /// Protocol version offered in Hello (Welcomes decode in it). + protocol: u32, } impl Player { @@ -130,7 +134,11 @@ impl Player { let (ws, _) = tokio_tungstenite::connect_async(url) .await .map_err(|e| e.to_string())?; - Ok(Self { ws, seq: 0 }) + Ok(Self { + ws, + seq: 0, + protocol: PROTOCOL_VERSION, + }) } async fn send(&mut self, message: ClientMessage) { @@ -146,7 +154,7 @@ impl Player { async fn hello_near(&mut self, ticket: &str, near: bool) { self.send(ClientMessage::Hello(Hello { - protocol_version: PROTOCOL_VERSION, + protocol_version: self.protocol, content_version: String::new(), capabilities: 0, ticket: ticket.into(), @@ -172,11 +180,13 @@ impl Player { let left = deadline.saturating_duration_since(Instant::now()); let frame = tokio::time::timeout(left, self.ws.next()).await.ok()?; match frame { - Some(Ok(Message::Binary(bytes))) => match ServerMessage::decode(&bytes, &()) { - Ok(message) => return Some(Got::Server(message)), - // Deltas need the Welcome's schema; these tests skip them. - Err(_) => continue, - }, + Some(Ok(Message::Binary(bytes))) => { + match ServerMessage::decode(&bytes, &ClientMirror::speaking(self.protocol)) { + Ok(message) => return Some(Got::Server(message)), + // Deltas need the Welcome's schema; these tests skip them. + Err(_) => continue, + } + } Some(Ok(Message::Close(frame))) => { return Some(Got::Closed(frame.map(|f| u16::from(f.code)))) } @@ -209,8 +219,12 @@ impl Player { } async fn welcome(&mut self) { + self.welcome_message().await; + } + + async fn welcome_message(&mut self) -> Welcome { self.expect(Duration::from_secs(20), "Welcome", |got| match got { - Got::Server(ServerMessage::Welcome(_)) => Some(()), + Got::Server(ServerMessage::Welcome(w)) => Some(w.clone()), Got::Server(ServerMessage::Error { code, message }) => { panic!("join refused: {code} {message}") } @@ -312,6 +326,30 @@ async fn single_node() { assert_eq!(ana.total(2).await, 5); until(&base, "/tally/ana", long, |v| exactly_once(v, 5)).await; + // The zone answers in the client's protocol version: v1 Welcome for a + // v1 client, v2 (runtime features) for v2; a version it does not speak + // is refused before the ticket is used, so the same ticket then joins. + let mut old = Player::connect(&base, "room-a").await; + old.protocol = 1; + old.hello(&ticket_as("old", "room-a", Role::Observer)).await; + let w = old.welcome_message().await; + assert_eq!((w.protocol_version, w.features, w.origin), (1, 0, None)); + let mut new = Player::connect(&base, "room-a").await; + new.hello(&ticket_as("new", "room-a", Role::Observer)).await; + let w = new.welcome_message().await; + assert_eq!(w.protocol_version, 2); + assert_eq!(w.features & feature::FOCUS, feature::FOCUS); + new.send(ClientMessage::Focus { x: 1.0, y: 2.0 }).await; + let mut future = Player::connect(&base, "room-a").await; + future.protocol = PROTOCOL_VERSION + 1; + let same = ticket_as("future", "room-a", Role::Observer); + future.hello(&same).await; + assert_eq!(future.refused().await, error::VERSION_MISMATCH); + let mut future = Player::connect(&base, "room-a").await; + future.hello(&same).await; + future.welcome().await; + drop((old, new, future)); + // A used ticket is refused. let mut replay = Player::connect(&base, "room-a").await; replay.hello(&first).await; diff --git a/distributed_game/tests/fixtures/messages.json b/distributed_game/tests/fixtures/messages.json index 3c2ae4981..40d01f7fa 100644 --- a/distributed_game/tests/fixtures/messages.json +++ b/distributed_game/tests/fixtures/messages.json @@ -107,10 +107,25 @@ "hex": "070000ccc100005040", "name": "focus" }, + { + "decoded": { + "capabilities": 5, + "content_version": "c1", + "near": false, + "protocol_version": 2, + "reconnect_token": "", + "ticket": "t.k.n", + "type": "Hello" + }, + "direction": "client_to_server", + "hex": "01020263310505742e6b2e6e0000", + "name": "hello_v2" + }, { "decoded": { "content_version": "c1", "patch_rate": 20, + "protocol_version": 1, "reconnect_token": "rt", "role": "player", "schema": [ @@ -201,6 +216,203 @@ "hex": "80027331027a31001e142a0000000000428f400272740263310201046865726f0a05616c69766500056c6576656c010268700204676f6c640302647a04057370656564050178060000c8c20000c8421003796177070a046e616d6508037461670902036f72620101780600000000000080420c", "name": "welcome" }, + { + "decoded": { + "content_version": "c1", + "features": 1, + "origin": { + "x": -48.0, + "y": 1000000.5 + }, + "patch_rate": 20, + "protocol_version": 2, + "reconnect_token": "rt", + "role": "observer", + "schema": [ + { + "fields": [ + { + "name": "alive", + "type": "bool" + }, + { + "name": "level", + "type": "u8" + }, + { + "name": "hp", + "type": "u16" + }, + { + "name": "gold", + "type": "u32" + }, + { + "name": "dz", + "type": "i32" + }, + { + "name": "speed", + "type": "f32" + }, + { + "bits": 16, + "max": { + "bits": "42c80000", + "f32": 100.0 + }, + "min": { + "bits": "c2c80000", + "f32": -100.0 + }, + "name": "x", + "type": "quant_f32" + }, + { + "bits": 10, + "name": "yaw", + "type": "angle" + }, + { + "name": "name", + "type": "short_str" + }, + { + "name": "tag", + "type": "bytes" + } + ], + "id": 1, + "name": "hero" + }, + { + "fields": [ + { + "bits": 12, + "max": { + "bits": "42800000", + "f32": 64.0 + }, + "min": { + "bits": "00000000", + "f32": 0.0 + }, + "name": "x", + "type": "quant_f32" + } + ], + "id": 2, + "name": "orb" + } + ], + "server_tick": 42, + "server_time": 1000.25, + "session_id": "s1", + "tick_rate": 30, + "type": "Welcome", + "zone_id": "z1" + }, + "direction": "server_to_client", + "hex": "80027331027a31011e142a0000000000428f400272740263310201046865726f0a05616c69766500056c6576656c010268700204676f6c640302647a04057370656564050178060000c8c20000c8421003796177070a046e616d6508037461670902036f72620101780600000000000080420c010100000000000048c00000000081842e41", + "name": "welcome_v2" + }, + { + "decoded": { + "content_version": "c1", + "features": 1, + "origin": null, + "patch_rate": 20, + "protocol_version": 2, + "reconnect_token": "rt", + "role": "player", + "schema": [ + { + "fields": [ + { + "name": "alive", + "type": "bool" + }, + { + "name": "level", + "type": "u8" + }, + { + "name": "hp", + "type": "u16" + }, + { + "name": "gold", + "type": "u32" + }, + { + "name": "dz", + "type": "i32" + }, + { + "name": "speed", + "type": "f32" + }, + { + "bits": 16, + "max": { + "bits": "42c80000", + "f32": 100.0 + }, + "min": { + "bits": "c2c80000", + "f32": -100.0 + }, + "name": "x", + "type": "quant_f32" + }, + { + "bits": 10, + "name": "yaw", + "type": "angle" + }, + { + "name": "name", + "type": "short_str" + }, + { + "name": "tag", + "type": "bytes" + } + ], + "id": 1, + "name": "hero" + }, + { + "fields": [ + { + "bits": 12, + "max": { + "bits": "42800000", + "f32": 64.0 + }, + "min": { + "bits": "00000000", + "f32": 0.0 + }, + "name": "x", + "type": "quant_f32" + } + ], + "id": 2, + "name": "orb" + } + ], + "server_tick": 42, + "server_time": 1000.25, + "session_id": "s1", + "tick_rate": 30, + "type": "Welcome", + "zone_id": "z1" + }, + "direction": "server_to_client", + "hex": "80027331027a31001e142a0000000000428f400272740263310201046865726f0a05616c69766500056c6576656c010268700204676f6c640302647a04057370656564050178060000c8c20000c8421003796177070a046e616d6508037461670902036f72620101780600000000000080420c0100", + "name": "welcome_v2_unplaced" + }, { "decoded": { "ack_seq": 0, diff --git a/distributed_game/tests/golden/main.rs b/distributed_game/tests/golden/main.rs index 7f1fa8c92..d763f2faf 100644 --- a/distributed_game/tests/golden/main.rs +++ b/distributed_game/tests/golden/main.rs @@ -9,12 +9,37 @@ use std::collections::BTreeMap; use std::path::PathBuf; use distributed_game::protocol::{ - error, kick, ClientMessage, Delta, EntityCreate, EntityUpdate, Hello, Notice, Reader, - ServerMessage, Welcome, Writer, + error, feature, kick, ClientMessage, Delta, EntityCreate, EntityKinds, EntityUpdate, Hello, + Notice, Reader, ServerMessage, Welcome, Writer, +}; +use distributed_game::{ + EntitySchema, FieldType, FieldValue, KindDef, Origin, Role, PROTOCOL_VERSION, }; -use distributed_game::{EntitySchema, FieldType, FieldValue, KindDef, Role, PROTOCOL_VERSION}; use serde_json::{json, Value}; +/// Schema, known ids and the negotiated protocol version of a fixture. +struct Kinds<'a>(&'a EntitySchema, &'a BTreeMap, u32); + +impl EntityKinds for Kinds<'_> { + fn protocol_version(&self) -> u32 { + self.2 + } + fn schema(&self) -> Option<&EntitySchema> { + Some(self.0) + } + fn kind_of(&self, id: u32) -> Option { + self.1.get(&id).copied() + } +} + +/// Protocol version a server fixture is encoded in (Welcome v2 entries). +fn version_of(m: &ServerMessage) -> u32 { + match m { + ServerMessage::Welcome(w) => w.protocol_version, + _ => PROTOCOL_VERSION, + } +} + fn hex(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } @@ -110,8 +135,10 @@ fn client_json(m: &ClientMessage) -> Value { fn server_json(m: &ServerMessage) -> Value { match m { - ServerMessage::Welcome(w) => json!({ + ServerMessage::Welcome(w) => { + let mut v = json!({ "type": "Welcome", + "protocol_version": w.protocol_version, "session_id": w.session_id, "zone_id": w.zone_id, "role": match w.role { Role::Player => "player", Role::Observer => "observer" }, @@ -122,7 +149,16 @@ fn server_json(m: &ServerMessage) -> Value { "reconnect_token": w.reconnect_token, "content_version": w.content_version, "schema": schema_json(&w.schema), - }), + }); + if w.protocol_version >= 2 { + v["features"] = json!(w.features); + v["origin"] = match w.origin { + Some(o) => json!({ "x": o.x, "y": o.y }), + None => Value::Null, + }; + } + v + } ServerMessage::Delta(d) => json!({ "type": "Delta", "server_tick": d.server_tick, @@ -321,7 +357,7 @@ fn messages() { ( "hello", ClientMessage::Hello(Hello { - protocol_version: PROTOCOL_VERSION, + protocol_version: 1, content_version: "c1".into(), capabilities: 5, ticket: "t.k.n".into(), @@ -332,7 +368,7 @@ fn messages() { ( "hello_reconnect", ClientMessage::Hello(Hello { - protocol_version: PROTOCOL_VERSION, + protocol_version: 1, content_version: "c1".into(), capabilities: 0, ticket: String::new(), @@ -364,7 +400,7 @@ fn messages() { ( "hello_near", ClientMessage::Hello(Hello { - protocol_version: PROTOCOL_VERSION, + protocol_version: 1, content_version: "c1".into(), capabilities: 0, ticket: "t.k.n".into(), @@ -375,6 +411,17 @@ fn messages() { ("near", ClientMessage::Near { near: true }), ("passive", ClientMessage::Near { near: false }), ("focus", ClientMessage::Focus { x: -25.5, y: 3.25 }), + ( + "hello_v2", + ClientMessage::Hello(Hello { + protocol_version: 2, + content_version: "c1".into(), + capabilities: 5, + ticket: "t.k.n".into(), + reconnect_token: String::new(), + near: false, + }), + ), ]; for (name, msg) in client { let bytes = msg.encode(); @@ -386,7 +433,6 @@ fn messages() { } let known: BTreeMap = [(5u32, 1u16), (6, 2)].into_iter().collect(); - let ctx = (schema.clone(), known.clone()); let hero_fields = vec![ FieldValue::Bool(true), FieldValue::U(7), @@ -399,20 +445,39 @@ fn messages() { FieldValue::Str("Ana".into()), FieldValue::Bytes(vec![0xde, 0xad]), ]; + let welcome = Welcome { + protocol_version: 1, + session_id: "s1".into(), + zone_id: "z1".into(), + role: Role::Player, + tick_rate: 30, + patch_rate: 20, + server_tick: 42, + server_time: 1000.25, + reconnect_token: "rt".into(), + content_version: "c1".into(), + schema: schema.clone(), + features: 0, + origin: None, + }; let server = vec![ + ("welcome", ServerMessage::Welcome(welcome.clone())), ( - "welcome", + "welcome_v2", ServerMessage::Welcome(Welcome { - session_id: "s1".into(), - zone_id: "z1".into(), - role: Role::Player, - tick_rate: 30, - patch_rate: 20, - server_tick: 42, - server_time: 1000.25, - reconnect_token: "rt".into(), - content_version: "c1".into(), - schema: schema.clone(), + protocol_version: 2, + role: Role::Observer, + features: feature::FOCUS, + origin: Some(Origin::new(-48.0, 1e6 + 0.5)), + ..welcome.clone() + }), + ), + ( + "welcome_v2_unplaced", + ServerMessage::Welcome(Welcome { + protocol_version: 2, + features: feature::FOCUS, + ..welcome }), ), ( @@ -527,6 +592,7 @@ fn messages() { ), ]; for (name, msg) in server { + let ctx = Kinds(&schema, &known, version_of(&msg)); let bytes = msg.encode(&ctx).unwrap(); let decoded = ServerMessage::decode(&bytes, &ctx).unwrap(); // Quantized values decode to their canonical form; compare JSON of @@ -559,8 +625,54 @@ fn messages() { m["decoded"] ); } else { - let msg = ServerMessage::decode(&bytes, &ctx).unwrap(); + // Welcomes decode in the version the client offered. + let version = m["decoded"]["protocol_version"] + .as_u64() + .map_or(PROTOCOL_VERSION, |v| v as u32); + let msg = ServerMessage::decode(&bytes, &Kinds(&schema, &known, version)).unwrap(); assert_eq!(server_json(&msg), m["decoded"]); } } } + +#[test] +fn welcome_decodes_only_in_the_version_offered() { + use distributed_game::protocol::DecodeError; + let fixture = |name: &str| { + let text = std::fs::read_to_string(fixtures_dir().join("messages.json")).unwrap(); + let parsed: Value = serde_json::from_str(&text).unwrap(); + let hex = parsed["messages"] + .as_array() + .unwrap() + .iter() + .find(|m| m["name"] == name) + .unwrap()["hex"] + .as_str() + .unwrap() + .to_string(); + unhex(&hex) + }; + let schema = fixture_schema(); + let known = BTreeMap::new(); + // v2 bytes are v1 bytes plus the v2 tail: a decoder must know which it + // asked for (trailing bytes stay an error). + assert!(matches!( + ServerMessage::decode(&fixture("welcome_v2"), &Kinds(&schema, &known, 1)), + Err(DecodeError::TrailingBytes(_)) + )); + assert_eq!( + ServerMessage::decode(&fixture("welcome"), &Kinds(&schema, &known, 2)), + Err(DecodeError::UnexpectedEof) + ); + let mut v1 = distributed_game::ClientMirror::speaking(1); + let Ok(ServerMessage::Welcome(w)) = v1.apply(&fixture("welcome")) else { + panic!("v1 welcome") + }; + assert_eq!((w.protocol_version, w.features, w.origin), (1, 0, None)); + let mut v2 = distributed_game::ClientMirror::new(); + let Ok(ServerMessage::Welcome(w)) = v2.apply(&fixture("welcome_v2")) else { + panic!("v2 welcome") + }; + assert_eq!(w.features, feature::FOCUS); + assert_eq!(w.origin, Some(Origin::new(-48.0, 1e6 + 0.5))); +} diff --git a/distributed_game/tests/net_e2e/main.rs b/distributed_game/tests/net_e2e/main.rs index 57c763aeb..114b57633 100644 --- a/distributed_game/tests/net_e2e/main.rs +++ b/distributed_game/tests/net_e2e/main.rs @@ -948,3 +948,64 @@ async fn factory_failures_answer_joins_with_typed_errors() { .await; stop(server, handle).await; } + +/// Hello offering `version` with a fresh ticket. +async fn hello_version(ws: &mut Ws, version: u32, ticket: String) { + send( + ws, + ClientMessage::Hello(Hello { + protocol_version: version, + content_version: String::new(), + capabilities: 0, + ticket, + reconnect_token: String::new(), + near: false, + }), + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn negotiates_the_protocol_version_in_hello() { + use distributed_game::protocol::feature; + let (url, server, handle) = start(None).await; + + // v1 client: v1 Welcome (no features, no origin). + let mut ws = connect(&url, "z1").await; + hello_version(&mut ws, 1, ticket("ana", "z1", "v-1", Role::Player)).await; + let mut v1 = ClientMirror::speaking(1); + let Some(ServerMessage::Welcome(w)) = next(&mut ws, &mut v1).await else { + panic!("v1 Welcome") + }; + assert_eq!((w.protocol_version, w.features), (1, 0)); + + // v2 client: v2 Welcome with the runtime features. + let mut ws2 = connect(&url, "z1").await; + hello_version(&mut ws2, 2, ticket("bo", "z1", "v-2", Role::Observer)).await; + let Some(ServerMessage::Welcome(w)) = next(&mut ws2, &mut ClientMirror::new()).await else { + panic!("v2 Welcome") + }; + assert_eq!(w.protocol_version, 2); + assert_eq!(w.features & feature::FOCUS, feature::FOCUS); + + // A version the server does not speak is refused before the ticket is + // used (message starting `protocol`): the same ticket then joins. + let mut ws3 = connect(&url, "z1").await; + let same = ticket("cy", "z1", "v-3", Role::Player); + hello_version(&mut ws3, 3, same.clone()).await; + match next(&mut ws3, &mut ClientMirror::new()).await { + Some(ServerMessage::Error { code, message }) => { + assert_eq!(code, error::VERSION_MISMATCH); + assert!(message.starts_with("protocol"), "{message}"); + } + other => panic!("expected VERSION_MISMATCH, got {other:?}"), + } + let mut ws4 = connect(&url, "z1").await; + hello_version(&mut ws4, 2, same).await; + assert!(matches!( + next(&mut ws4, &mut ClientMirror::new()).await, + Some(ServerMessage::Welcome(_)) + )); + drop((ws, ws2, ws3, ws4)); + stop(server, handle).await; +} diff --git a/distributed_game/tests/replication/main.rs b/distributed_game/tests/replication/main.rs index fdcb2d7c7..a871f8efc 100644 --- a/distributed_game/tests/replication/main.rs +++ b/distributed_game/tests/replication/main.rs @@ -60,6 +60,7 @@ impl Client { // Seed the mirror with the schema as a Welcome would. let mut mirror = ClientMirror::new(); mirror.welcome = Some(distributed_game::protocol::Welcome { + protocol_version: distributed_game::PROTOCOL_VERSION, session_id: "s".into(), zone_id: "z".into(), role: distributed_game::Role::Player, @@ -70,6 +71,8 @@ impl Client { reconnect_token: String::new(), content_version: String::new(), schema: schema(), + features: 0, + origin: None, }); Self { replica: ClientReplica::new(), diff --git a/distributed_game/tests/zone_runtime/arena.rs b/distributed_game/tests/zone_runtime/arena.rs index a0d152dd6..163423619 100644 --- a/distributed_game/tests/zone_runtime/arena.rs +++ b/distributed_game/tests/zone_runtime/arena.rs @@ -23,6 +23,8 @@ pub struct ArenaConfig { pub spawn_every: Option, /// Allocation payload handed over by the zone factory. pub allocation: Option, + /// `Zone::origin` (Welcome v2). + pub origin: Option, } #[derive(Debug, Clone)] @@ -189,6 +191,10 @@ impl Zone for Arena { .push(format!("unobserve:{}", principal.as_deref().unwrap_or("?"))); } + fn origin(&self) -> Option { + self.cfg.origin + } + fn decode_input(&self, _player: PlayerId, payload: &[u8]) -> Option<(i8, i8)> { match payload { [dx, dy] => Some((*dx as i8, *dy as i8)), diff --git a/distributed_game/tests/zone_runtime/main.rs b/distributed_game/tests/zone_runtime/main.rs index f183163d1..52f6e079f 100644 --- a/distributed_game/tests/zone_runtime/main.rs +++ b/distributed_game/tests/zone_runtime/main.rs @@ -573,6 +573,43 @@ fn observers_receive_deltas_without_owning_a_player() { assert!(h.zone().log.contains(&"unobserve:watcher".to_string())); } +#[test] +fn welcome_is_encoded_in_the_clients_protocol_version() { + use distributed_game::protocol::feature; + use distributed_game::Origin; + let clock = ManualClock::new(); + let cfg = ArenaConfig { + origin: Some(Origin::new(120.0, -40.0)), + ..ArenaConfig::default() + }; + let mut h = host(HostConfig::default(), &clock, cfg); + let welcome = |h: &mut ZoneHost, version: Option, who: &str| { + let joined = match version { + Some(v) => h.speaking(v).join(&claims(who, "z1"), 0, ""), + None => h.join(&claims(who, "z1"), 0, ""), + } + .unwrap(); + let frame = frames_for(&h.drain_outbound(), joined.client).remove(0); + let mut mirror = ClientMirror::speaking(version.unwrap_or(2)); + match mirror.apply(&frame).unwrap() { + ServerMessage::Welcome(w) => w, + other => panic!("{other:?}"), + } + }; + let v1 = welcome(&mut h, Some(1), "ana"); + assert_eq!((v1.protocol_version, v1.features, v1.origin), (1, 0, None)); + let v2 = welcome(&mut h, None, "bo"); + assert_eq!(v2.protocol_version, 2); + assert_eq!(v2.features & feature::FOCUS, feature::FOCUS); + assert_eq!(v2.origin, Some(Origin::new(120.0, -40.0))); + // `speaking` applies to one admission: a failed one consumes it too. + assert!(h + .speaking(1) + .join(&claims("cy", "elsewhere"), 0, "") + .is_err()); + assert_eq!(welcome(&mut h, None, "cy").protocol_version, 2); +} + #[test] fn same_principal_replaces_previous_connection() { let clock = ManualClock::new(); diff --git a/js/src/game/connection.ts b/js/src/game/connection.ts index 7233cad6e..5b2e07e12 100644 --- a/js/src/game/connection.ts +++ b/js/src/game/connection.ts @@ -14,6 +14,7 @@ import { GAME_PROTOCOL_VERSION, GameClientMirror, GameErrorCode, + GameFeature, GameKickCode, type DeltaMessage, type ErrorMessage, @@ -190,6 +191,8 @@ export type GameConnectionEvents = { type Attempt = { socket: GameSocket; + /** Protocol version offered in this attempt's Hello. */ + protocol: number; mode: 'fresh' | 'resume'; welcomed: boolean; ticket: string; @@ -200,6 +203,22 @@ type Attempt = { const defaultBackoff = (attempt: number): number => Math.min(5000, 250 * 2 ** (attempt - 1)); +/** + * Hosts (connection URLs) that refused protocol 2 and answered 1: later + * connections and reconnects offer 1 at once (for this page's lifetime). + */ +const v1Hosts = new Set(); + +/** + * Whether a handshake `Error` is a server refusing the offered protocol + * version (servers check it before the ticket, so the Hello can be resent + * with an older version): `VERSION_MISMATCH` with a message starting + * `protocol` (content mismatches start `content version`). + */ +function protocolRefused(error: ErrorMessage): boolean { + return error.code === GameErrorCode.VERSION_MISMATCH && error.message.startsWith('protocol'); +} + /** Observers: at most one `Focus` per this many ms (see `setFocus`). */ export const FOCUS_INTERVAL_MS = 200; /** Observers: a `Focus` is sent only after a move of this many metres. */ @@ -298,6 +317,16 @@ export class GameConnection { return this.mirror.welcome?.role; } + /** Protocol version the server answered in (last Welcome). */ + get protocolVersion(): number | undefined { + return this.mirror.welcome?.protocolVersion; + } + + /** Whether the server announced a runtime feature (`GameFeature`) in its last Welcome. */ + hasFeature(feature: number): boolean { + return ((this.mirror.welcome?.features ?? 0) & feature) !== 0; + } + /** Reconnect token from the latest Welcome (rotated on every Welcome). */ get reconnectToken(): string { return this.#reconnectToken; @@ -457,7 +486,9 @@ export class GameConnection { * the zone's interest management follows it). Call it as often as you * like (e.g. every frame): it is sent at most every * {@link FOCUS_INTERVAL_MS} and only after a move of - * {@link FOCUS_MIN_MOVE} or more, and again after every (re)connect. + * {@link FOCUS_MIN_MOVE} or more, and again after every (re)connect; + * never to a server without the `FOCUS` feature (protocol 1). For a + * player's position, `GameClient.setPosition` does this per zone. */ setFocus(x: number, y: number): void { if (!Number.isFinite(x) || !Number.isFinite(y)) return; @@ -468,6 +499,7 @@ export class GameConnection { #sendFocus(): void { const focus = this.#focus; if (focus === undefined || this.#state !== 'open' || this.mirror.welcome?.role !== 'observer') return; + if (!this.hasFeature(GameFeature.FOCUS)) return; const now = this.#scheduler.now(); const last = this.#focusSent; if (last !== undefined && (now - last.at < FOCUS_INTERVAL_MS || Math.hypot(focus.x - last.x, focus.y - last.y) < FOCUS_MIN_MOVE)) return; @@ -544,7 +576,8 @@ export class GameConnection { this.#attemptFailed(error); return; } - const attempt: Attempt = { socket, mode, welcomed: false, ticket, token }; + const protocol = v1Hosts.has(this.#url) ? 1 : GAME_PROTOCOL_VERSION; + const attempt: Attempt = { socket, protocol, mode, welcomed: false, ticket, token }; this.#attempt = attempt; try { socket.binaryType = 'arraybuffer'; @@ -553,9 +586,10 @@ export class GameConnection { } socket.onopen = () => { if (this.#attempt !== attempt) return; + this.mirror.protocolVersion = protocol; this.#send({ type: 'Hello', - protocolVersion: GAME_PROTOCOL_VERSION, + protocolVersion: protocol, contentVersion: this.#contentVersion, capabilities: this.#options.capabilities ?? 0, ticket, @@ -648,6 +682,15 @@ export class GameConnection { this.#emitter.emit('redirect', message); break; case 'Error': + if (!attempt.welcomed && attempt.protocol > 1 && protocolRefused(message)) { + // A server that only speaks 1: the same Hello again with + // 1 (the ticket was not used), once; remembered per host. + v1Hosts.add(this.#url); + this.#attempt = undefined; + attempt.socket.close(1000, 'protocol fallback'); + void this.#open(attempt.mode, { ticket: attempt.ticket, token: attempt.token }); + break; + } this.#onError(attempt, message); if (!attempt.welcomed && this.#attempt === attempt) { // A refused Hello ends the attempt: nothing else follows it. diff --git a/js/src/game/index.ts b/js/src/game/index.ts index 1aaa64566..ae509d8fb 100644 --- a/js/src/game/index.ts +++ b/js/src/game/index.ts @@ -51,6 +51,7 @@ export { GAME_PROTOCOL_VERSION, GameClientMirror, GameErrorCode, + GameFeature, GameKickCode, GameMessageTag, decodeClientMessage, @@ -78,6 +79,7 @@ export { type PingMessage, type PongMessage, type RedirectMessage, + type WorldPoint, type ResumeMessage, type WelcomeMessage } from './messages.js'; diff --git a/js/src/game/messages.ts b/js/src/game/messages.ts index f7da0ab0e..18ff2dd71 100644 --- a/js/src/game/messages.ts +++ b/js/src/game/messages.ts @@ -12,8 +12,20 @@ import { } from './schema.js'; import { GameDecodeError, WireReader, WireWriter } from './wire.js'; -/** Wire protocol version spoken by this client. */ -export const GAME_PROTOCOL_VERSION = 1; +/** + * Newest wire protocol version this client speaks (offered in Hello; a + * server that only speaks 1 is retried once with 1, see `GameConnection`). + */ +export const GAME_PROTOCOL_VERSION = 2; + +/** + * Runtime feature bits a server announces in Welcome v2 (`features`): the + * client sends a message a feature covers only when its bit is set. + */ +export const GameFeature = Object.freeze({ + /** The server handles `Focus` (observers steer their view). */ + FOCUS: 1 << 0 +}); /** Message type bytes (§6.1). */ export const GameMessageTag = Object.freeze({ @@ -106,8 +118,16 @@ export type GameClientMessage = HelloMessage | InputMessage | PingMessage | Leav export type GameRole = 'player' | 'observer'; +/** A point in world coordinates (or a zone's origin in them). */ +export type WorldPoint = Readonly<{ x: number; y: number }>; + export type WelcomeMessage = Readonly<{ type: 'Welcome'; + /** + * Protocol version the Welcome is encoded in: the version the client + * offered (not itself on the wire). 1: `features` 0, no `origin`. + */ + protocolVersion: number; sessionId: string; zoneId: string; role: GameRole; @@ -118,6 +138,14 @@ export type WelcomeMessage = Readonly<{ reconnectToken: string; contentVersion: string; schema: GameSchema; + /** v2: runtime features of the server (`GameFeature` bits); 0 in v1. */ + features: number; + /** + * v2: where the zone sits in the world (world = origin + zone-local), + * when it says (`Zone::origin`); `GameClient.setPosition` focuses the + * zone from it. + */ + origin?: WorldPoint; }>; export type EntityCreate = Readonly<{ id: number; @@ -163,8 +191,12 @@ export type GameServerMessage = | ErrorMessage | RedirectMessage; -/** What the Delta codec needs: the Welcome schema and the kind of each known id. */ +/** + * What the codecs need: the negotiated protocol version (Welcome; default + * the newest), the Welcome schema and the kind of each known id (Deltas). + */ export type EntityKinds = Readonly<{ + protocolVersion?: number; schema: GameSchema | undefined; kindOf(id: number): number | undefined; }>; @@ -270,6 +302,11 @@ export function encodeServerMessage( .string(message.reconnectToken) .string(message.contentVersion); encodeSchema(w, message.schema); + if ((message.protocolVersion ?? GAME_PROTOCOL_VERSION) >= 2) { + const origin = message.origin; + w.varuint(message.features ?? 0).bool(origin !== undefined); + if (origin !== undefined) w.f64(origin.x).f64(origin.y); + } break; case 'Delta': encodeDelta(w, message, kinds); @@ -317,11 +354,13 @@ export function decodeServerMessage( const zoneId = r.string(); const roleByte = r.u8(); if (roleByte > 1) throw new GameDecodeError('INVALID', 'role'); - message = { - type: 'Welcome', + const protocolVersion = kinds.protocolVersion ?? GAME_PROTOCOL_VERSION; + const welcome = { + type: 'Welcome' as const, + protocolVersion, sessionId, zoneId, - role: roleByte === 0 ? 'player' : 'observer', + role: (roleByte === 0 ? 'player' : 'observer') as GameRole, tickRate: r.varuint(), patchRate: r.varuint(), serverTick: r.varuint(), @@ -330,6 +369,13 @@ export function decodeServerMessage( contentVersion: r.string(), schema: decodeSchema(r) }; + if (protocolVersion >= 2) { + const features = r.varuint(); + const origin = r.bool() ? { x: r.f64(), y: r.f64() } : undefined; + message = origin === undefined ? { ...welcome, features } : { ...welcome, features, origin }; + } else { + message = { ...welcome, features: 0 }; + } break; } case GameMessageTag.DELTA: @@ -463,6 +509,8 @@ export type MirrorEntity = { kind: number; fields: FieldValue[] }; * the decoded messages it returns. */ export class GameClientMirror implements EntityKinds { + /** Protocol version offered in Hello (Welcomes decode in it). */ + protocolVersion = GAME_PROTOCOL_VERSION; welcome: WelcomeMessage | undefined; readonly entities = new Map(); serverTick = 0; diff --git a/js/tests/fixtures/game-harness.mjs b/js/tests/fixtures/game-harness.mjs index aaa786aa8..50f683b80 100644 --- a/js/tests/fixtures/game-harness.mjs +++ b/js/tests/fixtures/game-harness.mjs @@ -3,6 +3,8 @@ * The fake server side speaks the real binary protocol through the SDK codec. */ import { + GAME_PROTOCOL_VERSION, + GameFeature, decodeClientMessage, defineGameSchema, encodeServerMessage @@ -197,10 +199,12 @@ export class FakeNetwork { } } -/** A Welcome for the arena schema. */ +/** A Welcome for the arena schema (protocol 2, every runtime feature). */ export function welcome(overrides = {}) { return { type: 'Welcome', + protocolVersion: GAME_PROTOCOL_VERSION, + features: GameFeature.FOCUS, sessionId: 's1', zoneId: 'z1', role: 'player', diff --git a/js/tests/game-codec.test.mjs b/js/tests/game-codec.test.mjs index 5fb7a92de..0e332363a 100644 --- a/js/tests/game-codec.test.mjs +++ b/js/tests/game-codec.test.mjs @@ -5,6 +5,7 @@ import test from 'node:test'; import { GameClientMirror, GameDecodeError, + GameFeature, WireReader, WireWriter, decodeClientMessage, @@ -163,7 +164,10 @@ function messageFromJson(decoded, schema) { serverTime: decoded.server_time, reconnectToken: decoded.reconnect_token, contentVersion: decoded.content_version, - schema: schemaFromJson(decoded.schema) + schema: schemaFromJson(decoded.schema), + protocolVersion: decoded.protocol_version, + features: decoded.features ?? 0, + ...(decoded.origin ? { origin: decoded.origin } : {}) }; case 'Delta': return { @@ -250,7 +254,11 @@ function messageToJson(message, kinds) { session_id: message.sessionId, tick_rate: message.tickRate, type: 'Welcome', - zone_id: message.zoneId + zone_id: message.zoneId, + protocol_version: message.protocolVersion, + ...(message.protocolVersion >= 2 + ? { features: message.features, origin: message.origin === undefined ? null : message.origin } + : {}) }; case 'Delta': { const created = new Map(message.creates.map((c) => [c.id, c.kind])); @@ -387,11 +395,15 @@ function allMessages() { test('every message fixture encodes and decodes byte-exactly', () => { const messages = allMessages(); const names = messages.map((m) => m.name); - for (const required of ['hello', 'hello_reconnect', 'input', 'ping', 'leave', 'welcome', 'delta', 'delta_empty', 'event', 'pong', 'freeze', 'resume', 'handoff', 'kick', 'error', 'kick_reload', 'redirect', 'hello_near', 'near', 'passive']) { + for (const required of ['hello', 'hello_v2', 'hello_reconnect', 'input', 'ping', 'leave', 'welcome', 'welcome_v2', 'welcome_v2_unplaced', 'delta', 'delta_empty', 'event', 'pong', 'freeze', 'resume', 'handoff', 'kick', 'error', 'kick_reload', 'redirect', 'hello_near', 'near', 'passive', 'focus']) { assert.ok(names.includes(required), `fixture ${required}`); } for (const entry of messages) { - const kinds = entry.schema === undefined ? undefined : kindsFor(entry); + // Welcomes decode in the version the client offered. + const kinds = + entry.schema === undefined + ? { schema: undefined, kindOf: () => undefined, protocolVersion: entry.decoded.protocol_version } + : kindsFor(entry); const message = messageFromJson(entry.decoded); const bytes = entry.direction === 'client_to_server' @@ -416,6 +428,7 @@ test('client mirror applies the golden delta like Rust ClientMirror', () => { const welcome = messages.find((m) => m.name === 'welcome'); const delta = messages.find((m) => m.name === 'delta'); const mirror = new GameClientMirror(); + mirror.protocolVersion = welcome.decoded.protocol_version; const hello = mirror.apply(fromHex(welcome.hex)); assert.equal(hello.type, 'Welcome'); // Seed the entities the delta fixture assumes are known. @@ -433,6 +446,24 @@ test('client mirror applies the golden delta like Rust ClientMirror', () => { assert.equal(mirror.entities.get(301).fields[0], 0); }); +test('a Welcome decodes only in the protocol version the client offered', () => { + const hex = (name) => fixture('messages.json').messages.find((m) => m.name === name).hex; + const v1 = { schema: undefined, kindOf: () => undefined, protocolVersion: 1 }; + const v2 = { ...v1, protocolVersion: 2 }; + assert.throws(() => decodeServerMessage(fromHex(hex('welcome_v2')), v1), /TRAILING_BYTES/); + assert.throws(() => decodeServerMessage(fromHex(hex('welcome')), v2), /UNEXPECTED_EOF/); + const old = new GameClientMirror(); + old.protocolVersion = 1; + const w1 = old.apply(fromHex(hex('welcome'))); + assert.equal(w1.protocolVersion, 1); + assert.equal(w1.features, 0); + assert.equal(w1.origin, undefined); + const w2 = new GameClientMirror().apply(fromHex(hex('welcome_v2'))); + assert.equal(w2.features & GameFeature.FOCUS, GameFeature.FOCUS); + assert.deepEqual(w2.origin, { x: -48, y: 1_000_000.5 }); + assert.equal(new GameClientMirror().apply(fromHex(hex('welcome_v2_unplaced'))).origin, undefined); +}); + test('decoders reject malformed input like the Rust reference', () => { const reject = (hex, code, kinds) => assert.throws( diff --git a/js/tests/game-connection.test.mjs b/js/tests/game-connection.test.mjs index 1ef2e910e..04df3e8c8 100644 --- a/js/tests/game-connection.test.mjs +++ b/js/tests/game-connection.test.mjs @@ -7,6 +7,7 @@ import { GameClient, GameConnection, GameErrorCode, + GameFeature, GameKickCode } from '@hops-ops/distributed/game'; @@ -165,6 +166,101 @@ test('an observer sends Focus throttled, and again after a reconnect', async () player.close(); }); +/** + * A server built before protocol 2 (what `distributed_game` answered then): + * any Hello but version 1 gets `Error(VERSION_MISMATCH)` and a close, + * before the ticket is looked at (`message`: native "protocol 2 != 1", + * celld "protocol version"). Records the tickets it admitted. + */ +function v1OnlyServer(network, { message = 'protocol version', closeCode = 1008, role = 'player' } = {}) { + const admitted = []; + network.onMessage = (conn, m) => { + if (m.type !== 'Hello') { + // An old celld zone refuses unknown messages (e.g. Focus) and closes. + if (m.type === 'Focus') { + conn.send({ type: 'Error', code: GameErrorCode.BAD_MESSAGE, message: 'bad message' }); + conn.close(1008); + } + return; + } + if (m.protocolVersion !== 1) { + conn.send({ type: 'Error', code: GameErrorCode.VERSION_MISMATCH, message }); + conn.close(closeCode); + return; + } + admitted.push(m.ticket || m.reconnectToken); + conn.send( + welcome({ zoneId: conn.zoneId, role, protocolVersion: 1, features: 0, reconnectToken: `rt-${admitted.length}`, serverTime: 10_000 }) + ); + }; + return admitted; +} + +test('offers protocol 2 and falls back once to 1 against a v1-only server', async () => { + for (const [message, closeCode] of [ + ['protocol version', 1008], + ['protocol 2 != 1', 1005] + ]) { + const network = new FakeNetwork(); + const scheduler = new FakeScheduler(); + const admitted = v1OnlyServer(network, { message, closeCode, role: 'observer' }); + const url = `ws://old-${closeCode}.test/zones/z1`; + const { conn, tickets } = connection(network, scheduler, { url, role: 'observer' }); + const errors = []; + const disconnects = []; + conn.on('error', (e) => errors.push(e)); + conn.on('disconnect', (d) => disconnects.push(d)); + const w = await conn.connect(); + assert.equal(w.protocolVersion, 1); + assert.equal(conn.protocolVersion, 1); + assert.deepEqual( + network.connections.map((c) => c.lastHello.protocolVersion), + [2, 1], + 'the same Hello again with 1' + ); + assert.equal(network.connections[1].lastHello.ticket, network.connections[0].lastHello.ticket, 'same ticket'); + assert.equal(tickets.length, 1, 'no new ticket fetched'); + assert.deepEqual(admitted, ['t:z1:1']); + assert.deepEqual(errors, [], 'the refusal is not surfaced'); + assert.deepEqual(disconnects, []); + assert.equal(conn.state, 'open'); + // A v1 server has no FOCUS feature: Focus is never sent (an old + // celld zone would close the connection on it). + conn.setFocus(3, 4); + await scheduler.advance(300); + assert.deepEqual(network.last().inbox.filter((m) => m.type === 'Focus'), []); + assert.equal(conn.hasFeature(GameFeature.FOCUS), false); + // Remembered per host: a reconnect offers 1 at once. + network.last().close(1006, 'lost'); + await scheduler.advance(5_000); + assert.equal(conn.state, 'open'); + assert.deepEqual( + network.connections.slice(2).map((c) => c.lastHello.protocolVersion), + [1] + ); + conn.close(); + } +}); + +test('a content VERSION_MISMATCH still asks for a reload, without a protocol fallback', async () => { + const network = new FakeNetwork(); + const scheduler = new FakeScheduler(); + network.onMessage = (conn, m) => { + if (m.type === 'Hello') { + conn.send({ type: 'Error', code: GameErrorCode.VERSION_MISMATCH, message: 'content version c1 != c2: reload' }); + conn.close(1008); + } + }; + const { conn } = connection(network, scheduler, { url: 'ws://new.test/zones/z1' }); + const reloads = []; + conn.on('reloadRequired', (r) => reloads.push(r.reason)); + void conn.connect().catch(() => undefined); + await scheduler.advance(10); + assert.deepEqual(reloads, ['version-mismatch']); + assert.deepEqual(network.connections.map((c) => c.lastHello.protocolVersion), [2]); + conn.close(); +}); + test('ping/pong estimates offset, rtt and jitter', async () => { const network = new FakeNetwork(); const scheduler = new FakeScheduler(); From 42def9e641f84884dcfa7819c1fd2ef251c1a60a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 14:59:06 -0500 Subject: [PATCH 15/22] feat(game): the SDK focuses placed neighbours from setPosition Each game converted the player's position into every neighbour's coordinates and called setFocus, and Mob Pit overrode on_observer_focus only to use its view radius. Zones now declare their world origin and player view radius once (Zone::origin, Zone::view_radius); the default on_observer_focus gives a focused observer that radius. The JS GameClient takes the player's position with one call (setPosition(x, y, zoneId?), local to a zone or world) and focuses every observed zone with an origin itself, also after its Welcome; setFocus stays for unplaced zones. Mob Pit declares both and drops its override; its client and the SvelteKit loop drop their coordinate math. [[zone-placement-auto-focus]] --- .../skills/distributed-game/SKILL.md | 21 +++++++-- distributed_game/PROTOCOL.md | 5 +- distributed_game/README.md | 19 +++++--- distributed_game/src/zone.rs | 22 +++++++-- distributed_game/tests/zone_runtime/arena.rs | 6 +++ distributed_game/tests/zone_runtime/main.rs | 35 ++++++++++++++ examples/game/client/src/mobpit/client.ts | 21 ++++----- examples/game/server/rooms/src/zone.rs | 26 +++++------ examples/game/server/rooms/tests/rooms.rs | 18 ++++++++ examples/game/sveltekit/src/lib/pit/loop.ts | 16 +++---- js/src/game/client.ts | 46 +++++++++++++++++-- js/src/game/watch.ts | 12 +++-- js/tests/game-connection.test.mjs | 39 ++++++++++++++++ 13 files changed, 227 insertions(+), 59 deletions(-) diff --git a/distributed_cli/skills/distributed-game/SKILL.md b/distributed_cli/skills/distributed-game/SKILL.md index 6d606ae9c..cefe0b296 100644 --- a/distributed_cli/skills/distributed-game/SKILL.md +++ b/distributed_cli/skills/distributed-game/SKILL.md @@ -77,10 +77,23 @@ Results come back on a later tick in `on_fact_settled`. Creates and removes are never held. `UpdateRates::every_patch()` turns the rates off. -- A client watching the zone next door tells it where the player is - (`watchZones(...).setFocus(at)`, zone-local coordinates). The - neighbour's `Zone::on_observer_focus` then centres that observer's view - on the player, so interest continues across the border. +- Interest across borders: place each zone in the world and give its + players' view radius, once: + + ```rust + fn origin(&self) -> Option { // world = origin + zone-local + let (x, y) = self.room.room; + Some(Origin::new(f64::from(x) * 48.0, f64::from(y) * 48.0)) + } + fn view_radius(&self) -> Option { Some(VIEW_RADIUS) } + ``` + + The client then reports the player once per frame or tick + (`client.setPosition(p.x, p.y, me.zoneId)`). The SDK focuses every + watched neighbour on the player in that neighbour's coordinates. The + default `Zone::on_observer_focus` gives that watcher the same view + radius there. No coordinate math in the game. Zones without an origin: + `watchZones(...).setFocus(at)`. - Each changed entity is encoded once per patch and shared by every client that was up to date. A dense room costs selection plus copying, not one encode per client. diff --git a/distributed_game/PROTOCOL.md b/distributed_game/PROTOCOL.md index f98bceeb7..3bb03ecdd 100644 --- a/distributed_game/PROTOCOL.md +++ b/distributed_game/PROTOCOL.md @@ -296,8 +296,9 @@ Focus := 0x07, f32 x, f32 y it): where an observer is looking, in the zone's local coordinates (e.g. its player next door, seen from this zone: world position minus the zone's Welcome `origin`). The zone decides what it means; by - default the observer's view centre moves there and keeps its radius. - It only steers what that observer is sent. Clients SHOULD throttle it (the JS + default the observer's view centre moves there, with the zone's player + view radius when it declares one (else the observer's radius). It only + steers what that observer is sent. Clients SHOULD throttle it (the JS SDK: at most every 200 ms, after a move of 0.25 or more) and send it again after each Welcome. Players' are ignored; non-finite values are ignored. diff --git a/distributed_game/README.md b/distributed_game/README.md index 066df5aad..12e62c215 100644 --- a/distributed_game/README.md +++ b/distributed_game/README.md @@ -414,13 +414,18 @@ client already has: the client when they leave it. A spatial grid makes this cheap. Mark entities that everyone must always see (players in a small room, a boss, pillars) with `.global()`. -- **Across borders.** A client watching the zone next door (an observer) - tells it where the player is in that zone's coordinates (`Focus`; JS: - `watchZones(...).setFocus(at)`, throttled for you). By default the - observer's view centre follows (`Zone::on_observer_focus`; override it - to resize or clamp, as Mob Pit does with the player's own radius), so - interest and update rates carry on past the wall instead of the - neighbour sending all it has. +- **Across borders.** A zone says where it sits in the world once + (`fn origin(&self) -> Option`, e.g. from the room in its zone + id) and how far its players see (`fn view_radius(&self) -> Option`). + The origin travels in Welcome; the game tells the JS client where the + player is with one call (`client.setPosition(x, y, zoneId)`, local to + the player's zone, or world coordinates without `zoneId`), and the SDK + focuses every zone it watches next door on the player in that zone's + coordinates (`Focus`, throttled). The default + `Zone::on_observer_focus` gives that observer the zone's player view + radius around the player, so interest and update rates carry on past + the wall instead of the neighbour sending all it has. Zones without an + origin are focused by hand (`watchZones(...).setFocus(at)`). - **Bandwidth budget.** Each client has `byte_budget` bytes per Delta (4 KiB). Creates and updates compete by priority (kind × entity × field × nearness). Anything deferred builds up priority, so it is always diff --git a/distributed_game/src/zone.rs b/distributed_game/src/zone.rs index 95190e550..26a168e1d 100644 --- a/distributed_game/src/zone.rs +++ b/distributed_game/src/zone.rs @@ -285,15 +285,18 @@ pub trait Zone: Sized + Send + 'static { /// An observer says where it is looking, in this zone's local /// coordinates (`Focus`; e.g. a player next door, so this zone's /// interest management follows that player across the border). The - /// default moves the observer's view centre there and keeps its - /// radius; override to clamp, resize or ignore it. Untrusted: it only - /// steers what the observer sees. + /// default centres the observer's view there with the zone's + /// [`view_radius`](Self::view_radius) (the player sees this zone as it + /// sees its own), else keeps the observer's radius; override to clamp, + /// resize or ignore it. Untrusted: it only steers what the observer + /// sees. fn on_observer_focus(&mut self, ctx: &mut Ctx<'_, Self>, observer: ObserverId, x: f32, y: f32) { if !(x.is_finite() && y.is_finite()) { return; } if let Some(view) = ctx.view_of(observer) { - ctx.set_view(observer, View::new(x, y, view.radius)); + let radius = self.view_radius().unwrap_or(view.radius); + ctx.set_view(observer, View::new(x, y, radius)); } } @@ -301,13 +304,22 @@ pub trait Zone: Sized + Send + 'static { /// (0, 0), for zones whose positions are local to a place in one /// continuous world (a grid of rooms, map tiles). Sent in Welcome, so a /// client that knows where its player is focuses every neighbour it - /// watches by itself. Derive it from the + /// watches by itself (JS: `GameClient.setPosition`). Derive it from the /// zone's config (e.g. the room parsed from its zone id). `None` /// (default): the zone is not placed and clients focus it only by hand. fn origin(&self) -> Option { None } + /// How far a player sees in this zone (the radius the zone gives player + /// views), when it is one radius. The default + /// [`on_observer_focus`](Self::on_observer_focus) gives a player next + /// door the same view here. `None` (default): focused observers keep + /// their radius. + fn view_radius(&self) -> Option { + None + } + /// Decode raw input bytes from the wire; `None` drops the input. fn decode_input(&self, player: PlayerId, payload: &[u8]) -> Option; diff --git a/distributed_game/tests/zone_runtime/arena.rs b/distributed_game/tests/zone_runtime/arena.rs index 163423619..fdbeda1fb 100644 --- a/distributed_game/tests/zone_runtime/arena.rs +++ b/distributed_game/tests/zone_runtime/arena.rs @@ -25,6 +25,8 @@ pub struct ArenaConfig { pub allocation: Option, /// `Zone::origin` (Welcome v2). pub origin: Option, + /// `Zone::view_radius` (the default observer focus). + pub view_radius: Option, } #[derive(Debug, Clone)] @@ -195,6 +197,10 @@ impl Zone for Arena { self.cfg.origin } + fn view_radius(&self) -> Option { + self.cfg.view_radius + } + fn decode_input(&self, _player: PlayerId, payload: &[u8]) -> Option<(i8, i8)> { match payload { [dx, dy] => Some((*dx as i8, *dy as i8)), diff --git a/distributed_game/tests/zone_runtime/main.rs b/distributed_game/tests/zone_runtime/main.rs index 52f6e079f..6fc4e1af8 100644 --- a/distributed_game/tests/zone_runtime/main.rs +++ b/distributed_game/tests/zone_runtime/main.rs @@ -573,6 +573,41 @@ fn observers_receive_deltas_without_owning_a_player() { assert!(h.zone().log.contains(&"unobserve:watcher".to_string())); } +/// An observer's mirror after a few patches (entity count). +fn observed(h: &mut ZoneHost, clock: &ManualClock, mirror: &mut ClientMirror, o: ClientId) { + for _ in 0..4 { + clock.advance(MS(100)); + h.advance(); + for frame in frames_for(&h.drain_outbound(), o) { + mirror.apply(&frame).unwrap(); + } + } +} + +#[test] +fn focus_gives_an_observer_the_zones_view_radius() { + for (view_radius, seen) in [(None, 1), (Some(5.0), 0)] { + let clock = ManualClock::new(); + let cfg = ArenaConfig { + view_radius, + ..ArenaConfig::default() + }; + let mut h = host(HostConfig::default(), &clock, cfg); + h.join(&claims("alice", "z1"), 0, "").unwrap(); + let mut obs = claims("watcher", "z1"); + obs.role = Role::Observer; + let o = h.join(&obs, 0, "").unwrap(); + let mut mirror = ClientMirror::new(); + observed(&mut h, &clock, &mut mirror, o.client); + assert_eq!(mirror.entities.len(), 1, "the whole arena before Focus"); + // Alice is 10..90 on both axes: 20 m beyond the corner is out of a + // 5 m view, inside the observer's own 1000 m one. + h.observer_focus(o.client, -20.0, -20.0); + observed(&mut h, &clock, &mut mirror, o.client); + assert_eq!(mirror.entities.len(), seen, "view radius {view_radius:?}"); + } +} + #[test] fn welcome_is_encoded_in_the_clients_protocol_version() { use distributed_game::protocol::feature; diff --git a/examples/game/client/src/mobpit/client.ts b/examples/game/client/src/mobpit/client.ts index 98f1f4414..e95e30835 100644 --- a/examples/game/client/src/mobpit/client.ts +++ b/examples/game/client/src/mobpit/client.ts @@ -18,9 +18,10 @@ * * Neighbours: the client also watches the 8 rooms around its own as an * observer (`watchZones` over `neighbourRooms`), so their mobs are drawn where they are. - * Each watcher sends the player's position in that room's coordinates - * (`setFocus`), and the room replicates only what is within the player's - * own view radius around it. On + * The client tells the SDK where the player is (`setPosition`); every room + * says where it sits in the world (Welcome origin), so each watcher is + * focused on the player in that room's coordinates, and the room + * replicates only what is within the player's own view radius around it. On * celld an observer is a passive watcher: it keeps no room awake and wakes * none (a sleeping room answers `ASLEEP`; it is tried again every few * seconds and shows nothing meanwhile). @@ -541,14 +542,12 @@ export function createMobPitClient(options: MobPitClientOptions): MobPitClient { reachBits = 0; } const meNow = me(); - // Each neighbour applies the player's own view around it (its - // interest radius carries on across the wall). - const atWorld = meNow === undefined ? undefined : worldOf(meNow); - if (atWorld !== undefined) { - watcher.setFocus((zone) => { - const r = parseRoom(zone); - return r === undefined ? undefined : { x: atWorld.x - r[0] * ROOM_SIZE, y: atWorld.y - r[1] * ROOM_SIZE }; - }); + // Each watched room (placed in the world by its Welcome origin) + // applies the player's own view around it: its interest radius + // carries on across the wall. + if (meNow !== undefined) { + const p = pit.playerFields(meNow); + typed.client.setPosition(p.x, p.y, meNow.zoneId); } let bits = 0; if (wake && at !== undefined && meNow !== undefined && meNow.zoneId === room && pures.loaded()) { diff --git a/examples/game/server/rooms/src/zone.rs b/examples/game/server/rooms/src/zone.rs index 001868b1d..8216850dd 100644 --- a/examples/game/server/rooms/src/zone.rs +++ b/examples/game/server/rooms/src/zone.rs @@ -54,7 +54,7 @@ use distributed_game::rewind::{Historic, HitQuery, RewindFrame, ShotTiming}; use distributed_game::{ schema, ConsumePolicy, Ctx, EmitError, Entities, Entity, EntityId, EntitySchema, Fact, FactOutcome, FactSettlement, GameEvent, GameInput, GameRecord, GameSurface, LeaveReason, - ObserverId, PlayerId, Project, Replicated, View, Zone, + ObserverId, Origin, PlayerId, Project, Replicated, View, Zone, }; use game_sim::arena::{self as movement, PLAYER_RADIUS}; use game_sim::mobpit::PitState as MoveState; @@ -1861,22 +1861,22 @@ impl Zone for MobPit { } /// A watcher (a player next door, a board page): the whole room, until - /// it says where it stands (`on_observer_focus`). + /// it says where it stands (`Focus`: then [`VIEW_RADIUS`] around it, as + /// in its own room, by the default `on_observer_focus`). fn on_observe(&mut self, ctx: &mut Ctx<'_, Self>, observer: ObserverId) { ctx.set_view(observer, View::new(0.0, 0.0, WATCH_RADIUS)); } - /// A player next door, at `(x, y)` in this room's coordinates, sees this - /// room as it sees its own: [`VIEW_RADIUS`] around it, so interest and - /// update rates carry on across the wall. - fn on_observer_focus(&mut self, ctx: &mut Ctx<'_, Self>, observer: ObserverId, x: f32, y: f32) { - if !(x.is_finite() && y.is_finite()) { - return; - } - // A watcher stands at most one room away. - let reach = 3.0 * movement::HALF; - let view = View::new(x.clamp(-reach, reach), y.clamp(-reach, reach), VIEW_RADIUS); - ctx.set_view(observer, view); + /// Room `(x, y)` is centred at world `(x, y) × ROOM_SIZE`: clients focus + /// the rooms they watch from the player's position by themselves. + fn origin(&self) -> Option { + let (x, y) = self.room.room; + let size = f64::from(rooms::ROOM_SIZE); + Some(Origin::new(f64::from(x) * size, f64::from(y) * size)) + } + + fn view_radius(&self) -> Option { + Some(VIEW_RADIUS) } fn decode_input(&self, _player: PlayerId, payload: &[u8]) -> Option { diff --git a/examples/game/server/rooms/tests/rooms.rs b/examples/game/server/rooms/tests/rooms.rs index 9133c60c5..324494a85 100644 --- a/examples/game/server/rooms/tests/rooms.rs +++ b/examples/game/server/rooms/tests/rooms.rs @@ -887,6 +887,23 @@ fn a_player_next_door_sees_this_room_from_where_it_stands() { let watcher = room.join(&claims, 0, "").expect("observe").client; let mut mirror = distributed_game::ClientMirror::new(); observe(&mut room, &clock, watcher, &mut mirror); + // The room says where it sits in the world (Welcome), so the client + // focuses it from the player's position by itself. + let welcome = mirror.welcome.as_ref().expect("welcome"); + assert_eq!( + welcome.origin, + Some(distributed_game::Origin::new(0.0, 0.0)) + ); + let mut east = host("pit.2.-1", &clock); + let claims_east = TicketClaims::new("ana", east.zone_id(), u64::MAX, "ana-e", Role::Observer); + let joined = east.join(&claims_east, 0, "").expect("observe").client; + let mut east_mirror = distributed_game::ClientMirror::new(); + observe(&mut east, &clock, joined, &mut east_mirror); + let size = f64::from(game_sim::mobpit::ROOM_SIZE); + assert_eq!( + east_mirror.welcome.as_ref().and_then(|w| w.origin), + Some(distributed_game::Origin::new(2.0 * size, -size)) + ); let whole = mob_positions(&mirror); assert!( whole.iter().any(|&(x, _)| x > 5.0), @@ -894,6 +911,7 @@ fn a_player_next_door_sees_this_room_from_where_it_stands() { ); // Ana stands in the west room, 5 m from the shared wall: (-25, 0) here. + // The default `on_observer_focus` gives her the room's `view_radius`. let focus = (-25.0_f32, 0.0_f32); room.observer_focus(watcher, focus.0, focus.1); observe(&mut room, &clock, watcher, &mut mirror); diff --git a/examples/game/sveltekit/src/lib/pit/loop.ts b/examples/game/sveltekit/src/lib/pit/loop.ts index 1f97832b7..0fbca500b 100644 --- a/examples/game/sveltekit/src/lib/pit/loop.ts +++ b/examples/game/sveltekit/src/lib/pit/loop.ts @@ -68,9 +68,9 @@ export function reachRooms(room: readonly [number, number], near: number): strin * are made near on `watch` — the rooms wake and stay live (the ticket * route grants `wake` only for rooms within two doorways of the player's * last room). `enabled()` false: none (passive watchers). Every tick also - * tells each watched neighbour where the player stands in its coordinates - * (`setFocus`), so it sends only what the player's own view covers. - * Returns stop. + * tells the client where the player stands (`setPosition`); the SDK focuses + * each watched neighbour on it in that room's coordinates, so it sends + * only what the player's own view covers. Returns stop. */ export function startProximityWake( client: GameClient, @@ -90,13 +90,11 @@ export function startProximityWake( bits = 0; } const me = client.render().find((e) => e.predicted && e.zoneId === zone && isPlayer(e)); - if (at !== undefined && me !== undefined) { - // Each neighbour applies the player's own view around it. + if (me !== undefined) { + // Each watched room (placed by its Welcome origin) applies the + // player's own view around it. const p = playerFields(me); - watch.setFocus((neighbour) => { - const r = parseRoom(neighbour); - return r === undefined ? undefined : { x: p.x + (at[0] - r[0]) * ROOM, y: p.y + (at[1] - r[1]) * ROOM }; - }); + client.setPosition(p.x, p.y, me.zoneId); } let next = 0; if (enabled() && at !== undefined && me !== undefined && pures.loaded()) { diff --git a/js/src/game/client.ts b/js/src/game/client.ts index 3300a9d6c..1dcc47acb 100644 --- a/js/src/game/client.ts +++ b/js/src/game/client.ts @@ -24,7 +24,8 @@ import type { HandoffMessage, KickMessage, ResumeMessage, - WelcomeMessage + WelcomeMessage, + WorldPoint } from './messages.js'; import { AdaptiveDelay, type AdaptiveDelayOptions } from './pacing.js'; import { Predictor, type Correction, type PredictorOptions } from './prediction.js'; @@ -207,6 +208,8 @@ export class GameClient { | { zoneId: string; base: RenderedEntity; fields: EntityFields; unsent: { input: TInput; dtMs: number }[] } | undefined; #closed = false; + /** The player's world position (`setPosition`): observed zones focus on it. */ + #position: WorldPoint | undefined; #contentVersion: string; #predictionEnabled: boolean; /** Adaptive delay per zone, keyed by the replica (shared across a zone's connections). */ @@ -320,10 +323,46 @@ export class GameClient { if (zone?.role === 'observer') zone.connection.setNear(near); } + /** + * Where the player is: world coordinates, or local to `zoneId` (its own + * zone, converted with that zone's Welcome `origin`). Every observed + * zone that declares its origin (`Zone::origin`) is then told where the + * player is in its own coordinates (`Focus`), now and after each + * (re)connect, so each neighbour sends what the player's view covers + * there. Call it every frame or tick; connections throttle it. Ignored + * while `zoneId`'s origin is unknown. Zones without an origin are + * focused by hand (`setFocus`). + */ + setPosition(x: number, y: number, zoneId?: string): void { + if (!Number.isFinite(x) || !Number.isFinite(y)) return; + let at: WorldPoint = { x, y }; + if (zoneId !== undefined) { + const origin = this.zone(zoneId)?.connection.welcome?.origin; + if (origin === undefined) return; + at = { x: origin.x + x, y: origin.y + y }; + } + this.#position = at; + for (const zone of this.#zones.values()) this.#focus(zone); + } + + /** The player's last world position (`setPosition`). */ + get position(): WorldPoint | undefined { + return this.#position; + } + + /** An observed, placed zone looks at the player's position. */ + #focus(zone: Zone): void { + const at = this.#position; + const origin = zone.connection.welcome?.origin; + if (zone.role !== 'observer' || at === undefined || origin === undefined) return; + zone.connection.setFocus(at.x - origin.x, at.y - origin.y); + } + /** * Tell an observed zone where this client is looking, in that zone's - * local coordinates (`GameConnection.setFocus`). No-op for zones not - * observed. + * local coordinates (`GameConnection.setFocus`), for zones that do not + * declare an origin or odd cases (prefer `setPosition`). No-op for zones + * not observed. */ setFocus(zoneId: string, x: number, y: number): void { const zone = this.#zones.get(zoneId); @@ -579,6 +618,7 @@ export class GameClient { this.#bindControlled(true); this.#emitter.emit('authority', zone.zoneId, welcome); } + this.#focus(zone); this.#emitter.emit('welcome', zone.zoneId, welcome); }), c.on('delta', (delta) => this.#onDelta(zone, delta)), diff --git a/js/src/game/watch.ts b/js/src/game/watch.ts index ffa936d48..e45adfb8c 100644 --- a/js/src/game/watch.ts +++ b/js/src/game/watch.ts @@ -21,11 +21,13 @@ * ticket while its zone stays near — at once the first time, then backing * off like a sleeping zone. * - * **Focus** (interest across borders): `setFocus(at)` tells every watched - * zone where the player is in that zone's local coordinates (`at(zone)`), - * so each neighbour applies the player's own view around it instead of - * sending everything. Call it every frame; connections throttle it and - * resend after reconnects. + * **Focus** (interest across borders): zones that declare their world + * origin (Welcome v2) are focused by the client itself from + * `client.setPosition(x, y, zoneId)`, so each neighbour applies the + * player's own view around it instead of sending everything. For zones + * without an origin, `setFocus(at)` tells every watched zone where the + * player is in that zone's local coordinates (`at(zone)`). Call either + * every frame; connections throttle it and resend after reconnects. */ import type { GameClient } from './client.js'; import { GameErrorCode } from './messages.js'; diff --git a/js/tests/game-connection.test.mjs b/js/tests/game-connection.test.mjs index 04df3e8c8..99c40d397 100644 --- a/js/tests/game-connection.test.mjs +++ b/js/tests/game-connection.test.mjs @@ -261,6 +261,45 @@ test('a content VERSION_MISMATCH still asks for a reload, without a protocol fal conn.close(); }); +test('setPosition focuses every observed zone that declares its origin', async () => { + const network = new FakeNetwork(); + const scheduler = new FakeScheduler(); + // Rooms 48 m wide: pit.. centred at (x, y) × 48; pit.9.9 is unplaced. + const origin = (zone) => { + const m = /^pit\.(-?\d+)\.(-?\d+)$/.exec(zone); + return m === null || zone === 'pit.9.9' ? undefined : { x: Number(m[1]) * 48, y: Number(m[2]) * 48 }; + }; + network.onMessage = (conn, m) => { + if (m.type !== 'Hello') return; + const role = m.ticket.endsWith(':observer') ? 'observer' : 'player'; + const at = origin(conn.zoneId); + conn.send(welcome({ zoneId: conn.zoneId, role, serverTime: 10_000, ...(at ? { origin: at } : {}) })); + }; + const client = gameClient(network, scheduler); + const focus = (zone) => network.last(zone).inbox.filter((m) => m.type === 'Focus').map(({ x, y }) => [x, y]); + await client.connect('pit.0.0'); + await client.observe('pit.1.0'); + await client.observe('pit.9.9'); + // Local to the player's own room: (5, 3) there is (-43, 3) east. + client.setPosition(5, 3, 'pit.0.0'); + assert.deepEqual(client.position, { x: 5, y: 3 }); + assert.deepEqual(focus('pit.1.0'), [[-43, 3]]); + assert.deepEqual(focus('pit.9.9'), [], 'an unplaced zone is focused by hand only'); + assert.deepEqual(focus('pit.0.0'), [], 'the own zone is not an observer'); + // A zone observed later is focused right after its Welcome. + await client.observe('pit.0.-1'); + assert.deepEqual(focus('pit.0.-1'), [[5, 51]]); + // World coordinates work too. + await scheduler.advance(250); + client.setPosition(40, -10); + assert.deepEqual(focus('pit.1.0').at(-1), [-8, -10]); + assert.deepEqual(focus('pit.0.-1').at(-1), [40, 38]); + // A zone whose origin is unknown cannot place a local position. + client.setPosition(1, 1, 'pit.7.7'); + assert.deepEqual(client.position, { x: 40, y: -10 }); + client.close(); +}); + test('ping/pong estimates offset, rtt and jitter', async () => { const network = new FakeNetwork(); const scheduler = new FakeScheduler(); From 2c6eea16f3035fa140548fd0b92a1e242b202c30 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 15:01:46 -0500 Subject: [PATCH 16/22] perf(game): no zone writes before a waking room's Welcome celld holds a room's frames until the writes issued before them are durable. A wake wrote the zone id, the status and the activity fact before the first player's Welcome (and the join wrote the status again), so under a recovery burst Welcomes waited on the bucket. Now the wake only reads; the zone id is written once per zone, the status only when it changed and in the same batch as the facts, after the Welcome. Own celld node on the Hops RustFS, 150 fresh rooms: wake joins 89 -> 150 of 150 in the ramp; after a SIGKILL p50 38 -> 16 s, p90 44 -> 31 s; 2709 -> 2157 txns shipped in recovery. [[celld-fleet-recovery-hops]] --- distributed_game/src/celld_zone/object.rs | 130 ++++++++++++++++------ 1 file changed, 95 insertions(+), 35 deletions(-) diff --git a/distributed_game/src/celld_zone/object.rs b/distributed_game/src/celld_zone/object.rs index b695ab6b3..8443a6898 100644 --- a/distributed_game/src/celld_zone/object.rs +++ b/distributed_game/src/celld_zone/object.rs @@ -606,7 +606,8 @@ struct WakeTrace { outbox_resent: usize, /// Snapshot read. snapshot_read_ms: Option, - /// Restore + the first status write. + /// Restore (and, before the join writes moved after the Welcome, the + /// wake's status write). restore_ms: Option, /// The join's own status write and start of the loop, after the wake. join_ms: Option, @@ -644,6 +645,8 @@ struct Inner { activity_seq: u64, outbox_loaded: bool, last_status: String, + /// The status last written (`None`: not read yet this activation). + stored_status: Option, node: String, stats: TickStats, last_persist_ms: f64, @@ -984,6 +987,7 @@ impl ZoneObject { activity_seq: 0, outbox_loaded: false, last_status: String::new(), + stored_status: None, node, stats: TickStats::default(), last_persist_ms: 0.0, @@ -1315,7 +1319,12 @@ impl ZoneObject { inner.sink = Some(durable); inner.activity = Some(queue); } - let _ = self.state.storage().put("zone:id", &zone_id).await; + // Written once per zone, not per activation: a wake after a crash + // writes nothing before its first player is in. + let stored_id: Option = self.state.storage().get("zone:id").await.ok().flatten(); + if stored_id.as_deref() != Some(zone_id.as_str()) { + let _ = self.state.storage().put("zone:id", &zone_id).await; + } // Resend the previous activations' unsettled facts. let stored = self .state @@ -1348,11 +1357,14 @@ impl ZoneObject { } } - /// Make sure the zone is in memory (restoring its snapshot). - async fn wake(&self) -> Result<()> { + /// Make sure the zone is in memory (restoring its snapshot); true when + /// it was not. Reads only: the caller stores the zone's status (and the + /// facts of the wake) once its frames are out, so celld's output gate + /// does not hold a joining player's Welcome for writes of the wake. + async fn wake(&self) -> Result { self.start_facts().await; if self.inner.borrow().host.is_some() { - return Ok(()); + return Ok(false); } let read_at = now_ms(); let stored: Option = self.state.storage().get(SNAPSHOT_KEY).await.ok().flatten(); @@ -1360,7 +1372,7 @@ impl ZoneObject { let host = { let inner = self.inner.borrow(); if inner.host.is_some() { - return Ok(()); + return Ok(false); } let options = inner .options @@ -1434,7 +1446,7 @@ impl ZoneObject { { let mut inner = self.inner.borrow_mut(); if inner.host.is_some() { - return Ok(()); + return Ok(false); } inner.host = Some(host); inner.restored = restored; @@ -1448,10 +1460,6 @@ impl ZoneObject { inner.state = RoomState::Active; inner.last_persist_ms = now_ms(); inner.report(ACTIVITY_ACTIVE); - } - self.store_status().await; - { - let mut inner = self.inner.borrow_mut(); let trace = &mut inner.trace; trace.snapshot_read_ms.get_or_insert(restore_at - read_at); trace.restore_ms.get_or_insert(now_ms() - restore_at); @@ -1466,7 +1474,7 @@ impl ZoneObject { self.inner.borrow_mut().world_versions.clear(); self.reconcile_world(true); self.refresh(); - Ok(()) + Ok(true) } /// Run [`CelldZone::refresh`] off the tick and deliver its message. @@ -1698,6 +1706,7 @@ impl ZoneObject { // Transfer tickets are single use across activations too. let mut principal = None; let mut wake_grant = false; + let mut woke = false; let joined = if hello.reconnect_token.is_empty() { let verified = { let inner = self.inner.borrow(); @@ -1755,10 +1764,13 @@ impl ZoneObject { .put(&nonce_key, claims.expires_at) .await; } - if let Err(e) = self.wake().await { - reject(ws, error::NOT_FOUND, &e.to_string()); - return Ok(()); - } + woke = match self.wake().await { + Ok(woke) => woke, + Err(e) => { + reject(ws, error::NOT_FOUND, &e.to_string()); + return Ok(()); + } + }; if claims.role == Role::Player { principal = Some(claims.principal.clone()); } @@ -1842,6 +1854,9 @@ impl ZoneObject { } } if observer { + if woke { + self.store_status().await; + } return Ok(()); } let joined_at = now_ms(); @@ -1859,7 +1874,12 @@ impl ZoneObject { self.after_join(principal); } } - Err(e) => reject(ws, e.code, &e.message), + Err(e) => { + reject(ws, e.code, &e.message); + if woke { + self.store_status().await; + } + } } Ok(()) } @@ -2090,28 +2110,50 @@ impl ZoneObject { /// Write the facts emitted since the last call to the outbox. async fn spool(&self) { + self.spool_with(None).await; + } + + /// [`spool`](Self::spool), with the zone's `status` in the same batch. + async fn spool_with(&self, status: Option) { let facts = self.inner.borrow_mut().emitted(); - if facts.is_empty() { + if facts.is_empty() && status.is_none() { return; } - let storage = self.state.storage(); // One batched write (the storage API takes at most 128 keys). - for chunk in facts.chunks(128) { - let batch = js_sys::Object::new(); - for fact in chunk { - let text = serde_json::to_string(&StoredFact::of(fact)).unwrap_or_default(); - let _ = js_sys::Reflect::set( - &batch, - &StoredFact::storage_key(fact.key()).into(), - &text.into(), - ); + let mut batches: Vec<(js_sys::Object, usize)> = facts + .chunks(128) + .map(|chunk| { + let batch = js_sys::Object::new(); + for fact in chunk { + let text = serde_json::to_string(&StoredFact::of(fact)).unwrap_or_default(); + let _ = js_sys::Reflect::set( + &batch, + &StoredFact::storage_key(fact.key()).into(), + &text.into(), + ); + } + (batch, chunk.len()) + }) + .collect(); + if let Some(status) = status { + if batches.last().is_none_or(|(_, keys)| *keys >= 128) { + batches.push((js_sys::Object::new(), 0)); } + if let Some((batch, keys)) = batches.last_mut() { + let _ = js_sys::Reflect::set(batch, &STATUS_KEY.into(), &status.into()); + *keys += 1; + } + } + let storage = self.state.storage(); + for (batch, _) in batches { if let Err(e) = storage.put_multiple_raw(batch).await { worker::console_error!("fact outbox write failed: {e}"); } } - self.inner.borrow_mut().spooled += facts.len() as u64; - self.arm_outbox().await; + if !facts.is_empty() { + self.inner.borrow_mut().spooled += facts.len() as u64; + self.arm_outbox().await; + } } /// [`spool`](Self::spool) from a synchronous handler. @@ -2120,12 +2162,30 @@ impl ZoneObject { wasm_bindgen_futures::spawn_local(async move { this.spool().await }); } + /// Spool the pending facts and write the zone's status with them when + /// it changed (one batch; an unchanged status writes nothing). async fn store_status(&self) { - self.spool().await; - let status = self.inner.borrow().last_status.clone(); - if !status.is_empty() { - let _ = self.state.storage().put(STATUS_KEY, status).await; - } + let (status, known) = { + let inner = self.inner.borrow(); + (inner.last_status.clone(), inner.stored_status.clone()) + }; + let stored = match known { + Some(stored) => Some(stored), + None => self + .state + .storage() + .get::(STATUS_KEY) + .await + .ok() + .flatten(), + }; + let changed = !status.is_empty() && stored.as_deref() != Some(status.as_str()); + self.inner.borrow_mut().stored_status = Some(if changed { + status.clone() + } else { + stored.unwrap_or_default() + }); + self.spool_with(changed.then_some(status)).await; } /// Set the object's alarm `in_ms` from now. From 4548a8a5dbe3699a8fdebd470d99d695cd925157 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 15:03:50 -0500 Subject: [PATCH 17/22] test(game): recovery bots rejoin as their room's own player Probes and the staying bot reuse the room's principal (a reconnecting player, not new Crawler and Presence cells per attempt), send the game's content version, log why a wake bot was lost and count rooms that lose their player again; t0 is when the socket dropped. [[celld-fleet-recovery-hops]] --- .../game/server/examples/mobpit-recovery.rs | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/examples/game/server/examples/mobpit-recovery.rs b/examples/game/server/examples/mobpit-recovery.rs index ac1d2cd78..5fe428150 100644 --- a/examples/game/server/examples/mobpit-recovery.rs +++ b/examples/game/server/examples/mobpit-recovery.rs @@ -168,7 +168,8 @@ async fn main() -> Result<(), String> { zones[zones.len() - 1], args.wake_secs ); - let (lost_tx, mut lost_rx) = tokio::sync::mpsc::unbounded_channel::<(usize, Instant)>(); + let (lost_tx, mut lost_rx) = + tokio::sync::mpsc::unbounded_channel::<(usize, Instant, Option)>(); let mut wake = Vec::new(); for (i, zone) in zones.iter().enumerate() { let delay = args.ramp_secs as f64 * i as f64 / zones.len().max(1) as f64; @@ -177,7 +178,7 @@ async fn main() -> Result<(), String> { wake.push(tokio::spawn(async move { tokio::time::sleep(Duration::from_secs_f64(delay)).await; let stats = run_bot(&config, 0, circle(i)).await; - let _ = lost.send((i, Instant::now())); + let _ = lost.send((i, Instant::now(), stats.error.clone())); stats })); } @@ -195,9 +196,13 @@ async fn main() -> Result<(), String> { tokio::time::sleep(Duration::from_secs(args.wake_secs.saturating_sub(3))).await; // A bot that already left (a room refused it) is not the kill. let mut early = 0; - while let Ok((i, _)) = lost_rx.try_recv() { + while let Ok((i, _, error)) = lost_rx.try_recv() { early += 1; - eprintln!("{} lost its bot before the kill", zones[i]); + eprintln!( + "{} lost its bot before the kill: {}", + zones[i], + error.as_deref().unwrap_or("?") + ); } if early > 0 { eprintln!("{early} rooms lost their bot before the kill"); @@ -219,11 +224,12 @@ async fn main() -> Result<(), String> { println!("READY {:.3}", unix_secs()); } let wait = Duration::from_secs(args.kill_wait_secs); - let (first, t0) = tokio::time::timeout(wait, lost_rx.recv()) + let (first, t0, _) = tokio::time::timeout(wait, lost_rx.recv()) .await .map_err(|_| "no bot lost its socket: celld was not killed")? .ok_or("bots gone")?; - let t0_unix = unix_secs(); + // When the socket was lost (the kill command may have run on). + let t0_unix = unix_secs() - t0.elapsed().as_secs_f64(); println!( "t0 {t0_unix:.3}: {} lost its socket{}", zones[first], @@ -291,8 +297,10 @@ async fn main() -> Result<(), String> { let probe = config( &args, &zone, - format!("rp{run}z{i}a{attempt}"), - i as u64 + 1, + // The room's player coming back (its own principal; a + // fresh nonce per attempt). + format!("rw{run}z{i}"), + (i as u64 + 1) * 1000 + u64::from(attempt), Duration::ZERO, ); let stats = @@ -329,7 +337,13 @@ async fn main() -> Result<(), String> { tokio::time::sleep_until(next).await; } // Back: a player stays (until the hold ends). - let config = config(&args, &zone, format!("rs{run}z{i}"), i as u64 + 7, FOREVER); + let config = config( + &args, + &zone, + format!("rw{run}z{i}"), + (i as u64 + 1) * 1000 + 999, + FOREVER, + ); let stats = run_bot(&config, 0, circle(i)).await; relost.lock().unwrap().push(( i, From e4877150325c2c546e1282a254ff5b9a3c5bde3d Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 15:44:03 -0500 Subject: [PATCH 18/22] fix(game): regenerate the Mob Pit client for hood/borough messages The watched world views (f93b24a5) added the `hood` and `borough` zone messages; the generated client was not regenerated, so CI's `game-client --check` failed in the game demos smoke job. [[zone-world-views-pushed]] --- .../client/src/generated/mobpit/events.ts | 65 ++++++++++++++ .../client/src/generated/mobpit/surface.json | 90 +++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/examples/game/client/src/generated/mobpit/events.ts b/examples/game/client/src/generated/mobpit/events.ts index 1c7321279..fe428ff51 100644 --- a/examples/game/client/src/generated/mobpit/events.ts +++ b/examples/game/client/src/generated/mobpit/events.ts @@ -348,6 +348,47 @@ export type WorldKeepCityMessage = WorldTiers; const MESSAGE_WORLD_KEEP_CITY_PAYLOAD: WireType = WORLD_TIERS_TYPE; +/** Message `hood` (id 4). */ +export interface HoodMessage { + readonly tier: number; + readonly bossDown: boolean; + readonly lastBy: string; +} + +const MESSAGE_HOOD_LAYOUT: PayloadLayout = [ + ['tier', { type: 'u32' }], + ['bossDown', { type: 'bool' }], + ['lastBy', { type: 'short_str' }], +]; + +/** Message `borough` (id 5). */ +export interface BoroughMessage { + readonly bossesDown: number; + readonly weather: number; + readonly event: string; + readonly eventUntilMs: number; + readonly hoodsDown: readonly string[]; +} + +const MESSAGE_BOROUGH_LAYOUT: PayloadLayout = [ + ['bossesDown', { type: 'u32' }], + ['weather', { type: 'u8' }], + ['event', { type: 'short_str' }], + ['eventUntilMs', { type: 'u64' }], + ['hoodsDown', { type: 'list', item: { type: 'short_str' } }], +]; + +/** Message `city` (id 6). */ +export interface CityMessage { + readonly bossesDown: number; + readonly banner: string; +} + +const MESSAGE_CITY_LAYOUT: PayloadLayout = [ + ['bossesDown', { type: 'u32' }], + ['banner', { type: 'short_str' }], +]; + /** Event type ids by name. */ export const EVENT = { hello: 1, @@ -454,6 +495,9 @@ export const MESSAGE = { loaded: 1, world: 2, worldKeepCity: 3, + hood: 4, + borough: 5, + city: 6, } as const; /** Zone inbox message payload types by name. */ @@ -461,6 +505,9 @@ export interface Messages { loaded: LoadedMessage; world: WorldMessage; worldKeepCity: WorldKeepCityMessage; + hood: HoodMessage; + borough: BoroughMessage; + city: CityMessage; } /** Codecs of every zone inbox message. */ @@ -483,6 +530,24 @@ export const MESSAGES: GameEventCodecs = { encode: (value) => encodeValue(MESSAGE_WORLD_KEEP_CITY_PAYLOAD, value), decode: (payload) => decodeValue(MESSAGE_WORLD_KEEP_CITY_PAYLOAD, payload) as WorldKeepCityMessage }, + hood: { + id: 4, + name: 'hood', + encode: (value) => encodePayload(MESSAGE_HOOD_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_HOOD_LAYOUT, payload) as unknown as HoodMessage + }, + borough: { + id: 5, + name: 'borough', + encode: (value) => encodePayload(MESSAGE_BOROUGH_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_BOROUGH_LAYOUT, payload) as unknown as BoroughMessage + }, + city: { + id: 6, + name: 'city', + encode: (value) => encodePayload(MESSAGE_CITY_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_CITY_LAYOUT, payload) as unknown as CityMessage + }, }; /** Encode a zone inbox message: `typeId` and `payload` of a `/admin/zone-message` body (base64url the payload). */ diff --git a/examples/game/client/src/generated/mobpit/surface.json b/examples/game/client/src/generated/mobpit/surface.json index b95ed46f2..b5a219d7b 100644 --- a/examples/game/client/src/generated/mobpit/surface.json +++ b/examples/game/client/src/generated/mobpit/surface.json @@ -1017,6 +1017,96 @@ "type": "record" } } + }, + { + "id": 4, + "name": "hood", + "payload": { + "fields": [ + { + "name": "tier", + "type": { + "type": "u32" + } + }, + { + "name": "boss_down", + "type": { + "type": "bool" + } + }, + { + "name": "last_by", + "type": { + "type": "short_str" + } + } + ], + "kind": "fields" + } + }, + { + "id": 5, + "name": "borough", + "payload": { + "fields": [ + { + "name": "bosses_down", + "type": { + "type": "u32" + } + }, + { + "name": "weather", + "type": { + "type": "u8" + } + }, + { + "name": "event", + "type": { + "type": "short_str" + } + }, + { + "name": "event_until_ms", + "type": { + "type": "u64" + } + }, + { + "name": "hoods_down", + "type": { + "item": { + "type": "short_str" + }, + "type": "list" + } + } + ], + "kind": "fields" + } + }, + { + "id": 6, + "name": "city", + "payload": { + "fields": [ + { + "name": "bosses_down", + "type": { + "type": "u32" + } + }, + { + "name": "banner", + "type": { + "type": "short_str" + } + } + ], + "kind": "fields" + } } ], "name": "mobpit", From 5a277a563f028ce4bec10ab7b282765cbb0a5978 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 16:27:56 -0500 Subject: [PATCH 19/22] test(game): wait for each watched view in the bosses test Hood, borough and city are separate watched cells, so a waking room's views arrive independently. The test waited for the hood tier only and then asserted the borough count and city banner at once (failed in CI and 2 of 3 local runs). Wait for each, and retry a polled tiers read that fails while celld is busy. [[zone-world-views-pushed]] --- .../game/client/tests/mobpit-bosses-headless.test.mjs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/game/client/tests/mobpit-bosses-headless.test.mjs b/examples/game/client/tests/mobpit-bosses-headless.test.mjs index baddfb933..a8508e0ce 100644 --- a/examples/game/client/tests/mobpit-bosses-headless.test.mjs +++ b/examples/game/client/tests/mobpit-bosses-headless.test.mjs @@ -83,7 +83,9 @@ test('a boss falls: its hood rises, the saga ripples it up, the hood gets toughe // Up the hierarchy: the hood's cell, then the saga into borough and city. const tiers = await eventually(async () => { - const t = await pitApi.tiers(bossZone); + // A polled read may fail while celld is busy; the next one retries. + const t = await pitApi.tiers(bossZone).catch(() => undefined); + if (t === undefined) return false; const hood = t.hoods.find((h) => h.hood === '0.0'); return hood?.bossDown && hood.tier === 1 && t.boroughs.find((b) => b.borough === '0.0')?.bossesDown === 1 && t.city?.bosses_down === 1 && t; }, 30_000, 'hood → borough → city'); @@ -94,9 +96,11 @@ test('a boss falls: its hood rises, the saga ripples it up, the hood gets toughe const visitor = player('Visitor'); await visitor.client.connect(`pit.${other[0]}.${other[1]}`); await visitor.ready; + // Hood, borough and city are separate watched cells: their views + // arrive independently, so wait for each. await until(() => visitor.world()?.hoodTier === 1 && visitor.world()?.bossDown === true, 15_000, 'the hood tier reaches its other rooms'); - assert.equal(visitor.world().boroughBossesDown, 1); - assert.ok(visitor.world().banner.length > 0, 'the city banner'); + await until(() => visitor.world()?.boroughBossesDown === 1, 15_000, 'the borough count reaches its other rooms'); + await until(() => visitor.world()?.banner.length > 0, 15_000, 'the city banner'); // The boss stays down in its room meanwhile. await sleep(5000); assert.equal(boss(), undefined, 'still down'); From 01e000fcb626ecc80684bcc4b00a1d4a72bfd395 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 17:04:30 -0500 Subject: [PATCH 20/22] fix(game): regenerate the SvelteKit Mob Pit client too Same hood/borough messages as e4877150; the SvelteKit copy of the generated client was stale, so `npm run check:games` failed in CI. [[zone-world-views-pushed]] --- .../src/lib/generated/mobpit/events.ts | 65 ++++++++++++++ .../src/lib/generated/mobpit/surface.json | 90 +++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/examples/game/sveltekit/src/lib/generated/mobpit/events.ts b/examples/game/sveltekit/src/lib/generated/mobpit/events.ts index 1c7321279..fe428ff51 100644 --- a/examples/game/sveltekit/src/lib/generated/mobpit/events.ts +++ b/examples/game/sveltekit/src/lib/generated/mobpit/events.ts @@ -348,6 +348,47 @@ export type WorldKeepCityMessage = WorldTiers; const MESSAGE_WORLD_KEEP_CITY_PAYLOAD: WireType = WORLD_TIERS_TYPE; +/** Message `hood` (id 4). */ +export interface HoodMessage { + readonly tier: number; + readonly bossDown: boolean; + readonly lastBy: string; +} + +const MESSAGE_HOOD_LAYOUT: PayloadLayout = [ + ['tier', { type: 'u32' }], + ['bossDown', { type: 'bool' }], + ['lastBy', { type: 'short_str' }], +]; + +/** Message `borough` (id 5). */ +export interface BoroughMessage { + readonly bossesDown: number; + readonly weather: number; + readonly event: string; + readonly eventUntilMs: number; + readonly hoodsDown: readonly string[]; +} + +const MESSAGE_BOROUGH_LAYOUT: PayloadLayout = [ + ['bossesDown', { type: 'u32' }], + ['weather', { type: 'u8' }], + ['event', { type: 'short_str' }], + ['eventUntilMs', { type: 'u64' }], + ['hoodsDown', { type: 'list', item: { type: 'short_str' } }], +]; + +/** Message `city` (id 6). */ +export interface CityMessage { + readonly bossesDown: number; + readonly banner: string; +} + +const MESSAGE_CITY_LAYOUT: PayloadLayout = [ + ['bossesDown', { type: 'u32' }], + ['banner', { type: 'short_str' }], +]; + /** Event type ids by name. */ export const EVENT = { hello: 1, @@ -454,6 +495,9 @@ export const MESSAGE = { loaded: 1, world: 2, worldKeepCity: 3, + hood: 4, + borough: 5, + city: 6, } as const; /** Zone inbox message payload types by name. */ @@ -461,6 +505,9 @@ export interface Messages { loaded: LoadedMessage; world: WorldMessage; worldKeepCity: WorldKeepCityMessage; + hood: HoodMessage; + borough: BoroughMessage; + city: CityMessage; } /** Codecs of every zone inbox message. */ @@ -483,6 +530,24 @@ export const MESSAGES: GameEventCodecs = { encode: (value) => encodeValue(MESSAGE_WORLD_KEEP_CITY_PAYLOAD, value), decode: (payload) => decodeValue(MESSAGE_WORLD_KEEP_CITY_PAYLOAD, payload) as WorldKeepCityMessage }, + hood: { + id: 4, + name: 'hood', + encode: (value) => encodePayload(MESSAGE_HOOD_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_HOOD_LAYOUT, payload) as unknown as HoodMessage + }, + borough: { + id: 5, + name: 'borough', + encode: (value) => encodePayload(MESSAGE_BOROUGH_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_BOROUGH_LAYOUT, payload) as unknown as BoroughMessage + }, + city: { + id: 6, + name: 'city', + encode: (value) => encodePayload(MESSAGE_CITY_LAYOUT, value), + decode: (payload) => decodePayload(MESSAGE_CITY_LAYOUT, payload) as unknown as CityMessage + }, }; /** Encode a zone inbox message: `typeId` and `payload` of a `/admin/zone-message` body (base64url the payload). */ diff --git a/examples/game/sveltekit/src/lib/generated/mobpit/surface.json b/examples/game/sveltekit/src/lib/generated/mobpit/surface.json index b95ed46f2..b5a219d7b 100644 --- a/examples/game/sveltekit/src/lib/generated/mobpit/surface.json +++ b/examples/game/sveltekit/src/lib/generated/mobpit/surface.json @@ -1017,6 +1017,96 @@ "type": "record" } } + }, + { + "id": 4, + "name": "hood", + "payload": { + "fields": [ + { + "name": "tier", + "type": { + "type": "u32" + } + }, + { + "name": "boss_down", + "type": { + "type": "bool" + } + }, + { + "name": "last_by", + "type": { + "type": "short_str" + } + } + ], + "kind": "fields" + } + }, + { + "id": 5, + "name": "borough", + "payload": { + "fields": [ + { + "name": "bosses_down", + "type": { + "type": "u32" + } + }, + { + "name": "weather", + "type": { + "type": "u8" + } + }, + { + "name": "event", + "type": { + "type": "short_str" + } + }, + { + "name": "event_until_ms", + "type": { + "type": "u64" + } + }, + { + "name": "hoods_down", + "type": { + "item": { + "type": "short_str" + }, + "type": "list" + } + } + ], + "kind": "fields" + } + }, + { + "id": 6, + "name": "city", + "payload": { + "fields": [ + { + "name": "bosses_down", + "type": { + "type": "u32" + } + }, + { + "name": "banner", + "type": { + "type": "short_str" + } + } + ], + "kind": "fields" + } } ], "name": "mobpit", From eccd9dbddd164ee59361d1fc4f6720766d947062 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 17:33:45 -0500 Subject: [PATCH 21/22] test(game): allow f32 rounding in the doorway jump bound One tick of walking is exactly the 0.25 m bound (5 m/s over a frame of up to 50 ms); crossing into the next room's coordinates adds f32 rounding, so CI saw 0.251 m (also on #291). Bound 0.26 m: a real doorway jump is a whole room or a visible snap. --- examples/game/sveltekit/tests/pit.test.mjs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/game/sveltekit/tests/pit.test.mjs b/examples/game/sveltekit/tests/pit.test.mjs index 75926082c..c2efc7f65 100644 --- a/examples/game/sveltekit/tests/pit.test.mjs +++ b/examples/game/sveltekit/tests/pit.test.mjs @@ -88,8 +88,14 @@ function start(command, args, options, ready) { }; } -/** Largest frame-to-frame move (m) of the own world position / view, ≈ one tick of walking. */ -const MAX_JUMP = 0.25; +/** + * Largest frame-to-frame move (m) of the own world position / view: one + * tick of walking (5 m/s × 50 ms = 0.25 m, frames are measured up to 50 ms + * apart) plus f32 rounding where the position crosses into the next room's + * coordinates (0.251 seen in CI). A real doorway jump is a room (40 m) or a + * visible snap. + */ +const MAX_JUMP = 0.26; /** * Walk `page`'s player through the doorway on `side`; resolves with the new From 45c922a3a94bd413b4f52c04eddca4c17298b2f9 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 24 Oct 2026 18:07:20 -0500 Subject: [PATCH 22/22] fix(game): retry a world push the zone could not take MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cell pushed each view once and logged only transport errors; a busy zone's 5xx/429 went unnoticed, so it caught up only at its next reconcile (45 ± 15 s). That is too late for a content rollout's freeze (the content headless test timed out waiting for both rooms frozen in CI). Retry after 0.25, 1 and 3 s on transport errors, 5xx and 429; a zone answers every parsed push with 204, so other 4xx are not retried. [[zone-world-views-pushed]] --- distributed_game/src/celld_zone/cells.rs | 63 +++++++++++++++--------- distributed_game/src/world/watch.rs | 20 ++++++++ 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/distributed_game/src/celld_zone/cells.rs b/distributed_game/src/celld_zone/cells.rs index 0ab239373..70a182280 100644 --- a/distributed_game/src/celld_zone/cells.rs +++ b/distributed_game/src/celld_zone/cells.rs @@ -12,13 +12,13 @@ use distributed::cell_host::{ use distributed::microsvc::Session; use distributed::Aggregate; use serde_json::{json, Value}; -use worker::{Env, Headers, Method, Request, RequestInit, Response, Result, State}; +use worker::{Delay, Env, Headers, Method, Request, RequestInit, Response, Result, State}; use crate::cell::{fact_session_from_headers, run_fact_batch}; use crate::world::content_rollout::{mount_content_rollout, ContentRollout}; use crate::world::watch::{ - parse_watcher_address, topic, watch_query, WatchUpdate, Watchers, PUSH_CONCURRENCY, PUSH_ROUTE, - STATS_ROUTE, + parse_watcher_address, push_retryable, topic, watch_query, WatchUpdate, Watchers, + PUSH_CONCURRENCY, PUSH_RETRY_MS, PUSH_ROUTE, STATS_ROUTE, }; use crate::world::zone_activity::{mount_zone_activity, ZoneActivity}; @@ -351,34 +351,49 @@ pub(crate) fn push_all(env: Env, watchers: Vec, update: WatchUpdate) { }); } +/// One push, retried after [`PUSH_RETRY_MS`] while it fails in transport or +/// the zone is too busy to take it ([`push_retryable`]). async fn push(env: &Env, watcher: &str, secret: &str, body: &str) { let Some((binding, name)) = parse_watcher_address(watcher) else { return; }; - let result: Result<()> = async { - let stub = env - .durable_object(binding)? - .id_from_name(name)? - .get_stub()?; - let headers = Headers::new(); - headers.set(CELL_INTERNAL_SECRET_HEADER, secret)?; - headers.set(super::ZONE_HEADER, name)?; - headers.set("content-type", "application/json")?; - let mut init = RequestInit::new(); - init.with_method(Method::Post) - .with_headers(headers) - .with_body(Some(body.into())); - let url = format!("https://zone/{name}/{PUSH_ROUTE}"); - stub.fetch_with_request(Request::new_with_init(&url, &init)?) - .await?; - Ok(()) - } - .await; - if let Err(error) = result { - worker::console_warn!("world push to {watcher}: {error}"); + let mut waits = PUSH_RETRY_MS.iter(); + loop { + let error = match push_once(env, binding, name, secret, body).await { + Ok(status) if !push_retryable(status) => return, + Ok(status) => format!("HTTP {status}"), + Err(error) => error.to_string(), + }; + let Some(wait) = waits.next() else { + worker::console_warn!( + "world push to {watcher}: {error}; giving up until its reconcile" + ); + return; + }; + Delay::from(std::time::Duration::from_millis(*wait)).await; } } +async fn push_once(env: &Env, binding: &str, name: &str, secret: &str, body: &str) -> Result { + let stub = env + .durable_object(binding)? + .id_from_name(name)? + .get_stub()?; + let headers = Headers::new(); + headers.set(CELL_INTERNAL_SECRET_HEADER, secret)?; + headers.set(super::ZONE_HEADER, name)?; + headers.set("content-type", "application/json")?; + let mut init = RequestInit::new(); + init.with_method(Method::Post) + .with_headers(headers) + .with_body(Some(body.into())); + let url = format!("https://zone/{name}/{PUSH_ROUTE}"); + let response = stub + .fetch_with_request(Request::new_with_init(&url, &init)?) + .await?; + Ok(response.status_code()) +} + fn internal_secret(env: &Env) -> Option { env.secret(CELL_INTERNAL_SECRET_ENV) .map(|value| value.to_string()) diff --git a/distributed_game/src/world/watch.rs b/distributed_game/src/world/watch.rs index 4bcce7c56..64447f27d 100644 --- a/distributed_game/src/world/watch.rs +++ b/distributed_game/src/world/watch.rs @@ -41,6 +41,16 @@ pub const RECONCILE_JITTER_MS: u64 = 15_000; pub const POLL_MS: u64 = 1_000; /// Watchers pushed to at once by one cell. pub const PUSH_CONCURRENCY: usize = 16; +/// Waits (ms) before each retry of a push that failed in transport or was +/// refused for load (5xx, 429): a busy zone still gets a rollout's freeze +/// within seconds instead of at its next reconcile. +pub const PUSH_RETRY_MS: [u64; 3] = [250, 1_000, 3_000]; + +/// Whether a push answered with `status` is worth retrying. A zone takes +/// every push it can parse with 204, asleep or not; other 4xx stay wrong. +pub fn push_retryable(status: u16) -> bool { + status == 429 || status >= 500 +} /// Query parameter naming the watcher (`/`). pub const WATCH_PARAM: &str = "watch"; /// Query parameter with the lease (ms). @@ -695,4 +705,14 @@ mod tests { assert_eq!(view_version(&json!({"version": 7})), 7); assert_eq!(view_version(&json!({})), 0); } + + #[test] + fn a_push_is_retried_only_when_the_zone_could_not_take_it() { + for status in [429, 500, 502, 503] { + assert!(push_retryable(status), "{status}"); + } + for status in [200, 204, 400, 401, 404] { + assert!(!push_retryable(status), "{status}"); + } + } }