From 7e99105ac511b4ffb335e560281980385fb5eede Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Fri, 4 Sep 2026 01:30:28 +0800 Subject: [PATCH 1/7] feat(wasm): JSON text export of deep values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add getDeepValueJson(): string on LoroDoc and every container class — serde_json serialization of the deep value in one WASM call, same content as JSON.stringify(x.toJSON()) — and getDeepValueJsonWithIds(): { json, cids } where cids lists container ids in pre-order DFS of the serialized tree so a consumer can re-attach ids in a single JS walk. The (json, cids) pair is produced by converting the with-id deep value to a serde_json::Value and stripping { cid, value } nodes in one pass, so the cids order always matches the key/item order a JS consumer sees after JSON.parse, regardless of serde_json's preserve_order feature. Benchmark on a ~70k-container doc (Map 15,632 / List 9,956 / Text 44,463, 4.1 MB JSON), release build: getDeepValueWithID 137.0 ms vs getDeepValueJson()+JSON.parse 56.3 ms (2.4x). The 5x target is not reachable from the JS side: profiling shows the Rust-side deep-value walk and serialization dominate (getDeepValueJson alone is 46 ms; JSON.parse of 4.1 MB is ~5 ms), not the boundary crossing. Wasm size (dev build, with debug info): +524 KB (+0.52%). --- .changeset/wasm-deep-value-json.md | 10 + AGENTS.md | 3 + context/wasm-bulk-read.md | 82 +++ crates/loro-internal/src/handler.rs | 109 ++++ crates/loro-internal/src/handler/tree.rs | 24 + crates/loro-internal/src/loro.rs | 21 + crates/loro-internal/src/state.rs | 133 +++++ crates/loro-internal/tests/deep_value_json.rs | 147 +++++ crates/loro-wasm/AGENTS.md | 5 + .../scripts/measure-deep-value-json.cjs | 184 +++++++ crates/loro-wasm/src/counter.rs | 42 ++ crates/loro-wasm/src/lib.rs | 501 ++++++++++++++++++ crates/loro-wasm/tests/deep_value.test.ts | 232 ++++++++ package.json | 1 + 14 files changed, 1494 insertions(+) create mode 100644 .changeset/wasm-deep-value-json.md create mode 100644 context/wasm-bulk-read.md create mode 100644 crates/loro-internal/tests/deep_value_json.rs create mode 100644 crates/loro-wasm/scripts/measure-deep-value-json.cjs diff --git a/.changeset/wasm-deep-value-json.md b/.changeset/wasm-deep-value-json.md new file mode 100644 index 000000000..94703a450 --- /dev/null +++ b/.changeset/wasm-deep-value-json.md @@ -0,0 +1,10 @@ +--- +"loro-crdt": minor +--- + +Add JSON text export of deep values. `LoroDoc`, `LoroMap`, `LoroList`, `LoroMovableList`, `LoroTree`, `LoroText`, and `LoroCounter` now expose: + +- `getDeepValueJson(): string` — the same content as `JSON.stringify(x.toJSON())`, produced inside WASM in a single call, avoiding the cost of crossing the WASM/JS boundary with a large structured value. +- `getDeepValueJsonWithIds(): { json: string, cids: ContainerID[] }` — `json` parses to the same content as `getDeepValueJson()` (the deep value WITHOUT container ids) and `cids` lists the container id strings in pre-order DFS of the serialized JSON tree, so a consumer can re-attach ids in a single JS walk to reconstruct the `getDeepValueWithID()` shape. For a container, `cids[0]` is that container's own id. + +Detached containers throw a readable error instead of trapping (except `LoroCounter`, which mirrors `toJSON()` and also works detached). Note: a plain object value that has exactly the keys `cid` and `value` with `cid` being a valid container id string is indistinguishable from a container node in the `cids` format. Tree node meta maps are plain deep values, so meta container ids do not appear in `cids`. diff --git a/AGENTS.md b/AGENTS.md index 58f657493..3367b73b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,9 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. [context/wasm-error-reporting.md](context/wasm-error-reporting.md). - WASM container id wrapper identity, lazy caching, and benchmark: [context/wasm-container-id-cache.md](context/wasm-container-id-cache.md). +- WASM bulk-read APIs (per-container/range deep reads, JSON text export, cid + format, cids pre-order contract): + [context/wasm-bulk-read.md](context/wasm-bulk-read.md). - Bounded decoded-value cache in `InnerStore` (second-chance FIFO, eviction safety contract, loro-dev/loro#1092): [context/container-value-cache.md](context/container-value-cache.md). diff --git a/context/wasm-bulk-read.md b/context/wasm-bulk-read.md new file mode 100644 index 000000000..40b3ed1a0 --- /dev/null +++ b/context/wasm-bulk-read.md @@ -0,0 +1,82 @@ +# WASM Bulk Read APIs + +Verified against code 2026-09-04 + +Bulk-read APIs return whole (sub)document values in one WASM call instead of +per-key/per-index accessor round trips. + +## APIs + +- `LoroDoc.getDeepValueWithID()` (`loro-internal` `DocState::get_deep_value_with_id`): + root-name → `{ cid, value }` nodes. Child containers inside `value` are + recursively replaced by their own `{ cid, value }` nodes. +- Per-container `getDeepValueWithID()` on `LoroMap`/`LoroList`/ + `LoroMovableList`/`LoroTree`/`LoroText` (`Handler::get_deep_value_with_id` → + `DocState::get_container_deep_value_with_id`); same node shape, `cid` is the + container's own id. Detached containers return an error (JS: throw). +- `LoroList`/`LoroMovableList` `getRangeValue(start, end)` / + `getRangeDeepValueWithID(start, end)` (`DocState::get_list_range_deep_value`): + deep-read a `[start, end)` slice; bounds clamp, empty/inverted → `[]`. +- `getDeepValueJson(): string` on `LoroDoc` and all six container classes + (`DocState::get_deep_value_json`, `*Handler::get_deep_value_json`): JSON text + of the plain deep value — same content as `JSON.stringify(x.toJSON())`, + serialized in one WASM call via `serde_json::to_string(&LoroValue)`. +- `getDeepValueJsonWithIds(): { json: string, cids: ContainerID[] }` + (`DocState::get_deep_value_json_with_ids`, `strip_container_id_nodes` in + `crates/loro-internal/src/state.rs`): `json` is the deep value WITHOUT ids; + `cids` is the pre-order DFS of container ids in the serialized tree. + +## cid format + +`cid` is the bare container id string — exactly what the JS `container.id` +getter returns: `cid:root-:` for roots, `cid:@:` +for op-created containers (e.g. `cid:root-map:Map`, +`cid:92@2311024965712536503:Map`). Parse the container type from the suffix +after the last `:`. + +## cids pre-order contract + +`strip_container_id_nodes` converts the with-id `LoroValue` to a +`serde_json::Value` and walks THAT value once, collecting `cids` and building +the stripped JSON in the same pass, iterating each `serde_json::Map` in its own +iteration order. This keeps `cids` consistent with the key/item order a JS +consumer sees after `JSON.parse(json)`, regardless of serde_json's +`preserve_order` feature (this workspace does not enable it, so emitted JSON +object keys are sorted — deterministic, but different from +`getDeepValueJson()`'s direct `LoroValue` serialization order; only the parsed +content is identical between the two APIs). + +Pre-order means: a container's cid is pushed before recursing into its value. +For a container-level call, `cids[0]` is that container's own id. Re-attach +walk (see `crates/loro-wasm/tests/deep_value.test.ts`): root object → every +value is a container (consume the next cid); Map → object entries; +List/MovableList/Tree → arrays; Text → string; Counter → number. A position is +treated as a container only when the next pending cid's type matches the value +shape. + +Caveats: + +- **Tree node meta is a plain deep value** (`get_meta_value` in + `crates/loro-internal/src/state/tree_state.rs` resolves meta via + `get_container_deep_value`, not the with-id variant), so meta map container + ids never appear in `cids`, and a Tree's `value` subtree contains no + `{ cid, value }` nodes. +- **Structural ambiguity**: node detection is "object with exactly the keys + `cid` (a string that parses as `ContainerID`) and `value`". A user map that + stores such an object as plain data would be mistaken for a container node — + inherent to the format, accepted. Symmetrically, a consumer-side re-attach + walk cannot distinguish a plain string/number entry from a Text/Counter + child without schema knowledge. +- `LoroCounter` has no `getDeepValueWithID()`; its `getDeepValueJson()` / + `getDeepValueJsonWithIds()` work on detached counters too, mirroring + `toJSON()`. + +## Performance + +`crates/loro-wasm/scripts/measure-deep-value-json.cjs` (run via root +`pnpm bench-deep-value-json`) builds a ~70k-container doc (Map 15,632 / List +9,956 / Text 44,463, ~3.9 MB JSON) and compares `toJSON()`, +`getDeepValueWithID()`, `getDeepValueJson() (+ JSON.parse)`, and +`getDeepValueJsonWithIds() (+ parse + re-attach walk)`. The JSON text path is +multiple times faster than structured-cloning the with-id value across the +WASM boundary; see the script output for current numbers. diff --git a/crates/loro-internal/src/handler.rs b/crates/loro-internal/src/handler.rs index 402b3d366..92aff4eec 100644 --- a/crates/loro-internal/src/handler.rs +++ b/crates/loro-internal/src/handler.rs @@ -2698,6 +2698,29 @@ impl TextHandler { })) } + /// Get the deep value of the text (its content string) as JSON text. + /// + /// The content is identical to serializing the deep value, but the JSON + /// text is produced in one pass. + pub fn get_deep_value_json(&self) -> LoroResult { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) + } + + /// Get the deep value of the text as JSON text plus container ids. + /// + /// Returns `(json, cids)` where `json` parses to the same content + /// as `get_deep_value_json()` (object key order may differ) and `cids` + /// lists the container ids in + /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this text's + /// own id. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| { + state.get_container_deep_value_json_with_ids(inner.container_idx) + }) + } + pub fn get_cursor(&self, event_index: usize, side: Side) -> Option { self.get_cursor_internal(event_index, side, true) } @@ -3368,6 +3391,29 @@ impl ListHandler { })) } + /// Get the deep value of this list as JSON text. + /// + /// The content is identical to serializing the deep value, but the JSON + /// text is produced in one pass. + pub fn get_deep_value_json(&self) -> LoroResult { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) + } + + /// Get the deep value of this list as JSON text plus container ids. + /// + /// Returns `(json, cids)` where `json` parses to the same content + /// as `get_deep_value_json()` (object key order may differ) and `cids` + /// lists the container ids in + /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this list's + /// own id. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| { + state.get_container_deep_value_json_with_ids(inner.container_idx) + }) + } + /// Get the deep value of the elements in the range `[start, end)`. /// /// Child containers in the range are recursively resolved to `{ cid, value }` @@ -4079,6 +4125,29 @@ impl MovableListHandler { })) } + /// Get the deep value of this list as JSON text. + /// + /// The content is identical to serializing the deep value, but the JSON + /// text is produced in one pass. + pub fn get_deep_value_json(&self) -> LoroResult { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) + } + + /// Get the deep value of this list as JSON text plus container ids. + /// + /// Returns `(json, cids)` where `json` parses to the same content + /// as `get_deep_value_json()` (object key order may differ) and `cids` + /// lists the container ids in + /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this list's + /// own id. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| { + state.get_container_deep_value_json_with_ids(inner.container_idx) + }) + } + /// Get the deep value of the elements in the range `[start, end)`. /// /// Child containers in the range are recursively resolved to `{ cid, value }` @@ -4518,6 +4587,29 @@ impl MapHandler { } } + /// Get the deep value of the map as JSON text. + /// + /// The content is identical to serializing the deep value, but the JSON + /// text is produced in one pass. + pub fn get_deep_value_json(&self) -> LoroResult { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) + } + + /// Get the deep value of the map as JSON text plus container ids. + /// + /// Returns `(json, cids)` where `json` parses to the same content + /// as `get_deep_value_json()` (object key order may differ) and `cids` + /// lists the container ids in + /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this map's + /// own id. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| { + state.get_container_deep_value_json_with_ids(inner.container_idx) + }) + } + pub fn get(&self, key: &str) -> Option { match &self.inner { MaybeDetached::Detached(m) => { @@ -4906,6 +4998,23 @@ pub mod counter { pub fn clear(&self) -> LoroResult<()> { self.decrement(self.get_value().into_double().unwrap()) } + + /// Get the counter value as JSON text (a JSON number). + /// + /// Unlike the other container types this also works on a detached + /// counter, mirroring `get_value`. + pub fn get_deep_value_json(&self) -> LoroResult { + crate::state::deep_value_to_json(&self.get_value()) + } + + /// Get the counter value as JSON text plus container ids. + /// + /// Returns `(json, cids)` where `json` is the same string + /// `get_deep_value_json()` returns and `cids` contains only this + /// counter's own id (a counter has no children). + pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + Ok((self.get_deep_value_json()?, vec![self.id().to_string()])) + } } impl std::fmt::Debug for CounterHandler { diff --git a/crates/loro-internal/src/handler/tree.rs b/crates/loro-internal/src/handler/tree.rs index 52fad6a31..f77053311 100644 --- a/crates/loro-internal/src/handler/tree.rs +++ b/crates/loro-internal/src/handler/tree.rs @@ -363,6 +363,30 @@ impl TreeHandler { })) } + /// Get the deep value of the tree as JSON text. + /// + /// The content is identical to serializing the deep value, but the JSON + /// text is produced in one pass. + pub fn get_deep_value_json(&self) -> LoroResult { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) + } + + /// Get the deep value of the tree as JSON text plus container ids. + /// + /// Returns `(json, cids)` where `json` parses to the same content + /// as `get_deep_value_json()` (object key order may differ) and `cids` + /// lists the container ids in + /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this tree's + /// own id. Tree node meta maps are plain deep values, so their container + /// ids do not appear in `cids`. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + let inner = self.inner.try_attached_state()?; + inner.with_doc_state(|state| { + state.get_container_deep_value_json_with_ids(inner.container_idx) + }) + } + pub fn delete(&self, target: TreeID) -> LoroResult<()> { match &self.inner { MaybeDetached::Detached(t) => { diff --git a/crates/loro-internal/src/loro.rs b/crates/loro-internal/src/loro.rs index 6c4c532d4..23719f9ce 100644 --- a/crates/loro-internal/src/loro.rs +++ b/crates/loro-internal/src/loro.rs @@ -1609,6 +1609,27 @@ impl LoroDoc { self.state.lock().get_deep_value_with_id() } + /// JSON text of the document's deep value. + /// + /// The content is identical to serializing [`Self::get_deep_value`], but + /// the JSON text is produced in one pass. + #[inline] + pub fn get_deep_value_json(&self) -> LoroResult { + self.state.lock().get_deep_value_json() + } + + /// Returns `(json, cids)` for the document's deep value. + /// + /// `json` parses to the same content as [`Self::get_deep_value_json`] (the + /// deep value WITHOUT container ids; object key order may differ between + /// the two strings) and `cids` lists the container id strings in pre-order + /// DFS of the serialized JSON tree, so a consumer can re-attach ids in a + /// single walk. + #[inline] + pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + self.state.lock().get_deep_value_json_with_ids() + } + pub fn checkout_to_latest(&self) { let (options, _guard) = self.implicit_commit_then_stop(); if !self.is_detached() { diff --git a/crates/loro-internal/src/state.rs b/crates/loro-internal/src/state.rs index e0b04ba3f..da62db193 100644 --- a/crates/loro-internal/src/state.rs +++ b/crates/loro-internal/src/state.rs @@ -94,6 +94,96 @@ fn state_decode_error(message: impl Into>) -> LoroError { LoroError::DecodeError(message.into()) } +/// Serialize a deep value to JSON text. +pub(crate) fn deep_value_to_json(value: &LoroValue) -> LoroResult { + serde_json::to_string(value) + .map_err(|e| state_decode_error(format!("Failed to serialize deep value to JSON: {e}"))) +} + +/// Convert a with-id deep value (the node shape produced by +/// [`DocState::get_deep_value_with_id`] / [`DocState::get_container_deep_value_with_id`]) +/// into `(json, cids)`: +/// +/// - `json` is the JSON text of the same deep value WITHOUT ids — the same +/// content as the plain deep value, serialized through `serde_json::Value` +/// (object keys follow serde_json's map order). +/// - `cids` lists the container id strings in pre-order DFS of the serialized +/// JSON tree: when a container node is visited, its `cid` is pushed before +/// recursing into its `value`. +/// +/// The conversion runs in ONE pass: the with-id value is first converted to a +/// `serde_json::Value`, then that value is walked once, collecting `cids` and +/// building the stripped value while iterating each `serde_json::Map` in its +/// own iteration order. This keeps `cids` consistent with the key/item order a +/// consumer sees after parsing `json`, regardless of serde_json's +/// `preserve_order` feature. +/// +/// Node detection is structural: a JSON object is treated as a container node +/// iff it has exactly two keys, `cid` (a string that parses as a +/// [`ContainerID`]) and `value`. Because the with-id builder emits nodes with +/// exactly this shape, the walk does not need to dispatch on the container +/// type: every container node's `value` is walked recursively and any nested +/// nodes are found the same way. (Tree node objects carry their meta map as a +/// plain deep value, so they contain no nodes.) +/// +/// Known ambiguity: a user map that legitimately stores an object with exactly +/// the keys `cid` + `value` where `cid` happens to be a valid container id +/// string would be mistaken for a container node. This is inherent to the +/// format and accepted. +fn strip_container_id_nodes(with_id: LoroValue) -> LoroResult<(String, Vec)> { + let value = serde_json::to_value(&with_id) + .map_err(|e| state_decode_error(format!("Failed to serialize deep value to JSON: {e}")))?; + let mut cids = Vec::new(); + let stripped = strip_container_id_nodes_in_json(value, &mut cids); + let json = serde_json::to_string(&stripped) + .map_err(|e| state_decode_error(format!("Failed to serialize deep value to JSON: {e}")))?; + Ok((json, cids)) +} + +fn strip_container_id_nodes_in_json( + value: serde_json::Value, + cids: &mut Vec, +) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => match take_container_node(map) { + Ok((cid, inner)) => { + cids.push(cid); + strip_container_id_nodes_in_json(inner, cids) + } + Err(map) => serde_json::Value::Object( + map.into_iter() + .map(|(k, v)| (k, strip_container_id_nodes_in_json(v, cids))) + .collect(), + ), + }, + serde_json::Value::Array(arr) => serde_json::Value::Array( + arr.into_iter() + .map(|v| strip_container_id_nodes_in_json(v, cids)) + .collect(), + ), + scalar => scalar, + } +} + +/// If `map` is a container node (`{ cid, value }` with a `cid` string that +/// parses as a [`ContainerID`]), return `(cid, value)`; otherwise return the +/// map unchanged. +fn take_container_node( + map: serde_json::Map, +) -> Result<(String, serde_json::Value), serde_json::Map> { + if map.len() == 2 + && matches!(map.get("cid"), Some(serde_json::Value::String(cid)) if ContainerID::try_from(cid.as_str()).is_ok()) + && map.contains_key("value") + { + let mut map = map; + let cid = map.remove("cid").unwrap().as_str().unwrap().to_string(); + let value = map.remove("value").unwrap(); + Ok((cid, value)) + } else { + Err(map) + } +} + fn decode_peer_table(bytes: &mut &[u8], context: &str) -> LoroResult> { let peer_num = leb128::read::unsigned(bytes) .map_err(|_| state_decode_error(format!("{context}: invalid peer table length")))?; @@ -1355,6 +1445,30 @@ impl DocState { LoroValue::Map(ans.into()) } + /// JSON text of [`Self::get_deep_value`]. + /// + /// The content is identical to serializing `get_deep_value()`, but the JSON + /// text is produced in one pass so callers (e.g. the WASM bindings) can + /// avoid a structured-clone round trip. + pub fn get_deep_value_json(&mut self) -> LoroResult { + deep_value_to_json(&self.get_deep_value()) + } + + /// Returns `(json, cids)` for the document's deep value: + /// + /// - `json` parses to the same content as [`Self::get_deep_value_json`] + /// (the deep value WITHOUT container ids). Object key order may differ + /// between the two strings: this method serializes through a + /// `serde_json::Value` while `get_deep_value_json` serializes the + /// `LoroValue` directly. + /// - `cids` lists the container id strings in pre-order DFS of the + /// serialized JSON tree, so a consumer can re-attach ids in a single walk. + /// + /// See [`strip_container_id_nodes`] for the format contract. + pub fn get_deep_value_json_with_ids(&mut self) -> LoroResult<(String, Vec)> { + strip_container_id_nodes(self.get_deep_value_with_id()) + } + pub(crate) fn preferred_root_containers(&mut self) -> Vec { let flag = self.store.load_root_containers(); // Mergeable cids live in a private namespace and are logically children of a regular @@ -1556,6 +1670,25 @@ impl DocState { } } + /// JSON text of [`Self::get_container_deep_value`]. + pub(crate) fn get_container_deep_value_json( + &mut self, + container: ContainerIdx, + ) -> LoroResult { + deep_value_to_json(&self.get_container_deep_value(container)) + } + + /// Container-level variant of [`Self::get_deep_value_json_with_ids`]. + /// + /// `json` is the container's deep value (without ids) and `cids` lists the + /// container ids in pre-order DFS, so `cids[0]` is this container's own id. + pub(crate) fn get_container_deep_value_json_with_ids( + &mut self, + container: ContainerIdx, + ) -> LoroResult<(String, Vec)> { + strip_container_id_nodes(self.get_container_deep_value_with_id(container, None)) + } + pub fn get_container_deep_value(&mut self, container: ContainerIdx) -> LoroValue { let Some(value) = self.store.get_value_ephemeral(container) else { return container.get_type().default_value(); diff --git a/crates/loro-internal/tests/deep_value_json.rs b/crates/loro-internal/tests/deep_value_json.rs new file mode 100644 index 000000000..014b9d450 --- /dev/null +++ b/crates/loro-internal/tests/deep_value_json.rs @@ -0,0 +1,147 @@ +use loro_common::LoroResult; +use loro_internal::{ + handler::{ListHandler, TextHandler}, + HandlerTrait, LoroDoc, TreeParentId, +}; + +/// Build a doc whose root map contains a list containing a text, plus a tree +/// with meta and a root counter. +fn build_doc() -> LoroResult<(LoroDoc, Vec)> { + let doc = LoroDoc::new_auto_commit(); + let map = doc.get_map("map"); + map.insert("flag", true)?; + let list = map.insert_container("list", ListHandler::new_detached())?; + list.insert(0, "item")?; + let text = list.insert_container(1, TextHandler::new_detached())?; + text.insert_unicode(0, "Hello")?; + let tree = doc.get_tree("tree"); + let root = tree.create(TreeParentId::Root)?; + tree.get_meta(root)?.insert("name", "root")?; + #[cfg(feature = "counter")] + let counter = doc.get_counter("counter"); + #[cfg(feature = "counter")] + counter.increment(2.5)?; + doc.commit_then_renew(); + + // serde_json serializes maps in sorted key order (no `preserve_order` + // feature in this workspace), so the pre-order cids are deterministic: + // root keys sorted: counter < map < tree. Inside map: flag (plain) then + // list; inside list: "item" (plain) then text. + #[cfg(feature = "counter")] + let cids = vec![ + counter.id().to_string(), + map.id().to_string(), + list.id().to_string(), + text.id().to_string(), + tree.id().to_string(), + ]; + #[cfg(not(feature = "counter"))] + let cids = vec![ + map.id().to_string(), + list.id().to_string(), + text.id().to_string(), + tree.id().to_string(), + ]; + Ok((doc, cids)) +} + +#[test] +fn deep_value_json_matches_plain_deep_value_serialization() -> LoroResult<()> { + let (doc, _) = build_doc()?; + let expected = serde_json::to_string(&doc.get_deep_value()).unwrap(); + assert_eq!(doc.get_deep_value_json()?, expected); + Ok(()) +} + +#[test] +fn deep_value_json_with_ids_doc_level() -> LoroResult<()> { + let (doc, expected_cids) = build_doc()?; + let (json, cids) = doc.get_deep_value_json_with_ids()?; + + // json parses to the same content as the plain deep value JSON (object + // key order may differ between the two strings; see the API docs) + assert_eq!( + serde_json::from_str::(&json).unwrap(), + serde_json::from_str::(&doc.get_deep_value_json()?).unwrap(), + ); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + serde_json::to_value(doc.get_deep_value()).unwrap(), + ); + + // cids are in pre-order DFS of the serialized tree + assert_eq!(cids, expected_cids); + + // tree meta maps are plain deep values: the meta container id does not + // appear in cids + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + let nodes = parsed["tree"].as_array().unwrap(); + assert_eq!(nodes.len(), 1); + assert_eq!(nodes[0]["meta"], serde_json::json!({ "name": "root" })); + #[cfg(feature = "counter")] + assert_eq!(parsed["counter"], serde_json::json!(2.5)); + Ok(()) +} + +#[test] +fn deep_value_json_with_ids_per_container() -> LoroResult<()> { + let (doc, _) = build_doc()?; + + let map = doc.get_map("map"); + let (json, cids) = map.get_deep_value_json_with_ids()?; + // cids[0] is the container's own id (pre-order includes the root container) + assert_eq!(cids[0], map.id().to_string()); + assert_eq!(cids.len(), 3, "map, list, text"); + // json equals the container's plain deep value + assert_eq!( + serde_json::from_str::(&json).unwrap(), + serde_json::to_value(map.get_deep_value()).unwrap(), + ); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + serde_json::from_str::(&map.get_deep_value_json()?).unwrap(), + ); + + let text = doc.get_text("text"); + text.insert_unicode(0, "abc")?; + let (json, cids) = text.get_deep_value_json_with_ids()?; + assert_eq!(json, "\"abc\""); + assert_eq!(cids, vec![text.id().to_string()]); + + let tree = doc.get_tree("tree"); + let (json, cids) = tree.get_deep_value_json_with_ids()?; + assert_eq!(cids, vec![tree.id().to_string()]); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + serde_json::to_value(tree.get_deep_value()).unwrap(), + ); + + #[cfg(feature = "counter")] + { + let counter = doc.get_counter("counter"); + let (json, cids) = counter.get_deep_value_json_with_ids()?; + assert_eq!(json, "2.5"); + assert_eq!(cids, vec![counter.id().to_string()]); + } + Ok(()) +} + +#[test] +fn deep_value_json_empty_doc() -> LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + assert_eq!(doc.get_deep_value_json()?, "{}"); + let (json, cids) = doc.get_deep_value_json_with_ids()?; + assert_eq!(json, "{}"); + assert!(cids.is_empty()); + Ok(()) +} + +#[test] +fn deep_value_json_detached_container_errors() { + let text = TextHandler::new_detached(); + assert!(text.get_deep_value_json().is_err()); + assert!(text.get_deep_value_json_with_ids().is_err()); + let list = ListHandler::new_detached(); + assert!(list.get_deep_value_json().is_err()); + assert!(list.get_deep_value_json_with_ids().is_err()); +} diff --git a/crates/loro-wasm/AGENTS.md b/crates/loro-wasm/AGENTS.md index 425ba127f..f30ca4e3e 100644 --- a/crates/loro-wasm/AGENTS.md +++ b/crates/loro-wasm/AGENTS.md @@ -75,6 +75,11 @@ Container `id` wrapper identity, lazy-cache lifetime, and the focused benchmark are documented in [context/wasm-container-id-cache.md](../../context/wasm-container-id-cache.md). +Bulk-read APIs (per-container/range deep reads with ids, `getDeepValueJson`, +`getDeepValueJsonWithIds`, the cid format, and the cids pre-order contract) +are documented in +[context/wasm-bulk-read.md](../../context/wasm-bulk-read.md). + ## Packaging Rules - Preserve the public `loro-crdt` API names and package export paths used by diff --git a/crates/loro-wasm/scripts/measure-deep-value-json.cjs b/crates/loro-wasm/scripts/measure-deep-value-json.cjs new file mode 100644 index 000000000..7fa7dacc6 --- /dev/null +++ b/crates/loro-wasm/scripts/measure-deep-value-json.cjs @@ -0,0 +1,184 @@ +const { performance } = require("node:perf_hooks"); +const { LoroDoc, LoroList, LoroMap, LoroText } = require("../nodejs/index.js"); + +// Mirrors the distribution of a real-world document: +// Map 15,632 / List 9,956 / Text 44,463 containers (~70k in total), +// ~3.9 MB of JSON content. +const MAP_COUNT = 15_632; +const LIST_COUNT = 9_956; +const TEXT_COUNT = 44_463; +const PARENT_POOL_SIZE = 2_048; +const PARAGRAPH = + "The quick brown fox jumps over the lazy dog. Pack my box. "; + +const WARMUP_ROUNDS = 2; +const ROUNDS = 5; +let blackhole = 0; + +function buildDocument() { + const doc = new LoroDoc(); + const root = doc.getMap("root"); + const parents = [root]; + let mi = 1; + let li = 0; + let ti = 0; + let seq = 0; + const addChild = (child, isParent) => { + const parent = parents[seq % parents.length]; + seq++; + let attached; + if (parent.kind() === "Map") { + attached = parent.setContainer(`k${seq}`, child); + } else { + attached = parent.pushContainer(child); + } + if (isParent && parents.length < PARENT_POOL_SIZE) parents.push(attached); + return attached; + }; + while (mi < MAP_COUNT || li < LIST_COUNT || ti < TEXT_COUNT) { + if (mi < MAP_COUNT) { + const map = addChild(new LoroMap(), true); + map.set("id", mi); + map.set("flag", true); + map.set("name", `record-${mi}`); + mi++; + } + if (li < LIST_COUNT) { + const list = addChild(new LoroList(), true); + list.insert(0, li); + list.insert(1, `item-${li}`); + li++; + } + if (ti < TEXT_COUNT) { + const text = addChild(new LoroText(), false); + text.insert(0, `${PARAGRAPH}${ti}`); + ti++; + } + } + doc.commit(); + return doc; +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +} + +function timed(name, run) { + const samples = []; + for (let round = 0; round < WARMUP_ROUNDS + ROUNDS; round++) { + global.gc?.(); + const start = performance.now(); + run(); + const elapsed = performance.now() - start; + if (round >= WARMUP_ROUNDS) samples.push(elapsed); + } + return { name, medianMs: median(samples), samplesMs: samples }; +} + +// The same pre-order re-attach walk documented for consumers of +// getDeepValueJsonWithIds(): root object -> every value is a container; +// Map -> object entries; List/MovableList/Tree -> arrays; Text -> string; +// Counter -> number. +function makeReattach(cids) { + let i = 0; + const containerTypeOf = (cid) => cid.slice(cid.lastIndexOf(":") + 1); + const shapeMatches = (type, value) => { + switch (type) { + case "Text": + return typeof value === "string"; + case "Counter": + return typeof value === "number"; + case "Map": + return ( + typeof value === "object" && value !== null && !Array.isArray(value) + ); + case "List": + case "MovableList": + case "Tree": + return Array.isArray(value); + } + }; + const attach = (value) => { + const cid = cids[i]; + if (cid === undefined || !shapeMatches(containerTypeOf(cid), value)) { + return value; + } + return attachForced(value); + }; + const attachForced = (value) => { + const cid = cids[i++]; + return { cid, value: walkContainerValue(containerTypeOf(cid), value) }; + }; + const walkContainerValue = (type, value) => { + switch (type) { + case "Text": + case "Counter": + return value; + case "Map": { + const out = {}; + for (const k of Object.keys(value)) out[k] = attach(value[k]); + return out; + } + case "List": + case "MovableList": + return value.map(attach); + case "Tree": + return value; + } + }; + return (json) => { + const out = {}; + for (const k of Object.keys(json)) out[k] = attachForced(json[k]); + blackhole += i; + return out; + }; +} + +const doc = buildDocument(); +const stats = { + containers: MAP_COUNT + LIST_COUNT + TEXT_COUNT, + jsonBytes: Buffer.byteLength(doc.getDeepValueJson(), "utf8"), +}; + +const cases = [ + ["toJSON", () => blackhole += JSON.stringify(doc.toJSON()).length], + ["getDeepValueWithID", () => blackhole += Object.keys(doc.getDeepValueWithID()).length], + ["getDeepValueJson", () => blackhole += doc.getDeepValueJson().length], + [ + "getDeepValueJson+parse", + () => blackhole += Object.keys(JSON.parse(doc.getDeepValueJson())).length, + ], + [ + "getDeepValueJsonWithIds+parse+reattach", + () => { + const { json, cids } = doc.getDeepValueJsonWithIds(); + const out = makeReattach(cids)(JSON.parse(json)); + blackhole += Object.keys(out).length; + }, + ], +]; + +const result = cases.map(([name, run]) => timed(name, run)); +const withId = result.find((r) => r.name === "getDeepValueWithID").medianMs; +const jsonParse = result.find( + (r) => r.name === "getDeepValueJson+parse", +).medianMs; + +console.log( + JSON.stringify( + { + ...stats, + warmupRounds: WARMUP_ROUNDS, + rounds: ROUNDS, + blackhole, + result, + speedup: { + "getDeepValueJson+parse vs getDeepValueWithID": withId / jsonParse, + }, + }, + null, + 2, + ), +); +doc.free(); diff --git a/crates/loro-wasm/src/counter.rs b/crates/loro-wasm/src/counter.rs index 7920a88ca..d2b64525b 100644 --- a/crates/loro-wasm/src/counter.rs +++ b/crates/loro-wasm/src/counter.rs @@ -136,4 +136,46 @@ impl LoroCounter { .into_double() .map_err(|_| JsValue::from_str("Counter value is not a number")) } + + /// Get the counter value as JSON text (a JSON number). + /// + /// The content is identical to `JSON.stringify(counter.toJSON())`, but the + /// JSON text is produced inside WASM. Unlike the other container types, + /// this also works on a detached counter, mirroring `toJSON()`. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const counter = doc.getCounter("counter"); + /// counter.increment(1.5); + /// console.log(counter.getDeepValueJson()); // "1.5" + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] + pub fn get_deep_value_json(&self) -> JsResult { + Ok(self.handler.get_deep_value_json()?) + } + + /// Get the counter value as JSON text, plus container ids. + /// + /// Returns `{ json, cids }` where `json` is the same string + /// `getDeepValueJson()` returns and `cids` contains only this counter's + /// own id (a counter has no children). + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const counter = doc.getCounter("counter"); + /// counter.increment(1.5); + /// const { json, cids } = counter.getDeepValueJsonWithIds(); + /// // json === "1.5", cids === [counter.id] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] + pub fn get_deep_value_json_with_ids(&self) -> JsResult { + let (json, cids) = self.handler.get_deep_value_json_with_ids()?; + crate::deep_value_json_with_ids_to_js(json, cids) + } } diff --git a/crates/loro-wasm/src/lib.rs b/crates/loro-wasm/src/lib.rs index 65adddbc8..6ec8f541b 100644 --- a/crates/loro-wasm/src/lib.rs +++ b/crates/loro-wasm/src/lib.rs @@ -525,6 +525,17 @@ fn id_to_js(id: &ID) -> JsResult { Ok(obj.into()) } +pub(crate) fn deep_value_json_with_ids_to_js(json: String, cids: Vec) -> JsResult { + let obj = Object::new(); + Reflect::set(&obj, &"json".into(), &json.into())?; + let arr = Array::new(); + for cid in cids { + arr.push(&cid.into()); + } + Reflect::set(&obj, &"cids".into(), &arr)?; + Ok(obj.into()) +} + fn peer_id_to_js(peer: PeerID) -> JsStrPeerID { let v: JsValue = peer.to_string().into(); v.into() @@ -1468,6 +1479,54 @@ impl LoroDoc { self.doc.get_deep_value_with_id().into() } + /// Get the deep value of the document as JSON text. + /// + /// The content is identical to `JSON.stringify(doc.toJSON())`, but the + /// JSON text is produced inside WASM in a single call, avoiding the cost + /// of crossing the WASM/JS boundary with a large structured value. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// doc.getText("text").insert(0, "Hello"); + /// console.log(doc.getDeepValueJson()); // {"text":"Hello"} + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] + pub fn get_deep_value_json(&self) -> JsResult { + Ok(self.doc.get_deep_value_json()?) + } + + /// Get the deep value of the document as JSON text, plus container ids. + /// + /// Returns `{ json, cids }` where `json` parses to the same content as + /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key + /// order may differ between the two strings) and `cids` lists the + /// container id strings in pre-order DFS of the serialized JSON tree, so a + /// consumer can re-attach ids in a single JS walk to reconstruct the + /// `getDeepValueWithID()` shape. + /// + /// Note: a plain object value that has exactly the keys `cid` and `value` + /// with `cid` being a valid container id string is indistinguishable from + /// a container node in this format. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const text = doc.getText("text"); + /// text.insert(0, "Hello"); + /// const { json, cids } = doc.getDeepValueJsonWithIds(); + /// // json === '{"text":"Hello"}', cids === ["cid:root-text:Text"] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] + pub fn get_deep_value_json_with_ids(&self) -> JsResult { + let (json, cids) = self.doc.get_deep_value_json_with_ids()?; + deep_value_json_with_ids_to_js(json, cids) + } + /// Get the path from the root to the container #[wasm_bindgen(js_name = "getPathToContainer")] pub fn get_path_to_container(&self, id: JsContainerID) -> JsResult> { @@ -3253,6 +3312,61 @@ impl LoroText { pub fn get_deep_value_with_id(&self) -> JsResult { Ok(self.handler.get_deep_value_with_id()?.into()) } + + /// Get the deep value of the text as JSON text. + /// + /// The content is identical to `JSON.stringify(text.toJSON())`, but + /// the JSON text is produced inside WASM in a single call, avoiding the + /// cost of crossing the WASM/JS boundary with a large structured value. + /// + /// For a text container the JSON text is a JSON string of the text + /// content. + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const text = doc.getText("text"); + /// text.insert(0, "Hello"); + /// console.log(text.getDeepValueJson()); // \"Hello\" + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] + pub fn get_deep_value_json(&self) -> JsResult { + Ok(self.handler.get_deep_value_json()?) + } + + /// Get the deep value of the text as JSON text, plus container ids. + /// + /// Returns `{ json, cids }` where `json` parses to the same content as + /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key + /// order may differ between the two strings) and `cids` lists the + /// container ids in pre-order DFS of the serialized JSON tree, so + /// `cids[0]` is this container's own id. A single JS walk can re-attach + /// the ids to reconstruct the `getDeepValueWithID()` shape. + /// + /// Note: a plain object value that has exactly the keys `cid` and `value` + /// with `cid` being a valid container id string is indistinguishable from + /// a container node in this format. + /// + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const text = doc.getText("text"); + /// text.insert(0, "Hello"); + /// const { json, cids } = text.getDeepValueJsonWithIds(); + /// // json === '\"Hello\"', cids === ["cid:root-text:Text"] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] + pub fn get_deep_value_json_with_ids(&self) -> JsResult { + let (json, cids) = self.handler.get_deep_value_json_with_ids()?; + deep_value_json_with_ids_to_js(json, cids) + } } impl Default for LoroText { @@ -3542,6 +3656,59 @@ impl LoroMap { Ok(self.handler.get_deep_value_with_id()?.into()) } + /// Get the deep value of the map as JSON text. + /// + /// The content is identical to `JSON.stringify(map.toJSON())`, but + /// the JSON text is produced inside WASM in a single call, avoiding the + /// cost of crossing the WASM/JS boundary with a large structured value. + /// + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const map = doc.getMap("map"); + /// map.set("foo", "bar"); + /// console.log(map.getDeepValueJson()); // {\"foo\":\"bar\"} + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] + pub fn get_deep_value_json(&self) -> JsResult { + Ok(self.handler.get_deep_value_json()?) + } + + /// Get the deep value of the map as JSON text, plus container ids. + /// + /// Returns `{ json, cids }` where `json` parses to the same content as + /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key + /// order may differ between the two strings) and `cids` lists the + /// container ids in pre-order DFS of the serialized JSON tree, so + /// `cids[0]` is this container's own id. A single JS walk can re-attach + /// the ids to reconstruct the `getDeepValueWithID()` shape. + /// + /// Note: a plain object value that has exactly the keys `cid` and `value` + /// with `cid` being a valid container id string is indistinguishable from + /// a container node in this format. + /// + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const map = doc.getMap("map"); + /// map.set("foo", "bar"); + /// const { json, cids } = map.getDeepValueJsonWithIds(); + /// // json === '{\"foo\":\"bar\"}', cids === ["cid:root-map:Map"] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] + pub fn get_deep_value_json_with_ids(&self) -> JsResult { + let (json, cids) = self.handler.get_deep_value_json_with_ids()?; + deep_value_json_with_ids_to_js(json, cids) + } + /// Set the key with a regular child container. /// /// The inserted child receives a regular op-created container id. Use this @@ -3973,6 +4140,59 @@ impl LoroList { Ok(self.handler.get_deep_value_with_id()?.into()) } + /// Get the deep value of the list as JSON text. + /// + /// The content is identical to `JSON.stringify(list.toJSON())`, but + /// the JSON text is produced inside WASM in a single call, avoiding the + /// cost of crossing the WASM/JS boundary with a large structured value. + /// + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const list = doc.getList("list"); + /// list.insert(0, 100); + /// console.log(list.getDeepValueJson()); // [100] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] + pub fn get_deep_value_json(&self) -> JsResult { + Ok(self.handler.get_deep_value_json()?) + } + + /// Get the deep value of the list as JSON text, plus container ids. + /// + /// Returns `{ json, cids }` where `json` parses to the same content as + /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key + /// order may differ between the two strings) and `cids` lists the + /// container ids in pre-order DFS of the serialized JSON tree, so + /// `cids[0]` is this container's own id. A single JS walk can re-attach + /// the ids to reconstruct the `getDeepValueWithID()` shape. + /// + /// Note: a plain object value that has exactly the keys `cid` and `value` + /// with `cid` being a valid container id string is indistinguishable from + /// a container node in this format. + /// + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const list = doc.getList("list"); + /// list.insert(0, 100); + /// const { json, cids } = list.getDeepValueJsonWithIds(); + /// // json === '[100]', cids === ["cid:root-list:List"] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] + pub fn get_deep_value_json_with_ids(&self) -> JsResult { + let (json, cids) = self.handler.get_deep_value_json_with_ids()?; + deep_value_json_with_ids_to_js(json, cids) + } + /// Get the deep value of the elements in the range `[start, end)`, with /// container ids. /// @@ -4415,6 +4635,59 @@ impl LoroMovableList { Ok(self.handler.get_deep_value_with_id()?.into()) } + /// Get the deep value of the movableList as JSON text. + /// + /// The content is identical to `JSON.stringify(movableList.toJSON())`, but + /// the JSON text is produced inside WASM in a single call, avoiding the + /// cost of crossing the WASM/JS boundary with a large structured value. + /// + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const movableList = doc.getMovableList("list"); + /// movableList.insert(0, 100); + /// console.log(movableList.getDeepValueJson()); // [100] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] + pub fn get_deep_value_json(&self) -> JsResult { + Ok(self.handler.get_deep_value_json()?) + } + + /// Get the deep value of the movableList as JSON text, plus container ids. + /// + /// Returns `{ json, cids }` where `json` parses to the same content as + /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key + /// order may differ between the two strings) and `cids` lists the + /// container ids in pre-order DFS of the serialized JSON tree, so + /// `cids[0]` is this container's own id. A single JS walk can re-attach + /// the ids to reconstruct the `getDeepValueWithID()` shape. + /// + /// Note: a plain object value that has exactly the keys `cid` and `value` + /// with `cid` being a valid container id string is indistinguishable from + /// a container node in this format. + /// + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const movableList = doc.getMovableList("list"); + /// movableList.insert(0, 100); + /// const { json, cids } = movableList.getDeepValueJsonWithIds(); + /// // json === '[100]', cids === ["cid:root-list:MovableList"] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] + pub fn get_deep_value_json_with_ids(&self) -> JsResult { + let (json, cids) = self.handler.get_deep_value_json_with_ids()?; + deep_value_json_with_ids_to_js(json, cids) + } + /// Get the deep value of the elements in the range `[start, end)`, with /// container ids. /// @@ -5238,6 +5511,61 @@ impl LoroTree { Ok(self.handler.get_deep_value_with_id()?.into()) } + /// Get the deep value of the tree as JSON text. + /// + /// The content is identical to `JSON.stringify(tree.toJSON())`, but + /// the JSON text is produced inside WASM in a single call, avoiding the + /// cost of crossing the WASM/JS boundary with a large structured value. + /// + /// Tree node meta maps are plain deep values, so their container ids + /// do not appear in `cids`. + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const tree = doc.getTree("tree"); + /// tree.createNode(); + /// console.log(tree.getDeepValueJson()); // [{\"id\":\"...\",\"parent\":null,\"meta\":{},\"index\":0,\"fractional_index\":\"...\",\"children\":[]}] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] + pub fn get_deep_value_json(&self) -> JsResult { + Ok(self.handler.get_deep_value_json()?) + } + + /// Get the deep value of the tree as JSON text, plus container ids. + /// + /// Returns `{ json, cids }` where `json` parses to the same content as + /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key + /// order may differ between the two strings) and `cids` lists the + /// container ids in pre-order DFS of the serialized JSON tree, so + /// `cids[0]` is this container's own id. A single JS walk can re-attach + /// the ids to reconstruct the `getDeepValueWithID()` shape. + /// + /// Note: a plain object value that has exactly the keys `cid` and `value` + /// with `cid` being a valid container id string is indistinguishable from + /// a container node in this format. + /// + /// Throws if the container is detached. + /// + /// @example + /// ```ts + /// import { LoroDoc } from "loro-crdt"; + /// + /// const doc = new LoroDoc(); + /// const tree = doc.getTree("tree"); + /// tree.createNode(); + /// const { json, cids } = tree.getDeepValueJsonWithIds(); + /// // json === '[{\"id\":\"...\",\"parent\":null,\"meta\":{},\"index\":0,\"fractional_index\":\"...\",\"children\":[]}]', cids === ["cid:root-tree:Tree"] + /// ``` + #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] + pub fn get_deep_value_json_with_ids(&self) -> JsResult { + let (json, cids) = self.handler.get_deep_value_json_with_ids()?; + deep_value_json_with_ids_to_js(json, cids) + } + /// Get all tree nodes of the forest, including deleted nodes. /// /// @example @@ -6636,6 +6964,25 @@ export type ValueWithContainerID = { value: Value, } +/** + * The result of `getDeepValueJsonWithIds()`. + * + * `json` is the JSON text of the deep value WITHOUT container ids (it parses + * to the same content as `getDeepValueJson()`). `cids` lists the container id + * strings in pre-order DFS of the serialized JSON tree: the ids appear in the + * same order as the key/item order a consumer sees after `JSON.parse(json)`, + * so a single JS walk can re-attach ids and reconstruct the + * `getDeepValueWithID()` shape. + * + * Note: a plain object value that has exactly the keys `cid` and `value` with + * `cid` being a valid container id string is indistinguishable from a + * container node in this format. + */ +export type DeepValueJsonWithIds = { + json: string, + cids: ContainerID[], +} + export type IdSpan = { peer: PeerID, counter: number, @@ -7121,6 +7468,40 @@ interface LoroDoc { * You can debounce/throttle the callback before running `JSONPath(...)` to optimize heavy reads. */ subscribeJsonpath(path: string, callback: () => void): Subscription; + /** + * Get the deep value of the document as JSON text. + * + * The content is identical to `JSON.stringify(doc.toJSON())`, but the + * JSON text is produced inside WASM in a single call, avoiding the cost + * of crossing the WASM/JS boundary with a large structured value. + */ + getDeepValueJson(): string; + /** + * Get the deep value of the document as JSON text, plus container ids. + * + * `json` parses to the same content as `getDeepValueJson()` (the deep + * value WITHOUT container ids) and `cids` lists the container ids in + * pre-order DFS of the serialized JSON tree, so a single JS walk can + * re-attach the ids to reconstruct the `getDeepValueWithID()` shape. + */ + getDeepValueJsonWithIds(): DeepValueJsonWithIds; +} + +interface LoroCounter { + /** + * Get the counter value as JSON text (a JSON number). + * + * Unlike the other container types, this also works on a detached + * counter, mirroring `toJSON()`. + */ + getDeepValueJson(): string; + /** + * Get the counter value as JSON text, plus container ids. + * + * `json` is the same string `getDeepValueJson()` returns and `cids` + * contains only this counter's own id (a counter has no children). + */ + getDeepValueJsonWithIds(): DeepValueJsonWithIds; } interface UndoManager { @@ -7276,6 +7657,30 @@ interface LoroList { * own `{ cid, value }` nodes. */ getDeepValueWithID(): ValueWithContainerID; + + /** + * Get the deep value of the container as JSON text. + * + * The content is identical to `JSON.stringify` of the container's + * `toJSON()`, but the JSON text is produced inside WASM in a single call, + * avoiding the cost of crossing the WASM/JS boundary with a large + * structured value. + * + * Throws if the container is detached. + */ + getDeepValueJson(): string; + /** + * Get the deep value of the container as JSON text, plus container ids. + * + * `json` parses to the same content as `getDeepValueJson()` (the deep + * value WITHOUT container ids) and `cids` lists the container ids in + * pre-order DFS of the serialized JSON tree, so `cids[0]` is this + * container's own id. A single JS walk can re-attach the ids to + * reconstruct the `getDeepValueWithID()` shape. + * + * Throws if the container is detached. + */ + getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get the deep value of the elements in the range `[start, end)`, with * container ids. @@ -7379,6 +7784,30 @@ interface LoroMovableList { * own `{ cid, value }` nodes. */ getDeepValueWithID(): ValueWithContainerID; + + /** + * Get the deep value of the container as JSON text. + * + * The content is identical to `JSON.stringify` of the container's + * `toJSON()`, but the JSON text is produced inside WASM in a single call, + * avoiding the cost of crossing the WASM/JS boundary with a large + * structured value. + * + * Throws if the container is detached. + */ + getDeepValueJson(): string; + /** + * Get the deep value of the container as JSON text, plus container ids. + * + * `json` parses to the same content as `getDeepValueJson()` (the deep + * value WITHOUT container ids) and `cids` lists the container ids in + * pre-order DFS of the serialized JSON tree, so `cids[0]` is this + * container's own id. A single JS walk can re-attach the ids to + * reconstruct the `getDeepValueWithID()` shape. + * + * Throws if the container is detached. + */ + getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get the deep value of the elements in the range `[start, end)`, with * container ids. @@ -7548,6 +7977,30 @@ interface LoroMap = Record> { * `{ cid, value }` nodes. */ getDeepValueWithID(): ValueWithContainerID; + + /** + * Get the deep value of the container as JSON text. + * + * The content is identical to `JSON.stringify` of the container's + * `toJSON()`, but the JSON text is produced inside WASM in a single call, + * avoiding the cost of crossing the WASM/JS boundary with a large + * structured value. + * + * Throws if the container is detached. + */ + getDeepValueJson(): string; + /** + * Get the deep value of the container as JSON text, plus container ids. + * + * `json` parses to the same content as `getDeepValueJson()` (the deep + * value WITHOUT container ids) and `cids` lists the container ids in + * pre-order DFS of the serialized JSON tree, so `cids[0]` is this + * container's own id. A single JS walk can re-attach the ids to + * reconstruct the `getDeepValueWithID()` shape. + * + * Throws if the container is detached. + */ + getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get or create a regular child container at the given key. * @@ -7674,6 +8127,30 @@ interface LoroText { * string (the same as `text.id`) and `value` is the text content. */ getDeepValueWithID(): { cid: ContainerID, value: string }; + + /** + * Get the deep value of the container as JSON text. + * + * The content is identical to `JSON.stringify` of the container's + * `toJSON()`, but the JSON text is produced inside WASM in a single call, + * avoiding the cost of crossing the WASM/JS boundary with a large + * structured value. + * + * Throws if the container is detached. + */ + getDeepValueJson(): string; + /** + * Get the deep value of the container as JSON text, plus container ids. + * + * `json` parses to the same content as `getDeepValueJson()` (the deep + * value WITHOUT container ids) and `cids` lists the container ids in + * pre-order DFS of the serialized JSON tree, so `cids[0]` is this + * container's own id. A single JS walk can re-attach the ids to + * reconstruct the `getDeepValueWithID()` shape. + * + * Throws if the container is detached. + */ + getDeepValueJsonWithIds(): DeepValueJsonWithIds; insert(pos: number, text: string): void; delete(pos: number, len: number): void; subscribe(listener: Listener): Subscription; @@ -7719,6 +8196,30 @@ interface LoroTree = Record> * emits for every container. */ getDeepValueWithID(): ValueWithContainerID; + + /** + * Get the deep value of the container as JSON text. + * + * The content is identical to `JSON.stringify` of the container's + * `toJSON()`, but the JSON text is produced inside WASM in a single call, + * avoiding the cost of crossing the WASM/JS boundary with a large + * structured value. + * + * Throws if the container is detached. + */ + getDeepValueJson(): string; + /** + * Get the deep value of the container as JSON text, plus container ids. + * + * `json` parses to the same content as `getDeepValueJson()` (the deep + * value WITHOUT container ids) and `cids` lists the container ids in + * pre-order DFS of the serialized JSON tree, so `cids[0]` is this + * container's own id. A single JS walk can re-attach the ids to + * reconstruct the `getDeepValueWithID()` shape. + * + * Throws if the container is detached. + */ + getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Create a new tree node as the child of parent and return a `LoroTreeNode` instance. * If the parent is undefined, the tree node will be a root node. diff --git a/crates/loro-wasm/tests/deep_value.test.ts b/crates/loro-wasm/tests/deep_value.test.ts index 4fa55ee32..99904a3ba 100644 --- a/crates/loro-wasm/tests/deep_value.test.ts +++ b/crates/loro-wasm/tests/deep_value.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + LoroCounter, LoroDoc, LoroList, LoroMap, @@ -198,3 +199,234 @@ describe("range deep reads", () => { ).toThrow(); }); }); + +describe("getDeepValueJson", () => { + const setupAll = () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + map.set("flag", true); + map.set("n", 42); + const text = map.setContainer("text", new LoroText()); + text.insert(0, "Hello"); + const list = doc.getList("list"); + list.insert(0, "a"); + const sub = list.insertContainer(1, new LoroMap()); + sub.set("k", "v"); + const movable = doc.getMovableList("movable"); + movable.insert(0, "x"); + const tree = doc.getTree("tree"); + const root = tree.createNode(); + root.data.set("name", "root"); + const child = root.createNode(); + child.data.set("name", "child"); + const counter = doc.getCounter("counter"); + counter.increment(2.5); + doc.commit(); + return { doc, map, text, list, sub, movable, tree, counter }; + }; + + it("JSON.parse matches toJSON for a doc with every container type", () => { + const { doc, map, text, list, movable, tree, counter } = setupAll(); + expect(JSON.parse(doc.getDeepValueJson())).toStrictEqual(doc.toJSON()); + expect(JSON.parse(map.getDeepValueJson())).toStrictEqual(map.toJSON()); + expect(JSON.parse(text.getDeepValueJson())).toStrictEqual(text.toJSON()); + expect(JSON.parse(list.getDeepValueJson())).toStrictEqual(list.toJSON()); + expect(JSON.parse(movable.getDeepValueJson())).toStrictEqual( + movable.toJSON(), + ); + expect(JSON.parse(tree.getDeepValueJson())).toStrictEqual(tree.toJSON()); + expect(JSON.parse(counter.getDeepValueJson())).toStrictEqual( + counter.toJSON(), + ); + }); + + it("empty doc serializes to {}", () => { + const doc = new LoroDoc(); + expect(doc.getDeepValueJson()).toBe("{}"); + }); + + it("throws on detached containers instead of trapping", () => { + expect(() => new LoroMap().getDeepValueJson()).toThrow(); + expect(() => new LoroList().getDeepValueJson()).toThrow(); + expect(() => new LoroMovableList().getDeepValueJson()).toThrow(); + expect(() => new LoroTree().getDeepValueJson()).toThrow(); + expect(() => new LoroText().getDeepValueJson()).toThrow(); + expect(() => new LoroMap().getDeepValueJsonWithIds()).toThrow(); + expect(() => new LoroList().getDeepValueJsonWithIds()).toThrow(); + expect(() => new LoroMovableList().getDeepValueJsonWithIds()).toThrow(); + expect(() => new LoroTree().getDeepValueJsonWithIds()).toThrow(); + expect(() => new LoroText().getDeepValueJsonWithIds()).toThrow(); + // Counter mirrors toJSON(): it also works on a detached counter + expect(new LoroCounter().getDeepValueJson()).toBe("0.0"); + }); +}); + +describe("getDeepValueJsonWithIds", () => { + type ContainerTypeName = + | "Text" + | "Map" + | "List" + | "MovableList" + | "Tree" + | "Counter"; + + const containerTypeOf = (cid: string): ContainerTypeName => + cid.slice(cid.lastIndexOf(":") + 1) as ContainerTypeName; + + /** + * Rebuild the getDeepValueWithID() shape from the parsed `json` and the + * positional `cids` stream, walking against the known Loro grammar: + * root object -> every value is a container; Map -> object entries; + * List/MovableList/Tree -> arrays; Text -> string; Counter -> number. + */ + function reattachContainerIds( + json: unknown, + cids: readonly string[], + isRoot: boolean, + ): unknown { + let i = 0; + const shapeMatches = (type: ContainerTypeName, value: unknown): boolean => { + switch (type) { + case "Text": + return typeof value === "string"; + case "Counter": + return typeof value === "number"; + case "Map": + return ( + typeof value === "object" && value !== null && !Array.isArray(value) + ); + case "List": + case "MovableList": + case "Tree": + return Array.isArray(value); + } + }; + const walkContainerValue = ( + type: ContainerTypeName, + value: any, + ): unknown => { + switch (type) { + case "Text": + case "Counter": + return value; + case "Map": { + const out: Record = {}; + for (const [k, v] of Object.entries(value)) out[k] = attach(v); + return out; + } + case "List": + case "MovableList": + return value.map(attach); + case "Tree": + // Tree node meta maps are plain deep values; no container ids inside + return value; + } + }; + // Consume the next cid and wrap `value` as a { cid, value } node. + const attachForced = (value: unknown): ValueWithContainerID => { + const cid = cids[i++]; + return { + cid: cid as ValueWithContainerID["cid"], + value: walkContainerValue(containerTypeOf(cid), value) as never, + }; + }; + // Wrap `value` only if the next pending cid's type matches its shape; + // otherwise it is a plain value and is kept as-is. + const attach = (value: unknown): unknown => { + const cid = cids[i]; + if (cid === undefined || !shapeMatches(containerTypeOf(cid), value)) { + return value; + } + return attachForced(value); + }; + + if (!isRoot) return attachForced(json); + const out: Record = {}; + for (const [k, v] of Object.entries(json as Record)) { + out[k] = attachForced(v); + } + return out; + } + + const setupAll = () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + map.set("flag", true); + map.set("n", 42); + const text = map.setContainer("text", new LoroText()); + text.insert(0, "Hello"); + const list = doc.getList("list"); + list.insert(0, "a"); + const sub = list.insertContainer(1, new LoroMap()); + sub.set("k", "v"); + const movable = doc.getMovableList("movable"); + movable.insert(0, "x"); + const tree = doc.getTree("tree"); + const root = tree.createNode(); + root.data.set("name", "root"); + const child = root.createNode(); + child.data.set("name", "child"); + const counter = doc.getCounter("counter"); + counter.increment(2.5); + doc.commit(); + return { doc, map, text, list, sub, movable, tree, counter }; + }; + + it("doc-level: json + cids reconstruct getDeepValueWithID", () => { + const { doc, map, text, list, sub, movable, tree, counter } = setupAll(); + const { json, cids } = doc.getDeepValueJsonWithIds(); + + // json parses to the same content as toJSON() + expect(JSON.parse(json)).toStrictEqual(doc.toJSON()); + + // cids are the pre-order DFS of the serialized tree (root keys are + // serialized in sorted order: counter, list, map, movable, tree) + expect(cids).toStrictEqual([ + counter.id, + list.id, + sub.id, + map.id, + text.id, + movable.id, + tree.id, + ]); + + expect(reattachContainerIds(JSON.parse(json), cids, true)).toStrictEqual( + doc.getDeepValueWithID(), + ); + }); + + it("per-container: cids[0] is the container id and the walk round-trips", () => { + const { map, text, list, movable, tree, counter } = setupAll(); + const containers = [ + ["map", map], + ["text", text], + ["list", list], + ["movable", movable], + ["tree", tree], + ["counter", counter], + ] as const; + for (const [name, container] of containers) { + const { json, cids } = container.getDeepValueJsonWithIds(); + expect(cids[0], name).toBe(container.id); + expect(JSON.parse(json), name).toStrictEqual(container.toJSON()); + // LoroCounter has no getDeepValueWithID(); its node shape is trivially + // { cid: counter.id, value: number } + if (name === "counter") { + expect(cids, name).toStrictEqual([counter.id]); + continue; + } + expect( + reattachContainerIds(JSON.parse(json), cids, false), + name, + ).toStrictEqual(container.getDeepValueWithID()); + } + }); + + it("empty doc yields {} and no cids", () => { + const doc = new LoroDoc(); + const { json, cids } = doc.getDeepValueJsonWithIds(); + expect(json).toBe("{}"); + expect(cids).toStrictEqual([]); + }); +}); diff --git a/package.json b/package.json index f7aae33fe..70a66a94e 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "test-bundlers-next": "pnpm --dir examples/bundler-smoke-tests run test:next", "run-fuzz-corpus": "node ./scripts/cargo-fuzz-run.mjs all -- -max_total_time=1", "bench-wasm-container-id": "node --expose-gc ./crates/loro-wasm/scripts/measure-container-id.cjs", + "bench-deep-value-json": "node --expose-gc ./crates/loro-wasm/scripts/measure-deep-value-json.cjs", "fix": "cargo clippy --fix --features=test_utils", "vet": "cargo vet", "release-rust": "deno run -A ./scripts/cargo-release.ts" From 35d4c0b130daf4c348e9f0191347da18f66ad1b9 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Sat, 5 Sep 2026 17:14:31 +0800 Subject: [PATCH 2/7] fix(wasm): stream bulk JSON with exact container positions Replace ambiguous shape-based id stripping with a sparse Uint32 position index. Stream ephemeral container values directly into JSON, preserve numeric-key order and plain data, and benchmark identity-preserving reads and projection costs on cold imported documents. Model: gpt-6 --- .changeset/wasm-deep-value-json.md | 14 +- context/wasm-bulk-read.md | 226 +++++++---- crates/loro-internal/src/handler.rs | 79 ++-- crates/loro-internal/src/handler/tree.rs | 16 +- crates/loro-internal/src/lib.rs | 2 +- crates/loro-internal/src/loro.rs | 15 +- crates/loro-internal/src/state.rs | 128 ++---- crates/loro-internal/src/state/AGENTS.md | 4 + .../src/state/deep_value_json.rs | 233 +++++++++++ crates/loro-internal/tests/deep_value_json.rs | 100 ++++- .../scripts/measure-deep-value-json.cjs | 374 ++++++++++++------ crates/loro-wasm/src/counter.rs | 27 +- crates/loro-wasm/src/lib.rs | 323 +++++---------- crates/loro-wasm/tests/deep_value.test.ts | 240 ++++++----- 14 files changed, 1077 insertions(+), 704 deletions(-) create mode 100644 crates/loro-internal/src/state/deep_value_json.rs diff --git a/.changeset/wasm-deep-value-json.md b/.changeset/wasm-deep-value-json.md index 94703a450..92231b238 100644 --- a/.changeset/wasm-deep-value-json.md +++ b/.changeset/wasm-deep-value-json.md @@ -2,9 +2,11 @@ "loro-crdt": minor --- -Add JSON text export of deep values. `LoroDoc`, `LoroMap`, `LoroList`, `LoroMovableList`, `LoroTree`, `LoroText`, and `LoroCounter` now expose: - -- `getDeepValueJson(): string` — the same content as `JSON.stringify(x.toJSON())`, produced inside WASM in a single call, avoiding the cost of crossing the WASM/JS boundary with a large structured value. -- `getDeepValueJsonWithIds(): { json: string, cids: ContainerID[] }` — `json` parses to the same content as `getDeepValueJson()` (the deep value WITHOUT container ids) and `cids` lists the container id strings in pre-order DFS of the serialized JSON tree, so a consumer can re-attach ids in a single JS walk to reconstruct the `getDeepValueWithID()` shape. For a container, `cids[0]` is that container's own id. - -Detached containers throw a readable error instead of trapping (except `LoroCounter`, which mirrors `toJSON()` and also works detached). Note: a plain object value that has exactly the keys `cid` and `value` with `cid` being a valid container id string is indistinguishable from a container node in the `cids` format. Tree node meta maps are plain deep values, so meta container ids do not appear in `cids`. +Add streaming JSON deep reads on LoroDoc and all container classes. getDeepValueJson() +returns plain JSON; getDeepValueJsonWithIds() returns {json, cids, containerPositions}, +where containerPositions is an owned Uint32Array indexing every JSON value in +JavaScript pre-order traversal. This distinguishes scalars from containers, preserves +ordinary cid/value objects and handles numeric object keys correctly. The unmerged +preview's shape-based {json,cids} reconstruction must be replaced by position lookup. +Tree metadata remains plain deep data, matching getDeepValueWithID. Binary values +serialize as JSON arrays; document reads obey empty/deleted-root visibility. diff --git a/context/wasm-bulk-read.md b/context/wasm-bulk-read.md index 40b3ed1a0..903abd3c9 100644 --- a/context/wasm-bulk-read.md +++ b/context/wasm-bulk-read.md @@ -1,82 +1,154 @@ -# WASM Bulk Read APIs +# WASM bulk reads and the performance stack -Verified against code 2026-09-04 +Verified against code 2026-09-05. -Bulk-read APIs return whole (sub)document values in one WASM call instead of -per-key/per-index accessor round trips. +## Which cost each change removes + +- #1093 bounds the decoded-value cache used by individual handles. This fixes + retained WASM memory growth even for callers that keep their existing reads. +- #1085 caches wrapper `kind()`; #1086 adds subtree/range deep reads. These + reduce per-field JS/WASM calls. A visible history window should use range + reads rather than read an entire document just because a bulk API exists. +- #1087 serializes deep values in Rust and returns JSON text. Both JSON APIs + stream ephemeral container values into the output, without first building a + deep `LoroValue` tree or a `serde_json::Value` tree. The ID variant adds a + sparse position index. Tree metadata still uses its existing deep-value + conversion; the streaming improvement primarily targets Map/List/Text reads. Consumers can build their projection/registry in one + JS walk, without fetching each container again. +- #1090 defines the causal boundary for shallow snapshots. #1091 reduces the + cost of constructing their root state, with replay cost limits. These are + history/bootstrap/export changes, independent of the JSON read format. + +A faster bulk read does not automatically accelerate `new Mirror`: its caller +must adopt it while preserving schema decoding, ignored fields, container +registration, tree normalization and subscription behavior. The benchmark's +projection/registry cases model that read work, not the full Mirror constructor. ## APIs -- `LoroDoc.getDeepValueWithID()` (`loro-internal` `DocState::get_deep_value_with_id`): - root-name → `{ cid, value }` nodes. Child containers inside `value` are - recursively replaced by their own `{ cid, value }` nodes. -- Per-container `getDeepValueWithID()` on `LoroMap`/`LoroList`/ - `LoroMovableList`/`LoroTree`/`LoroText` (`Handler::get_deep_value_with_id` → - `DocState::get_container_deep_value_with_id`); same node shape, `cid` is the - container's own id. Detached containers return an error (JS: throw). -- `LoroList`/`LoroMovableList` `getRangeValue(start, end)` / - `getRangeDeepValueWithID(start, end)` (`DocState::get_list_range_deep_value`): - deep-read a `[start, end)` slice; bounds clamp, empty/inverted → `[]`. -- `getDeepValueJson(): string` on `LoroDoc` and all six container classes - (`DocState::get_deep_value_json`, `*Handler::get_deep_value_json`): JSON text - of the plain deep value — same content as `JSON.stringify(x.toJSON())`, - serialized in one WASM call via `serde_json::to_string(&LoroValue)`. -- `getDeepValueJsonWithIds(): { json: string, cids: ContainerID[] }` - (`DocState::get_deep_value_json_with_ids`, `strip_container_id_nodes` in - `crates/loro-internal/src/state.rs`): `json` is the deep value WITHOUT ids; - `cids` is the pre-order DFS of container ids in the serialized tree. - -## cid format - -`cid` is the bare container id string — exactly what the JS `container.id` -getter returns: `cid:root-:` for roots, `cid:@:` -for op-created containers (e.g. `cid:root-map:Map`, -`cid:92@2311024965712536503:Map`). Parse the container type from the suffix -after the last `:`. - -## cids pre-order contract - -`strip_container_id_nodes` converts the with-id `LoroValue` to a -`serde_json::Value` and walks THAT value once, collecting `cids` and building -the stripped JSON in the same pass, iterating each `serde_json::Map` in its own -iteration order. This keeps `cids` consistent with the key/item order a JS -consumer sees after `JSON.parse(json)`, regardless of serde_json's -`preserve_order` feature (this workspace does not enable it, so emitted JSON -object keys are sorted — deterministic, but different from -`getDeepValueJson()`'s direct `LoroValue` serialization order; only the parsed -content is identical between the two APIs). - -Pre-order means: a container's cid is pushed before recursing into its value. -For a container-level call, `cids[0]` is that container's own id. Re-attach -walk (see `crates/loro-wasm/tests/deep_value.test.ts`): root object → every -value is a container (consume the next cid); Map → object entries; -List/MovableList/Tree → arrays; Text → string; Counter → number. A position is -treated as a container only when the next pending cid's type matches the value -shape. - -Caveats: - -- **Tree node meta is a plain deep value** (`get_meta_value` in - `crates/loro-internal/src/state/tree_state.rs` resolves meta via - `get_container_deep_value`, not the with-id variant), so meta map container - ids never appear in `cids`, and a Tree's `value` subtree contains no - `{ cid, value }` nodes. -- **Structural ambiguity**: node detection is "object with exactly the keys - `cid` (a string that parses as `ContainerID`) and `value`". A user map that - stores such an object as plain data would be mistaken for a container node — - inherent to the format, accepted. Symmetrically, a consumer-side re-attach - walk cannot distinguish a plain string/number entry from a Text/Counter - child without schema knowledge. -- `LoroCounter` has no `getDeepValueWithID()`; its `getDeepValueJson()` / - `getDeepValueJsonWithIds()` work on detached counters too, mirroring - `toJSON()`. - -## Performance - -`crates/loro-wasm/scripts/measure-deep-value-json.cjs` (run via root -`pnpm bench-deep-value-json`) builds a ~70k-container doc (Map 15,632 / List -9,956 / Text 44,463, ~3.9 MB JSON) and compares `toJSON()`, -`getDeepValueWithID()`, `getDeepValueJson() (+ JSON.parse)`, and -`getDeepValueJsonWithIds() (+ parse + re-attach walk)`. The JSON text path is -multiple times faster than structured-cloning the with-id value across the -WASM boundary; see the script output for current numbers. +- `getDeepValueWithID()` on documents and Map/List/MovableList/Tree/Text returns + `{ cid, value }` nodes for containers, with bare `ContainerID` strings. +- List/MovableList `getRangeDeepValueWithID(start, end)` and + `getRangeValue(start, end)` read a clamped `[start, end)` slice in one call. +- `getDeepValueJson(): string` returns the plain deep value as JSON text. +- `getDeepValueJsonWithIds(): DeepValueJsonWithIds` returns: + +```ts +type DeepValueJsonWithIds = { + json: string; + cids: ContainerID[]; + containerPositions: Uint32Array; +}; +``` + +These JSON APIs exist on the document and all six container classes. Detached +containers throw, except Counter, which supports detached value reads. JSON +follows Rust's JSON serialization of the deep value: binary becomes an array, +non-finite numbers become null, and arbitrary plain objects stay ordinary data. +Document JSON obeys empty/deleted-root visibility. The older structured +`getDeepValueWithID()` does not apply those display filters. + +## Sparse position contract + +`cids[i]` identifies the value at `containerPositions[i]` in a **pre-order walk +of every value** in `JSON.parse(json)`: + +- Start at zero. Count each object, array and scalar once; do not count keys. +- Visit arrays by increasing index and objects in `Object.keys()` order. +- Binary is a JSON array: its byte values count too. +- A document's root object counts as zero, but is not a container. +- A container-level result marks position zero with its own cid. +- Tree metadata is plain deep data, matching the existing with-id API: count + those JSON values, but do not assign them additional cids. +- Positions increase strictly and have the same length as `cids`. The typed + array owns a copied buffer and survives subsequent WASM calls and doc.free(). + +For example, these values are distinct even when both fields contain `"same"`: + +```text +JSON: {"m":{"a":"same","b":"same"}} +walk: 0 1 2 3 + +Text at a: cids = [mapId, textId], positions = [1, 2] +Text at b: cids = [mapId, textId], positions = [1, 3] +``` + +The writer discovers identity from actual container edges, including mergeable +markers at map edges. It never strips objects that happen to contain `cid` and +`value`, nor guesses Text/Counter identity from a scalar type. It orders integer +property keys as JavaScript does (`"2"` before `"10"`; `"01"` and `"4294967295"` +are ordinary keys), independent of serde_json's `preserve_order` feature. + +A consumer can wrap known positions in `{cid, value}`, stamp map `$cid` fields, +or register identities directly during its own projection walk. Do not repeat +handle lookups to recover identities that are already present in the result. +The tests and benchmark contain complete reattachment examples; mutation of an +existing JSON.parse object preserves own `__proto__` keys without invoking the +inherited prototype setter. + +## Why positions instead of full paths + +For C containers at average depth D, full paths duplicate O(C*D) key/index +segments and require one path array per container. The sparse sidecar is O(C), +exactly 4*C bytes plus one JS typed-array allocation; it reuses the strings +already present in JSON. It also handles arbitrary keys without path escaping. + +The tradeoff is traversal: positions require visiting all JSON values, including +plain data. Paths allow direct navigation to each container and can have a faster +JS-only attachment step, especially with large plain subtrees. They still cost +path construction, serialization/transfer and extra allocations. Benchmark both +sides of this tradeoff instead of inferring speed from metadata bytes alone. + +The previous `{json,cids}` API in the unmerged PR did not record positions and +could not recover mixed primitive/container layouts. Consumers of that preview +must use the new index; the old shape-based reattachment is not compatible. + +## Reproducible measurements + +Run `pnpm release-wasm`, then `pnpm bench-deep-value-json`. The benchmark uses a +synthetic 70,051-container fixture (15,632 Map / 9,956 List / 44,463 Text) and +validates output before reporting results. No user document is stored. + +Each read case runs in a separate process with a newly imported snapshot. It +reports cold time, warm median (2 warm-ups + 5 measured rounds), and external, +heap and RSS deltas while retaining the first result. External memory includes +WASM linear-memory growth and other ArrayBuffers; it is not an exact Rust +allocation peak. The handle/indexed projection cases both stamp map identities +and register all containers. Path/position attachment timings isolate only JS +consumption, excluding path production/transfer; path payload size is also shown. + +Set `LORO_BENCH_MODULE=/absolute/path/to/nodejs/index.js` to run the same fixture +against another release build. A baseline without position support runs the +existing API cases only. Plain JSON and identity-preserving reads are separate +contracts and must not be advertised as interchangeable speed comparisons. + +## Measurement on this revision + +Release WASM, Node 22.23.1, macOS arm64, 2026-09-05; one run of the command above. Timings +are machine-dependent. The two identity-preserving rows reconstruct the same +with-id value; the two projection rows produce the same plain value and cid +registry (without Mirror schema/lifecycle work). + +| Read | Cold ms | Warm median ms | External delta MiB | +| --- | ---: | ---: | ---: | +| Structured getDeepValueWithID | 279.5 | 236.4 | 41.19 | +| Indexed JSON + parse + reattach | 248.1 | 204.4 | 19.39 | +| Per-handle projection + registry | 315.5 | 270.3 | 17.88 | +| Indexed JSON projection + registry | 249.1 | 207.2 | 19.13 | +| Plain streaming JSON + parse | 196.1 | 163.2 | 18.00 | + +Position metadata: 280,204 bytes. Full-path JSON metadata for the same +containers: 3,584,299 bytes (716,208 repeated key/index segments), excluding +cid strings common to both designs. Prebuilt-sidecar JS parse/attachment took +23.2 ms with positions vs 12.5 ms with paths. The latter excludes generating, +copying and decoding path metadata, so it is not an end-to-end path benchmark. +The position design trades a modest full-value JS walk for a 12.8x smaller +location payload and no per-container path arrays. It is not a claimed 5x +end-to-end speedup. + +The same benchmark against the original PR head (`7e99105a`, using +`LORO_BENCH_MODULE`) measured its legacy ID JSON producer at 365.3 ms cold / +234.3 ms warm and 96.06 MiB external growth. The streaming indexed producer +measured 226.8 ms / 174.3 ms and 19.39 MiB. These producer-only rows exclude +parse/reattachment, and the legacy result cannot represent all container layouts; +they are separate from the valid identity-preserving consumer comparison above. diff --git a/crates/loro-internal/src/handler.rs b/crates/loro-internal/src/handler.rs index 92aff4eec..628f1c9e3 100644 --- a/crates/loro-internal/src/handler.rs +++ b/crates/loro-internal/src/handler.rs @@ -2707,14 +2707,15 @@ impl TextHandler { inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) } - /// Get the deep value of the text as JSON text plus container ids. + /// Read plain JSON with a sparse container-position index. /// - /// Returns `(json, cids)` where `json` parses to the same content - /// as `get_deep_value_json()` (object key order may differ) and `cids` - /// lists the container ids in - /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this text's - /// own id. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult { let inner = self.inner.try_attached_state()?; inner.with_doc_state(|state| { state.get_container_deep_value_json_with_ids(inner.container_idx) @@ -3400,14 +3401,15 @@ impl ListHandler { inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) } - /// Get the deep value of this list as JSON text plus container ids. + /// Read plain JSON with a sparse container-position index. /// - /// Returns `(json, cids)` where `json` parses to the same content - /// as `get_deep_value_json()` (object key order may differ) and `cids` - /// lists the container ids in - /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this list's - /// own id. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult { let inner = self.inner.try_attached_state()?; inner.with_doc_state(|state| { state.get_container_deep_value_json_with_ids(inner.container_idx) @@ -4134,14 +4136,15 @@ impl MovableListHandler { inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) } - /// Get the deep value of this list as JSON text plus container ids. + /// Read plain JSON with a sparse container-position index. /// - /// Returns `(json, cids)` where `json` parses to the same content - /// as `get_deep_value_json()` (object key order may differ) and `cids` - /// lists the container ids in - /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this list's - /// own id. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult { let inner = self.inner.try_attached_state()?; inner.with_doc_state(|state| { state.get_container_deep_value_json_with_ids(inner.container_idx) @@ -4596,14 +4599,15 @@ impl MapHandler { inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) } - /// Get the deep value of the map as JSON text plus container ids. + /// Read plain JSON with a sparse container-position index. /// - /// Returns `(json, cids)` where `json` parses to the same content - /// as `get_deep_value_json()` (object key order may differ) and `cids` - /// lists the container ids in - /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this map's - /// own id. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult { let inner = self.inner.try_attached_state()?; inner.with_doc_state(|state| { state.get_container_deep_value_json_with_ids(inner.container_idx) @@ -5007,13 +5011,20 @@ pub mod counter { crate::state::deep_value_to_json(&self.get_value()) } - /// Get the counter value as JSON text plus container ids. + /// Read plain JSON with a sparse container-position index. /// - /// Returns `(json, cids)` where `json` is the same string - /// `get_deep_value_json()` returns and `cids` contains only this - /// counter's own id (a counter has no children). - pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { - Ok((self.get_deep_value_json()?, vec![self.id().to_string()])) + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult { + Ok(crate::DeepValueJsonWithIds { + json: self.get_deep_value_json()?, + cids: vec![self.id().to_string()], + container_positions: vec![0], + }) } } diff --git a/crates/loro-internal/src/handler/tree.rs b/crates/loro-internal/src/handler/tree.rs index f77053311..dd24d68fa 100644 --- a/crates/loro-internal/src/handler/tree.rs +++ b/crates/loro-internal/src/handler/tree.rs @@ -372,15 +372,15 @@ impl TreeHandler { inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) } - /// Get the deep value of the tree as JSON text plus container ids. + /// Read plain JSON with a sparse container-position index. /// - /// Returns `(json, cids)` where `json` parses to the same content - /// as `get_deep_value_json()` (object key order may differ) and `cids` - /// lists the container ids in - /// pre-order DFS of the serialized JSON tree, so `cids[0]` is this tree's - /// own id. Tree node meta maps are plain deep values, so their container - /// ids do not appear in `cids`. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. + pub fn get_deep_value_json_with_ids(&self) -> LoroResult { let inner = self.inner.try_attached_state()?; inner.with_doc_state(|state| { state.get_container_deep_value_json_with_ids(inner.container_idx) diff --git a/crates/loro-internal/src/lib.rs b/crates/loro-internal/src/lib.rs index d802af3a1..4691661ce 100644 --- a/crates/loro-internal/src/lib.rs +++ b/crates/loro-internal/src/lib.rs @@ -36,7 +36,7 @@ use pre_commit::{ PreCommitCallbackPayload, }; pub use rustc_hash::FxHashMap; -pub use state::DocState; +pub use state::{DeepValueJsonWithIds, DocState}; pub use state::{TreeNode, TreeNodeWithChildren, TreeParentId}; use subscription::{LocalUpdateCallback, Observer, PeerIdUpdateCallback}; use txn::Transaction; diff --git a/crates/loro-internal/src/loro.rs b/crates/loro-internal/src/loro.rs index 23719f9ce..8b08ef7b2 100644 --- a/crates/loro-internal/src/loro.rs +++ b/crates/loro-internal/src/loro.rs @@ -1618,15 +1618,16 @@ impl LoroDoc { self.state.lock().get_deep_value_json() } - /// Returns `(json, cids)` for the document's deep value. + /// Read plain JSON with a sparse container-position index. /// - /// `json` parses to the same content as [`Self::get_deep_value_json`] (the - /// deep value WITHOUT container ids; object key order may differ between - /// the two strings) and `cids` lists the container id strings in pre-order - /// DFS of the serialized JSON tree, so a consumer can re-attach ids in a - /// single walk. + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. #[inline] - pub fn get_deep_value_json_with_ids(&self) -> LoroResult<(String, Vec)> { + pub fn get_deep_value_json_with_ids(&self) -> LoroResult { self.state.lock().get_deep_value_json_with_ids() } diff --git a/crates/loro-internal/src/state.rs b/crates/loro-internal/src/state.rs index da62db193..d045b067f 100644 --- a/crates/loro-internal/src/state.rs +++ b/crates/loro-internal/src/state.rs @@ -35,6 +35,8 @@ pub(crate) mod container_store; #[cfg(feature = "counter")] mod counter_state; mod dead_containers_cache; +mod deep_value_json; +pub use deep_value_json::DeepValueJsonWithIds; mod list_state; mod map_state; mod mergeable; @@ -94,96 +96,13 @@ fn state_decode_error(message: impl Into>) -> LoroError { LoroError::DecodeError(message.into()) } -/// Serialize a deep value to JSON text. +/// Serialize a scalar counter value to JSON text. +#[cfg(feature = "counter")] pub(crate) fn deep_value_to_json(value: &LoroValue) -> LoroResult { serde_json::to_string(value) .map_err(|e| state_decode_error(format!("Failed to serialize deep value to JSON: {e}"))) } -/// Convert a with-id deep value (the node shape produced by -/// [`DocState::get_deep_value_with_id`] / [`DocState::get_container_deep_value_with_id`]) -/// into `(json, cids)`: -/// -/// - `json` is the JSON text of the same deep value WITHOUT ids — the same -/// content as the plain deep value, serialized through `serde_json::Value` -/// (object keys follow serde_json's map order). -/// - `cids` lists the container id strings in pre-order DFS of the serialized -/// JSON tree: when a container node is visited, its `cid` is pushed before -/// recursing into its `value`. -/// -/// The conversion runs in ONE pass: the with-id value is first converted to a -/// `serde_json::Value`, then that value is walked once, collecting `cids` and -/// building the stripped value while iterating each `serde_json::Map` in its -/// own iteration order. This keeps `cids` consistent with the key/item order a -/// consumer sees after parsing `json`, regardless of serde_json's -/// `preserve_order` feature. -/// -/// Node detection is structural: a JSON object is treated as a container node -/// iff it has exactly two keys, `cid` (a string that parses as a -/// [`ContainerID`]) and `value`. Because the with-id builder emits nodes with -/// exactly this shape, the walk does not need to dispatch on the container -/// type: every container node's `value` is walked recursively and any nested -/// nodes are found the same way. (Tree node objects carry their meta map as a -/// plain deep value, so they contain no nodes.) -/// -/// Known ambiguity: a user map that legitimately stores an object with exactly -/// the keys `cid` + `value` where `cid` happens to be a valid container id -/// string would be mistaken for a container node. This is inherent to the -/// format and accepted. -fn strip_container_id_nodes(with_id: LoroValue) -> LoroResult<(String, Vec)> { - let value = serde_json::to_value(&with_id) - .map_err(|e| state_decode_error(format!("Failed to serialize deep value to JSON: {e}")))?; - let mut cids = Vec::new(); - let stripped = strip_container_id_nodes_in_json(value, &mut cids); - let json = serde_json::to_string(&stripped) - .map_err(|e| state_decode_error(format!("Failed to serialize deep value to JSON: {e}")))?; - Ok((json, cids)) -} - -fn strip_container_id_nodes_in_json( - value: serde_json::Value, - cids: &mut Vec, -) -> serde_json::Value { - match value { - serde_json::Value::Object(map) => match take_container_node(map) { - Ok((cid, inner)) => { - cids.push(cid); - strip_container_id_nodes_in_json(inner, cids) - } - Err(map) => serde_json::Value::Object( - map.into_iter() - .map(|(k, v)| (k, strip_container_id_nodes_in_json(v, cids))) - .collect(), - ), - }, - serde_json::Value::Array(arr) => serde_json::Value::Array( - arr.into_iter() - .map(|v| strip_container_id_nodes_in_json(v, cids)) - .collect(), - ), - scalar => scalar, - } -} - -/// If `map` is a container node (`{ cid, value }` with a `cid` string that -/// parses as a [`ContainerID`]), return `(cid, value)`; otherwise return the -/// map unchanged. -fn take_container_node( - map: serde_json::Map, -) -> Result<(String, serde_json::Value), serde_json::Map> { - if map.len() == 2 - && matches!(map.get("cid"), Some(serde_json::Value::String(cid)) if ContainerID::try_from(cid.as_str()).is_ok()) - && map.contains_key("value") - { - let mut map = map; - let cid = map.remove("cid").unwrap().as_str().unwrap().to_string(); - let value = map.remove("value").unwrap(); - Ok((cid, value)) - } else { - Err(map) - } -} - fn decode_peer_table(bytes: &mut &[u8], context: &str) -> LoroResult> { let peer_num = leb128::read::unsigned(bytes) .map_err(|_| state_decode_error(format!("{context}: invalid peer table length")))?; @@ -1451,22 +1370,19 @@ impl DocState { /// text is produced in one pass so callers (e.g. the WASM bindings) can /// avoid a structured-clone round trip. pub fn get_deep_value_json(&mut self) -> LoroResult { - deep_value_to_json(&self.get_deep_value()) + Ok(self.write_deep_value_json(false)?.json) } - /// Returns `(json, cids)` for the document's deep value: - /// - /// - `json` parses to the same content as [`Self::get_deep_value_json`] - /// (the deep value WITHOUT container ids). Object key order may differ - /// between the two strings: this method serializes through a - /// `serde_json::Value` while `get_deep_value_json` serializes the - /// `LoroValue` directly. - /// - `cids` lists the container id strings in pre-order DFS of the - /// serialized JSON tree, so a consumer can re-attach ids in a single walk. + /// Read plain JSON with a sparse container-position index. /// - /// See [`strip_container_id_nodes`] for the format contract. - pub fn get_deep_value_json_with_ids(&mut self) -> LoroResult<(String, Vec)> { - strip_container_id_nodes(self.get_deep_value_with_id()) + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. + pub fn get_deep_value_json_with_ids(&mut self) -> LoroResult { + self.write_deep_value_json(true) } pub(crate) fn preferred_root_containers(&mut self) -> Vec { @@ -1675,18 +1591,22 @@ impl DocState { &mut self, container: ContainerIdx, ) -> LoroResult { - deep_value_to_json(&self.get_container_deep_value(container)) + Ok(self.write_container_deep_value_json(container, false)?.json) } - /// Container-level variant of [`Self::get_deep_value_json_with_ids`]. + /// Read plain JSON with a sparse container-position index. /// - /// `json` is the container's deep value (without ids) and `cids` lists the - /// container ids in pre-order DFS, so `cids[0]` is this container's own id. + /// `cids[i]` belongs to the JSON value at `container_positions[i]`. + /// Count every JSON value in pre-order, starting at zero, using JavaScript + /// `Object.keys` order for objects. Plain data never acquires a container id. + /// A document object counts as value zero but has no id; a container-level + /// result marks position zero. Tree metadata is plain deep data, as in + /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. pub(crate) fn get_container_deep_value_json_with_ids( &mut self, container: ContainerIdx, - ) -> LoroResult<(String, Vec)> { - strip_container_id_nodes(self.get_container_deep_value_with_id(container, None)) + ) -> LoroResult { + self.write_container_deep_value_json(container, true) } pub fn get_container_deep_value(&mut self, container: ContainerIdx) -> LoroValue { diff --git a/crates/loro-internal/src/state/AGENTS.md b/crates/loro-internal/src/state/AGENTS.md index 74c8f6680..6f34e8af5 100644 --- a/crates/loro-internal/src/state/AGENTS.md +++ b/crates/loro-internal/src/state/AGENTS.md @@ -15,6 +15,10 @@ before changing mergeable child behavior. bounded and evicted wrappers must stay re-creatable from KV; read [../../../../context/container-value-cache.md](../../../../context/container-value-cache.md) before changing read/caching paths there. +- `deep_value_json.rs`: streaming JSON reads and sparse container positions. + Identity comes from CRDT edges; positions count every JSON value in JavaScript + property order. Never infer containers from scalar types or cid/value objects. + See [../../../../context/wasm-bulk-read.md](../../../../context/wasm-bulk-read.md). - `map_state.rs`, `list_state.rs`, `richtext_state.rs`, `tree_state.rs`, `movable_list_state.rs`, `counter_state.rs`: per-container state and snapshot codecs. `richtext_state.rs` also hosts `redact_dead_style_values`, used by diff --git a/crates/loro-internal/src/state/deep_value_json.rs b/crates/loro-internal/src/state/deep_value_json.rs new file mode 100644 index 000000000..9e89253b0 --- /dev/null +++ b/crates/loro-internal/src/state/deep_value_json.rs @@ -0,0 +1,233 @@ +//! Streaming bulk JSON reads. Container identity comes from CRDT edges, never +//! from the shape of user values. The sparse sidecar indexes *all* JSON values. +use super::{ + deleted_root_container_value_is_cleared, get_meta_value, state_decode_error, + visible_container_value_is_empty, DocState, +}; +use crate::{container::idx::ContainerIdx, ContainerType, LoroValue}; +use loro_common::{ContainerID, LoroResult}; +use std::sync::atomic::Ordering; + +/// Plain JSON and a sparse index of container positions in its parsed value. +#[derive(Debug)] +pub struct DeepValueJsonWithIds { + pub json: String, + pub cids: Vec, + /// Zero-based pre-order positions: count every JSON value, including the + /// document object, scalars, plain objects/arrays, and binary array items. + /// Object children follow JavaScript's `Object.keys` order. + pub container_positions: Vec, +} + +#[derive(Default)] +struct JsonWriter { + bytes: Vec, + cids: Vec, + positions: Vec, + next: u32, + with_ids: bool, +} + +impl JsonWriter { + fn count(&mut self, n: u32) -> LoroResult<()> { + if !self.with_ids { + return Ok(()); + } + self.next = self.next.checked_add(n).ok_or_else(|| { + state_decode_error("Bulk JSON has too many values for its container position index") + })?; + Ok(()) + } + + fn json(&mut self, value: &impl serde::Serialize) -> LoroResult<()> { + serde_json::to_writer(&mut self.bytes, value) + .map_err(|e| state_decode_error(format!("Failed to serialize deep value: {e}"))) + } + + fn finish(self) -> LoroResult { + Ok(DeepValueJsonWithIds { + json: String::from_utf8(self.bytes) + .map_err(|e| state_decode_error(format!("Invalid bulk JSON UTF-8: {e}")))?, + cids: self.cids, + container_positions: self.positions, + }) + } + + fn container(&mut self, state: &mut DocState, idx: ContainerIdx) -> LoroResult<()> { + let id = state.arena.idx_to_id(idx).unwrap(); + let value = state + .store + .get_value_ephemeral(idx) + .unwrap_or_else(|| idx.get_type().default_value()); + self.container_value(state, &id, value) + } + + fn container_value( + &mut self, + state: &mut DocState, + id: &ContainerID, + mut value: LoroValue, + ) -> LoroResult<()> { + if self.with_ids { + self.positions.push(self.next); + self.cids.push(id.to_string()); + } + // Match getDeepValueWithID: tree metadata is plain deep data, not + // additional with-id nodes. Its JSON values still count as positions. + if id.container_type() == ContainerType::Tree { + if let LoroValue::List(list) = &mut value { + get_meta_value(list.make_mut(), state); + } + return self.value(state, &value, None); + } + self.value(state, &value, Some(id)) + } + + fn child(&mut self, state: &mut DocState, value: &LoroValue, resolve: bool) -> LoroResult<()> { + if resolve { + if let LoroValue::Container(id) = value { + let idx = state.arena.register_container(id); + return self.container(state, idx); + } + } + self.value(state, value, None) + } + + fn value( + &mut self, + state: &mut DocState, + value: &LoroValue, + parent: Option<&ContainerID>, + ) -> LoroResult<()> { + self.count(1)?; + match value { + LoroValue::Map(map) => { + let mut entries: Vec<_> = map.iter().collect(); + entries.sort_unstable_by(|(a, _), (b, _)| json_key_cmp(a, b)); + self.bytes.push(b'{'); + for (i, (key, value)) in entries.into_iter().enumerate() { + if i > 0 { + self.bytes.push(b','); + } + self.json(key)?; + self.bytes.push(b':'); + // Resolve compact mergeable markers only at actual map + // edges. Identical bytes in plain user data stay data. + let mergeable = parent.and_then(|id| { + loro_common::parse_mergeable_marker(id, key, value) + .map(|kind| ContainerID::new_mergeable(id, key, kind)) + }); + if let Some(id) = mergeable { + let idx = state.arena.register_container(&id); + self.container(state, idx)?; + } else { + self.child(state, value, parent.is_some())?; + } + } + self.bytes.push(b'}'); + } + LoroValue::List(list) => { + self.bytes.push(b'['); + for (i, value) in list.iter().enumerate() { + if i > 0 { + self.bytes.push(b','); + } + self.child(state, value, parent.is_some())?; + } + self.bytes.push(b']'); + } + LoroValue::Binary(bytes) => { + self.count( + u32::try_from(bytes.len()) + .map_err(|_| state_decode_error("Bulk JSON binary is too large"))?, + )?; + self.json(value)?; + } + _ => self.json(value)?, + } + Ok(()) + } +} + +// ECMA-262 array-index property keys precede other keys in Object.keys, even +// after JSON.parse. 2^32-1, leading-zero spellings and negative keys are NOT +// array indices. Sorting other keys gives deterministic output on every target. +fn array_index(key: &str) -> Option { + if key.is_empty() + || key.len() > 10 + || (key.len() > 1 && key.starts_with('0')) + || !key.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + key.parse::().ok().filter(|&n| n != u32::MAX) +} + +fn json_key_cmp(a: &str, b: &str) -> std::cmp::Ordering { + match (array_index(a), array_index(b)) { + (Some(a), Some(b)) => a.cmp(&b), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => a.cmp(b), + } +} + +impl DocState { + pub(super) fn write_deep_value_json( + &mut self, + with_ids: bool, + ) -> LoroResult { + let mut writer = JsonWriter { + with_ids, + ..Default::default() + }; + writer.count(1)?; // The document object is value zero, not a container. + writer.bytes.push(b'{'); + let mut roots: Vec<_> = self + .preferred_root_containers() + .into_iter() + .map(|idx| (self.root_container_name(idx).unwrap(), idx)) + .collect(); + roots.sort_unstable_by(|(a, _), (b, _)| json_key_cmp(a, b)); + let hide_empty = self + .config + .hide_empty_root_containers + .load(Ordering::Relaxed); + let mut first = true; + for (key, idx) in roots { + let id = self.arena.idx_to_id(idx).unwrap(); + let value = self + .store + .get_value_ephemeral(idx) + .unwrap_or_else(|| idx.get_type().default_value()); + if (hide_empty && visible_container_value_is_empty(idx.get_type(), &value)) + || (self.config.deleted_root_containers.lock().contains(&id) + && deleted_root_container_value_is_cleared(idx.get_type(), &value)) + { + continue; + } + if !first { + writer.bytes.push(b','); + } + first = false; + writer.json(&key)?; + writer.bytes.push(b':'); + writer.container_value(self, &id, value)?; + } + writer.bytes.push(b'}'); + writer.finish() + } + + pub(super) fn write_container_deep_value_json( + &mut self, + idx: ContainerIdx, + with_ids: bool, + ) -> LoroResult { + let mut writer = JsonWriter { + with_ids, + ..Default::default() + }; + writer.container(self, idx)?; + writer.finish() + } +} diff --git a/crates/loro-internal/tests/deep_value_json.rs b/crates/loro-internal/tests/deep_value_json.rs index 014b9d450..887299829 100644 --- a/crates/loro-internal/tests/deep_value_json.rs +++ b/crates/loro-internal/tests/deep_value_json.rs @@ -49,14 +49,18 @@ fn build_doc() -> LoroResult<(LoroDoc, Vec)> { fn deep_value_json_matches_plain_deep_value_serialization() -> LoroResult<()> { let (doc, _) = build_doc()?; let expected = serde_json::to_string(&doc.get_deep_value()).unwrap(); - assert_eq!(doc.get_deep_value_json()?, expected); + assert_eq!( + serde_json::from_str::(&doc.get_deep_value_json()?).unwrap(), + serde_json::from_str::(&expected).unwrap(), + ); Ok(()) } #[test] fn deep_value_json_with_ids_doc_level() -> LoroResult<()> { let (doc, expected_cids) = build_doc()?; - let (json, cids) = doc.get_deep_value_json_with_ids()?; + let loro_internal::DeepValueJsonWithIds { json, cids, .. } = + doc.get_deep_value_json_with_ids()?; // json parses to the same content as the plain deep value JSON (object // key order may differ between the two strings; see the API docs) @@ -88,7 +92,8 @@ fn deep_value_json_with_ids_per_container() -> LoroResult<()> { let (doc, _) = build_doc()?; let map = doc.get_map("map"); - let (json, cids) = map.get_deep_value_json_with_ids()?; + let loro_internal::DeepValueJsonWithIds { json, cids, .. } = + map.get_deep_value_json_with_ids()?; // cids[0] is the container's own id (pre-order includes the root container) assert_eq!(cids[0], map.id().to_string()); assert_eq!(cids.len(), 3, "map, list, text"); @@ -104,12 +109,14 @@ fn deep_value_json_with_ids_per_container() -> LoroResult<()> { let text = doc.get_text("text"); text.insert_unicode(0, "abc")?; - let (json, cids) = text.get_deep_value_json_with_ids()?; + let loro_internal::DeepValueJsonWithIds { json, cids, .. } = + text.get_deep_value_json_with_ids()?; assert_eq!(json, "\"abc\""); assert_eq!(cids, vec![text.id().to_string()]); let tree = doc.get_tree("tree"); - let (json, cids) = tree.get_deep_value_json_with_ids()?; + let loro_internal::DeepValueJsonWithIds { json, cids, .. } = + tree.get_deep_value_json_with_ids()?; assert_eq!(cids, vec![tree.id().to_string()]); assert_eq!( serde_json::from_str::(&json).unwrap(), @@ -119,7 +126,8 @@ fn deep_value_json_with_ids_per_container() -> LoroResult<()> { #[cfg(feature = "counter")] { let counter = doc.get_counter("counter"); - let (json, cids) = counter.get_deep_value_json_with_ids()?; + let loro_internal::DeepValueJsonWithIds { json, cids, .. } = + counter.get_deep_value_json_with_ids()?; assert_eq!(json, "2.5"); assert_eq!(cids, vec![counter.id().to_string()]); } @@ -130,7 +138,8 @@ fn deep_value_json_with_ids_per_container() -> LoroResult<()> { fn deep_value_json_empty_doc() -> LoroResult<()> { let doc = LoroDoc::new_auto_commit(); assert_eq!(doc.get_deep_value_json()?, "{}"); - let (json, cids) = doc.get_deep_value_json_with_ids()?; + let loro_internal::DeepValueJsonWithIds { json, cids, .. } = + doc.get_deep_value_json_with_ids()?; assert_eq!(json, "{}"); assert!(cids.is_empty()); Ok(()) @@ -145,3 +154,80 @@ fn deep_value_json_detached_container_errors() { assert!(list.get_deep_value_json().is_err()); assert!(list.get_deep_value_json_with_ids().is_err()); } + +#[test] +fn positions_distinguish_identical_values_with_different_container_layouts() -> LoroResult<()> { + let build = |text_key: &str, plain_key: &str| -> LoroResult<_> { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1)?; + let map = doc.get_map("m"); + map.insert_container(text_key, TextHandler::new_detached())? + .insert_unicode(0, "same")?; + map.insert(plain_key, "same")?; + doc.get_deep_value_json_with_ids() + }; + let a = build("a", "b")?; + let b = build("b", "a")?; + assert_eq!(a.json, b.json); + assert_eq!(a.cids, b.cids); + assert_eq!(a.container_positions, vec![1, 2]); + assert_eq!(b.container_positions, vec![1, 3]); + Ok(()) +} + +#[test] +fn positions_survive_numeric_keys_plain_lookalikes_and_snapshot_import() -> LoroResult<()> { + use loro_common::LoroValue; + use loro_internal::encoding::ExportMode; + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1)?; + let map = doc.get_map("m"); + // No shape-based stripping: all of this is ordinary user data. + map.insert("0", LoroValue::from(vec![1u8, 2, 3]))?; + let plain: LoroValue = + serde_json::from_str(r#"{"cid":"cid:root-fake:Map","value":{"n":1}}"#).unwrap(); + map.insert("1", plain)?; + map.insert_container("10", TextHandler::new_detached())? + .insert_unicode(0, "ten")?; + map.insert_container("2", TextHandler::new_detached())? + .insert_unicode(0, "two")?; + doc.commit_then_renew(); + let imported = LoroDoc::new(); + imported.import(&doc.export(ExportMode::Snapshot)?)?; + for d in [&doc, &imported] { + let out = d.get_deep_value_json_with_ids()?; + assert_eq!(out.container_positions, vec![1, 10, 11]); + assert_eq!(out.cids.len(), 3); + assert_eq!( + serde_json::from_str::(&out.json).unwrap(), + serde_json::to_value(d.get_deep_value()).unwrap() + ); + assert!(out.json.contains("cid:root-fake:Map")); + assert!(out.json.find("two").unwrap() < out.json.find("ten").unwrap()); + } + Ok(()) +} + +#[test] +fn streaming_read_resolves_mergeable_children_and_respects_root_visibility() -> LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + let map = doc.get_map("m"); + let child = map.ensure_mergeable_map("child")?; + child.insert("number", 42)?; + let out = doc.get_deep_value_json_with_ids()?; + assert_eq!(out.cids, vec![map.id().to_string(), child.id().to_string()]); + assert_eq!(out.container_positions, vec![1, 2]); + assert_eq!( + serde_json::from_str::(&out.json).unwrap(), + serde_json::to_value(doc.get_deep_value()).unwrap() + ); + doc.get_text("empty"); + doc.config().set_hide_empty_root_containers(true); + let out = doc.get_deep_value_json_with_ids()?; + assert_eq!(out.cids, vec![map.id().to_string(), child.id().to_string()]); + assert_eq!( + serde_json::from_str::(&out.json).unwrap(), + serde_json::to_value(doc.get_deep_value()).unwrap() + ); + Ok(()) +} diff --git a/crates/loro-wasm/scripts/measure-deep-value-json.cjs b/crates/loro-wasm/scripts/measure-deep-value-json.cjs index 7fa7dacc6..e549e7c53 100644 --- a/crates/loro-wasm/scripts/measure-deep-value-json.cjs +++ b/crates/loro-wasm/scripts/measure-deep-value-json.cjs @@ -1,15 +1,19 @@ const { performance } = require("node:perf_hooks"); -const { LoroDoc, LoroList, LoroMap, LoroText } = require("../nodejs/index.js"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const modulePath = + process.env.LORO_BENCH_MODULE || + path.resolve(__dirname, "../nodejs/index.js"); +const { LoroDoc, LoroList, LoroMap, LoroText } = require(modulePath); -// Mirrors the distribution of a real-world document: -// Map 15,632 / List 9,956 / Text 44,463 containers (~70k in total), -// ~3.9 MB of JSON content. const MAP_COUNT = 15_632; const LIST_COUNT = 9_956; const TEXT_COUNT = 44_463; const PARENT_POOL_SIZE = 2_048; -const PARAGRAPH = - "The quick brown fox jumps over the lazy dog. Pack my box. "; +const PARAGRAPH = "The quick brown fox jumps over the lazy dog. Pack my box. "; const WARMUP_ROUNDS = 2; const ROUNDS = 5; @@ -17,6 +21,7 @@ let blackhole = 0; function buildDocument() { const doc = new LoroDoc(); + doc.setPeerId("1"); const root = doc.getMap("root"); const parents = [root]; let mi = 1; @@ -59,126 +64,261 @@ function buildDocument() { return doc; } -function median(values) { - const sorted = [...values].sort((a, b) => a - b); - return sorted[Math.floor(sorted.length / 2)]; +function decode(result, project = false) { + let next = 0, + position = 0; + const registry = new Set(); + function walk(value) { + const cid = + result.containerPositions[next] === position++ + ? result.cids[next++] + : undefined; + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) value[i] = walk(value[i]); + } else if (value !== null && typeof value === "object") { + for (const key of Object.keys(value)) value[key] = walk(value[key]); + } + if (cid === undefined) return value; + if (!project) return { cid, value }; + registry.add(cid); + if (cid.endsWith(":Map")) + Object.defineProperty(value, "$cid", { value: cid }); + return value; + } + const value = walk(JSON.parse(result.json)); + assert.equal(next, result.cids.length); + return project ? { value, registry } : value; } -function timed(name, run) { - const samples = []; - for (let round = 0; round < WARMUP_ROUNDS + ROUNDS; round++) { - global.gc?.(); - const start = performance.now(); - run(); - const elapsed = performance.now() - start; - if (round >= WARMUP_ROUNDS) samples.push(elapsed); +function pathsFor(result) { + let next = 0, + position = 0; + const paths = [], + stack = []; + function walk(value) { + if (result.containerPositions[next] === position++) { + paths.push(stack.slice()); + next++; + } + if (value !== null && typeof value === "object") { + for (const key of Object.keys(value)) { + stack.push(Array.isArray(value) ? Number(key) : key); + walk(value[key]); + stack.pop(); + } + } } - return { name, medianMs: median(samples), samplesMs: samples }; + walk(JSON.parse(result.json)); + assert.equal(next, result.cids.length); + return paths; } -// The same pre-order re-attach walk documented for consumers of -// getDeepValueJsonWithIds(): root object -> every value is a container; -// Map -> object entries; List/MovableList/Tree -> arrays; Text -> string; -// Counter -> number. -function makeReattach(cids) { - let i = 0; - const containerTypeOf = (cid) => cid.slice(cid.lastIndexOf(":") + 1); - const shapeMatches = (type, value) => { - switch (type) { - case "Text": - return typeof value === "string"; - case "Counter": - return typeof value === "number"; - case "Map": - return ( - typeof value === "object" && value !== null && !Array.isArray(value) - ); - case "List": - case "MovableList": - case "Tree": - return Array.isArray(value); +function attachPaths(json, cids, paths) { + let value = JSON.parse(json); + // Descendants first so replacing a parent does not invalidate its paths. + for (let i = paths.length - 1; i >= 0; i--) { + const steps = paths[i]; + if (!steps.length) { + value = { cid: cids[i], value }; + continue; } - }; - const attach = (value) => { - const cid = cids[i]; - if (cid === undefined || !shapeMatches(containerTypeOf(cid), value)) { - return value; + let parent = value; + for (let j = 0; j < steps.length - 1; j++) parent = parent[steps[j]]; + const key = steps[steps.length - 1]; + parent[key] = { cid: cids[i], value: parent[key] }; + } + return value; +} + +function handleProjection(container, registry) { + const kind = container.kind(); + const cid = container.id; + registry.add(cid); + let out; + if (kind === "Map") { + out = {}; + Object.defineProperty(out, "$cid", { value: cid }); + for (const key of container.keys()) { + const child = container.get(key); + out[key] = + child && typeof child.kind === "function" + ? handleProjection(child, registry) + : child; } - return attachForced(value); - }; - const attachForced = (value) => { - const cid = cids[i++]; - return { cid, value: walkContainerValue(containerTypeOf(cid), value) }; - }; - const walkContainerValue = (type, value) => { - switch (type) { - case "Text": - case "Counter": - return value; - case "Map": { - const out = {}; - for (const k of Object.keys(value)) out[k] = attach(value[k]); - return out; - } - case "List": - case "MovableList": - return value.map(attach); - case "Tree": - return value; + } else if (kind === "List" || kind === "MovableList") { + out = []; + for (let i = 0, n = container.length; i < n; i++) { + const child = container.get(i); + out.push( + child && typeof child.kind === "function" + ? handleProjection(child, registry) + : child, + ); } - }; - return (json) => { - const out = {}; - for (const k of Object.keys(json)) out[k] = attachForced(json[k]); - blackhole += i; - return out; - }; + } else { + out = container.toJSON(); + } + container.free(); + return out; } -const doc = buildDocument(); -const stats = { - containers: MAP_COUNT + LIST_COUNT + TEXT_COUNT, - jsonBytes: Buffer.byteLength(doc.getDeepValueJson(), "utf8"), -}; - -const cases = [ - ["toJSON", () => blackhole += JSON.stringify(doc.toJSON()).length], - ["getDeepValueWithID", () => blackhole += Object.keys(doc.getDeepValueWithID()).length], - ["getDeepValueJson", () => blackhole += doc.getDeepValueJson().length], - [ - "getDeepValueJson+parse", - () => blackhole += Object.keys(JSON.parse(doc.getDeepValueJson())).length, - ], - [ - "getDeepValueJsonWithIds+parse+reattach", - () => { - const { json, cids } = doc.getDeepValueJsonWithIds(); - const out = makeReattach(cids)(JSON.parse(json)); - blackhole += Object.keys(out).length; +const name = process.argv[2]; +if (name) { + // Each case gets its own process and a freshly imported snapshot. Do not + // hide allocations in an already materialized, directly built document. + const snapshot = fs.readFileSync(process.argv[3]); + const doc = new LoroDoc(); + const startImport = performance.now(); + doc.import(snapshot); + const importMs = performance.now() - startImport; + const run = { + toJSON: () => doc.toJSON(), + getDeepValueWithID: () => doc.getDeepValueWithID(), + jsonParse: () => JSON.parse(doc.getDeepValueJson()), + indexedJson: () => doc.getDeepValueJsonWithIds(), + legacyJson: () => doc.getDeepValueJsonWithIds(), + indexedJsonParseReattach: () => decode(doc.getDeepValueJsonWithIds()), + indexedProjection: () => decode(doc.getDeepValueJsonWithIds(), true), + handleProjection: () => { + const registry = new Set(); + return { + value: { root: handleProjection(doc.getMap("root"), registry) }, + registry, + }; }, - ], -]; - -const result = cases.map(([name, run]) => timed(name, run)); -const withId = result.find((r) => r.name === "getDeepValueWithID").medianMs; -const jsonParse = result.find( - (r) => r.name === "getDeepValueJson+parse", -).medianMs; - -console.log( - JSON.stringify( - { - ...stats, - warmupRounds: WARMUP_ROUNDS, - rounds: ROUNDS, + }[name]; + global.gc?.(); + const before = process.memoryUsage(); + const start = performance.now(); + let held = run(); + const coldMs = performance.now() - start; + global.gc?.(); + const after = process.memoryUsage(); + const memory = Object.fromEntries( + ["external", "heapUsed", "rss"].map((k) => [ + k + "DeltaBytes", + after[k] - before[k], + ]), + ); + // Correctness outside timed regions; a faster read of the wrong shape is + // not an optimization. Projection includes cid registration and map $cid. + if (name === "indexedJson" || name === "indexedJsonParseReattach") { + assert.deepStrictEqual( + name === "indexedJson" ? decode(held) : held, + doc.getDeepValueWithID(), + ); + } else if (name === "indexedProjection" || name === "handleProjection") { + assert.deepStrictEqual(held.value, doc.toJSON()); + assert.equal(held.registry.size, MAP_COUNT + LIST_COUNT + TEXT_COUNT); + assert.equal(held.value.root.$cid, doc.getMap("root").id); + } else if (name === "legacyJson") { + assert.deepStrictEqual( + JSON.parse(held.json), + JSON.parse(doc.getDeepValueJson()), + ); + } else if (name !== "getDeepValueWithID") { + assert.deepStrictEqual(held, doc.toJSON()); + } + held = null; + const samples = []; + for (let round = 0; round < WARMUP_ROUNDS + ROUNDS; round++) { + global.gc?.(); + const t = performance.now(); + held = run(); + const elapsed = performance.now() - t; + blackhole += Object.keys(held).length; + held = null; + if (round >= WARMUP_ROUNDS) samples.push(elapsed); + } + samples.sort((a, b) => a - b); + console.log( + JSON.stringify({ + name, + importMs, + coldMs, + warmMedianMs: samples[Math.floor(samples.length / 2)], + ...memory, blackhole, - result, - speedup: { - "getDeepValueJson+parse vs getDeepValueWithID": withId / jsonParse, - }, - }, - null, - 2, - ), -); -doc.free(); + }), + ); + doc.free(); +} else { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "loro-bulk-bench-")); + try { + const doc = buildDocument(); + const file = path.join(dir, "fixture.bin"); + fs.writeFileSync(file, doc.export({ mode: "snapshot" })); + const result = doc.getDeepValueJsonWithIds(); + const supportsPositions = result.containerPositions instanceof Uint32Array; + const cases = [ + "toJSON", + "getDeepValueWithID", + "jsonParse", + "handleProjection", + ]; + let metadata; + if (supportsPositions) { + cases.push( + "indexedJson", + "indexedJsonParseReattach", + "indexedProjection", + ); + const paths = pathsFor(result); + assert.deepStrictEqual( + attachPaths(result.json, result.cids, paths), + decode(result), + ); + const positionSamples = [], + pathSamples = []; + for (let i = 0; i < WARMUP_ROUNDS + ROUNDS; i++) { + global.gc?.(); + let t = performance.now(); + decode(result); + const posMs = performance.now() - t; + t = performance.now(); + attachPaths(result.json, result.cids, paths); + const pathMs = performance.now() - t; + if (i >= WARMUP_ROUNDS) { + positionSamples.push(posMs); + pathSamples.push(pathMs); + } + } + const median = (xs) => + xs.sort((a, b) => a - b)[Math.floor(xs.length / 2)]; + metadata = { + positionsBytes: result.containerPositions.byteLength, + pathsJsonBytes: Buffer.byteLength(JSON.stringify(paths)), + pathSegments: paths.reduce((n, p) => n + p.length, 0), + cidsJsonBytes: Buffer.byteLength(JSON.stringify(result.cids)), + // These isolate JS consumption; path generation/transfer is excluded. + positionsParseAttachMs: median(positionSamples), + pathsParseAttachMs: median(pathSamples), + }; + } + if (!supportsPositions) cases.push("legacyJson"); + doc.free(); + const measurements = cases.map((name) => { + const child = spawnSync( + process.execPath, + ["--expose-gc", __filename, name, file], + { encoding: "utf8", env: process.env }, + ); + if (child.status !== 0) throw new Error(child.stderr || child.stdout); + return JSON.parse(child.stdout); + }); + console.log( + JSON.stringify( + { + containers: MAP_COUNT + LIST_COUNT + TEXT_COUNT, + jsonBytes: Buffer.byteLength(result.json), + metadata, + measurements, + }, + null, + 2, + ), + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} diff --git a/crates/loro-wasm/src/counter.rs b/crates/loro-wasm/src/counter.rs index d2b64525b..f78a4d879 100644 --- a/crates/loro-wasm/src/counter.rs +++ b/crates/loro-wasm/src/counter.rs @@ -157,25 +157,20 @@ impl LoroCounter { Ok(self.handler.get_deep_value_json()?) } - /// Get the counter value as JSON text, plus container ids. + /// Read plain JSON plus an exact, sparse container-position index. /// - /// Returns `{ json, cids }` where `json` is the same string - /// `getDeepValueJson()` returns and `cids` contains only this counter's - /// own id (a counter has no children). + /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. + /// Count every value in pre-order from zero, including scalars and plain + /// objects/arrays; visit object children in `Object.keys` order. + /// The document object counts as zero but has no id; for a container call, + /// position zero identifies that container. Tree metadata remains plain + /// deep data, matching `getDeepValueWithID()`. /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; - /// - /// const doc = new LoroDoc(); - /// const counter = doc.getCounter("counter"); - /// counter.increment(1.5); - /// const { json, cids } = counter.getDeepValueJsonWithIds(); - /// // json === "1.5", cids === [counter.id] - /// ``` + /// No schema or value-shape guesses are needed. The positions are a copied + /// Uint32Array (four bytes per container), not a view into WASM memory. + /// Detached containers throw, except counters which also support reads. #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] pub fn get_deep_value_json_with_ids(&self) -> JsResult { - let (json, cids) = self.handler.get_deep_value_json_with_ids()?; - crate::deep_value_json_with_ids_to_js(json, cids) + crate::deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) } } diff --git a/crates/loro-wasm/src/lib.rs b/crates/loro-wasm/src/lib.rs index 6ec8f541b..ced878011 100644 --- a/crates/loro-wasm/src/lib.rs +++ b/crates/loro-wasm/src/lib.rs @@ -525,7 +525,14 @@ fn id_to_js(id: &ID) -> JsResult { Ok(obj.into()) } -pub(crate) fn deep_value_json_with_ids_to_js(json: String, cids: Vec) -> JsResult { +pub(crate) fn deep_value_json_with_ids_to_js( + result: loro_internal::DeepValueJsonWithIds, +) -> JsResult { + let loro_internal::DeepValueJsonWithIds { + json, + cids, + container_positions, + } = result; let obj = Object::new(); Reflect::set(&obj, &"json".into(), &json.into())?; let arr = Array::new(); @@ -533,6 +540,8 @@ pub(crate) fn deep_value_json_with_ids_to_js(json: String, cids: Vec) -> arr.push(&cid.into()); } Reflect::set(&obj, &"cids".into(), &arr)?; + let positions = js_sys::Uint32Array::from(container_positions.as_slice()); + Reflect::set(&obj, &"containerPositions".into(), &positions)?; Ok(obj.into()) } @@ -1498,33 +1507,21 @@ impl LoroDoc { Ok(self.doc.get_deep_value_json()?) } - /// Get the deep value of the document as JSON text, plus container ids. - /// - /// Returns `{ json, cids }` where `json` parses to the same content as - /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key - /// order may differ between the two strings) and `cids` lists the - /// container id strings in pre-order DFS of the serialized JSON tree, so a - /// consumer can re-attach ids in a single JS walk to reconstruct the - /// `getDeepValueWithID()` shape. - /// - /// Note: a plain object value that has exactly the keys `cid` and `value` - /// with `cid` being a valid container id string is indistinguishable from - /// a container node in this format. + /// Read plain JSON plus an exact, sparse container-position index. /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; + /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. + /// Count every value in pre-order from zero, including scalars and plain + /// objects/arrays; visit object children in `Object.keys` order. + /// The document object counts as zero but has no id; for a container call, + /// position zero identifies that container. Tree metadata remains plain + /// deep data, matching `getDeepValueWithID()`. /// - /// const doc = new LoroDoc(); - /// const text = doc.getText("text"); - /// text.insert(0, "Hello"); - /// const { json, cids } = doc.getDeepValueJsonWithIds(); - /// // json === '{"text":"Hello"}', cids === ["cid:root-text:Text"] - /// ``` + /// No schema or value-shape guesses are needed. The positions are a copied + /// Uint32Array (four bytes per container), not a view into WASM memory. + /// Detached containers throw, except counters which also support reads. #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] pub fn get_deep_value_json_with_ids(&self) -> JsResult { - let (json, cids) = self.doc.get_deep_value_json_with_ids()?; - deep_value_json_with_ids_to_js(json, cids) + deep_value_json_with_ids_to_js(self.doc.get_deep_value_json_with_ids()?) } /// Get the path from the root to the container @@ -3337,35 +3334,21 @@ impl LoroText { Ok(self.handler.get_deep_value_json()?) } - /// Get the deep value of the text as JSON text, plus container ids. - /// - /// Returns `{ json, cids }` where `json` parses to the same content as - /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key - /// order may differ between the two strings) and `cids` lists the - /// container ids in pre-order DFS of the serialized JSON tree, so - /// `cids[0]` is this container's own id. A single JS walk can re-attach - /// the ids to reconstruct the `getDeepValueWithID()` shape. - /// - /// Note: a plain object value that has exactly the keys `cid` and `value` - /// with `cid` being a valid container id string is indistinguishable from - /// a container node in this format. + /// Read plain JSON plus an exact, sparse container-position index. /// - /// Throws if the container is detached. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; + /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. + /// Count every value in pre-order from zero, including scalars and plain + /// objects/arrays; visit object children in `Object.keys` order. + /// The document object counts as zero but has no id; for a container call, + /// position zero identifies that container. Tree metadata remains plain + /// deep data, matching `getDeepValueWithID()`. /// - /// const doc = new LoroDoc(); - /// const text = doc.getText("text"); - /// text.insert(0, "Hello"); - /// const { json, cids } = text.getDeepValueJsonWithIds(); - /// // json === '\"Hello\"', cids === ["cid:root-text:Text"] - /// ``` + /// No schema or value-shape guesses are needed. The positions are a copied + /// Uint32Array (four bytes per container), not a view into WASM memory. + /// Detached containers throw, except counters which also support reads. #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] pub fn get_deep_value_json_with_ids(&self) -> JsResult { - let (json, cids) = self.handler.get_deep_value_json_with_ids()?; - deep_value_json_with_ids_to_js(json, cids) + deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) } } @@ -3678,35 +3661,21 @@ impl LoroMap { Ok(self.handler.get_deep_value_json()?) } - /// Get the deep value of the map as JSON text, plus container ids. - /// - /// Returns `{ json, cids }` where `json` parses to the same content as - /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key - /// order may differ between the two strings) and `cids` lists the - /// container ids in pre-order DFS of the serialized JSON tree, so - /// `cids[0]` is this container's own id. A single JS walk can re-attach - /// the ids to reconstruct the `getDeepValueWithID()` shape. - /// - /// Note: a plain object value that has exactly the keys `cid` and `value` - /// with `cid` being a valid container id string is indistinguishable from - /// a container node in this format. - /// - /// Throws if the container is detached. + /// Read plain JSON plus an exact, sparse container-position index. /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; + /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. + /// Count every value in pre-order from zero, including scalars and plain + /// objects/arrays; visit object children in `Object.keys` order. + /// The document object counts as zero but has no id; for a container call, + /// position zero identifies that container. Tree metadata remains plain + /// deep data, matching `getDeepValueWithID()`. /// - /// const doc = new LoroDoc(); - /// const map = doc.getMap("map"); - /// map.set("foo", "bar"); - /// const { json, cids } = map.getDeepValueJsonWithIds(); - /// // json === '{\"foo\":\"bar\"}', cids === ["cid:root-map:Map"] - /// ``` + /// No schema or value-shape guesses are needed. The positions are a copied + /// Uint32Array (four bytes per container), not a view into WASM memory. + /// Detached containers throw, except counters which also support reads. #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] pub fn get_deep_value_json_with_ids(&self) -> JsResult { - let (json, cids) = self.handler.get_deep_value_json_with_ids()?; - deep_value_json_with_ids_to_js(json, cids) + deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) } /// Set the key with a regular child container. @@ -4162,35 +4131,21 @@ impl LoroList { Ok(self.handler.get_deep_value_json()?) } - /// Get the deep value of the list as JSON text, plus container ids. - /// - /// Returns `{ json, cids }` where `json` parses to the same content as - /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key - /// order may differ between the two strings) and `cids` lists the - /// container ids in pre-order DFS of the serialized JSON tree, so - /// `cids[0]` is this container's own id. A single JS walk can re-attach - /// the ids to reconstruct the `getDeepValueWithID()` shape. - /// - /// Note: a plain object value that has exactly the keys `cid` and `value` - /// with `cid` being a valid container id string is indistinguishable from - /// a container node in this format. - /// - /// Throws if the container is detached. + /// Read plain JSON plus an exact, sparse container-position index. /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; + /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. + /// Count every value in pre-order from zero, including scalars and plain + /// objects/arrays; visit object children in `Object.keys` order. + /// The document object counts as zero but has no id; for a container call, + /// position zero identifies that container. Tree metadata remains plain + /// deep data, matching `getDeepValueWithID()`. /// - /// const doc = new LoroDoc(); - /// const list = doc.getList("list"); - /// list.insert(0, 100); - /// const { json, cids } = list.getDeepValueJsonWithIds(); - /// // json === '[100]', cids === ["cid:root-list:List"] - /// ``` + /// No schema or value-shape guesses are needed. The positions are a copied + /// Uint32Array (four bytes per container), not a view into WASM memory. + /// Detached containers throw, except counters which also support reads. #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] pub fn get_deep_value_json_with_ids(&self) -> JsResult { - let (json, cids) = self.handler.get_deep_value_json_with_ids()?; - deep_value_json_with_ids_to_js(json, cids) + deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) } /// Get the deep value of the elements in the range `[start, end)`, with @@ -4657,35 +4612,21 @@ impl LoroMovableList { Ok(self.handler.get_deep_value_json()?) } - /// Get the deep value of the movableList as JSON text, plus container ids. - /// - /// Returns `{ json, cids }` where `json` parses to the same content as - /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key - /// order may differ between the two strings) and `cids` lists the - /// container ids in pre-order DFS of the serialized JSON tree, so - /// `cids[0]` is this container's own id. A single JS walk can re-attach - /// the ids to reconstruct the `getDeepValueWithID()` shape. - /// - /// Note: a plain object value that has exactly the keys `cid` and `value` - /// with `cid` being a valid container id string is indistinguishable from - /// a container node in this format. + /// Read plain JSON plus an exact, sparse container-position index. /// - /// Throws if the container is detached. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; + /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. + /// Count every value in pre-order from zero, including scalars and plain + /// objects/arrays; visit object children in `Object.keys` order. + /// The document object counts as zero but has no id; for a container call, + /// position zero identifies that container. Tree metadata remains plain + /// deep data, matching `getDeepValueWithID()`. /// - /// const doc = new LoroDoc(); - /// const movableList = doc.getMovableList("list"); - /// movableList.insert(0, 100); - /// const { json, cids } = movableList.getDeepValueJsonWithIds(); - /// // json === '[100]', cids === ["cid:root-list:MovableList"] - /// ``` + /// No schema or value-shape guesses are needed. The positions are a copied + /// Uint32Array (four bytes per container), not a view into WASM memory. + /// Detached containers throw, except counters which also support reads. #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] pub fn get_deep_value_json_with_ids(&self) -> JsResult { - let (json, cids) = self.handler.get_deep_value_json_with_ids()?; - deep_value_json_with_ids_to_js(json, cids) + deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) } /// Get the deep value of the elements in the range `[start, end)`, with @@ -5535,35 +5476,21 @@ impl LoroTree { Ok(self.handler.get_deep_value_json()?) } - /// Get the deep value of the tree as JSON text, plus container ids. + /// Read plain JSON plus an exact, sparse container-position index. /// - /// Returns `{ json, cids }` where `json` parses to the same content as - /// `getDeepValueJson()` (the deep value WITHOUT container ids; object key - /// order may differ between the two strings) and `cids` lists the - /// container ids in pre-order DFS of the serialized JSON tree, so - /// `cids[0]` is this container's own id. A single JS walk can re-attach - /// the ids to reconstruct the `getDeepValueWithID()` shape. - /// - /// Note: a plain object value that has exactly the keys `cid` and `value` - /// with `cid` being a valid container id string is indistinguishable from - /// a container node in this format. - /// - /// Throws if the container is detached. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; + /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. + /// Count every value in pre-order from zero, including scalars and plain + /// objects/arrays; visit object children in `Object.keys` order. + /// The document object counts as zero but has no id; for a container call, + /// position zero identifies that container. Tree metadata remains plain + /// deep data, matching `getDeepValueWithID()`. /// - /// const doc = new LoroDoc(); - /// const tree = doc.getTree("tree"); - /// tree.createNode(); - /// const { json, cids } = tree.getDeepValueJsonWithIds(); - /// // json === '[{\"id\":\"...\",\"parent\":null,\"meta\":{},\"index\":0,\"fractional_index\":\"...\",\"children\":[]}]', cids === ["cid:root-tree:Tree"] - /// ``` + /// No schema or value-shape guesses are needed. The positions are a copied + /// Uint32Array (four bytes per container), not a view into WASM memory. + /// Detached containers throw, except counters which also support reads. #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] pub fn get_deep_value_json_with_ids(&self) -> JsResult { - let (json, cids) = self.handler.get_deep_value_json_with_ids()?; - deep_value_json_with_ids_to_js(json, cids) + deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) } /// Get all tree nodes of the forest, including deleted nodes. @@ -6965,22 +6892,24 @@ export type ValueWithContainerID = { } /** - * The result of `getDeepValueJsonWithIds()`. + * Plain JSON plus a sparse index of container identities. * - * `json` is the JSON text of the deep value WITHOUT container ids (it parses - * to the same content as `getDeepValueJson()`). `cids` lists the container id - * strings in pre-order DFS of the serialized JSON tree: the ids appear in the - * same order as the key/item order a consumer sees after `JSON.parse(json)`, - * so a single JS walk can re-attach ids and reconstruct the - * `getDeepValueWithID()` shape. + * `cids[i]` belongs to value `containerPositions[i]` in a pre-order walk of + * JSON.parse(json). Count EVERY value (objects, arrays and scalars), starting + * at zero; visit arrays in index order and objects in Object.keys order. + * Binary data serializes as an array, so each byte counts as another value. + * Document calls count the root object but do not assign it a cid; container + * calls mark position zero. Tree metadata is plain deep data, matching the + * existing getDeepValueWithID format. Ordinary cid/value objects stay data. * - * Note: a plain object value that has exactly the keys `cid` and `value` with - * `cid` being a valid container id string is indistinguishable from a - * container node in this format. + * This JSON view follows getDeepValueJson's root visibility and JSON scalar + * representation (e.g. binary arrays and non-finite numbers as null). + * containerPositions owns its buffer and remains valid after further WASM calls. */ export type DeepValueJsonWithIds = { json: string, cids: ContainerID[], + containerPositions: Uint32Array, } export type IdSpan = { @@ -7476,14 +7405,7 @@ interface LoroDoc { * of crossing the WASM/JS boundary with a large structured value. */ getDeepValueJson(): string; - /** - * Get the deep value of the document as JSON text, plus container ids. - * - * `json` parses to the same content as `getDeepValueJson()` (the deep - * value WITHOUT container ids) and `cids` lists the container ids in - * pre-order DFS of the serialized JSON tree, so a single JS walk can - * re-attach the ids to reconstruct the `getDeepValueWithID()` shape. - */ + /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ getDeepValueJsonWithIds(): DeepValueJsonWithIds; } @@ -7495,12 +7417,7 @@ interface LoroCounter { * counter, mirroring `toJSON()`. */ getDeepValueJson(): string; - /** - * Get the counter value as JSON text, plus container ids. - * - * `json` is the same string `getDeepValueJson()` returns and `cids` - * contains only this counter's own id (a counter has no children). - */ + /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ getDeepValueJsonWithIds(): DeepValueJsonWithIds; } @@ -7669,17 +7586,7 @@ interface LoroList { * Throws if the container is detached. */ getDeepValueJson(): string; - /** - * Get the deep value of the container as JSON text, plus container ids. - * - * `json` parses to the same content as `getDeepValueJson()` (the deep - * value WITHOUT container ids) and `cids` lists the container ids in - * pre-order DFS of the serialized JSON tree, so `cids[0]` is this - * container's own id. A single JS walk can re-attach the ids to - * reconstruct the `getDeepValueWithID()` shape. - * - * Throws if the container is detached. - */ + /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get the deep value of the elements in the range `[start, end)`, with @@ -7796,17 +7703,7 @@ interface LoroMovableList { * Throws if the container is detached. */ getDeepValueJson(): string; - /** - * Get the deep value of the container as JSON text, plus container ids. - * - * `json` parses to the same content as `getDeepValueJson()` (the deep - * value WITHOUT container ids) and `cids` lists the container ids in - * pre-order DFS of the serialized JSON tree, so `cids[0]` is this - * container's own id. A single JS walk can re-attach the ids to - * reconstruct the `getDeepValueWithID()` shape. - * - * Throws if the container is detached. - */ + /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get the deep value of the elements in the range `[start, end)`, with @@ -7989,17 +7886,7 @@ interface LoroMap = Record> { * Throws if the container is detached. */ getDeepValueJson(): string; - /** - * Get the deep value of the container as JSON text, plus container ids. - * - * `json` parses to the same content as `getDeepValueJson()` (the deep - * value WITHOUT container ids) and `cids` lists the container ids in - * pre-order DFS of the serialized JSON tree, so `cids[0]` is this - * container's own id. A single JS walk can re-attach the ids to - * reconstruct the `getDeepValueWithID()` shape. - * - * Throws if the container is detached. - */ + /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get or create a regular child container at the given key. @@ -8139,17 +8026,7 @@ interface LoroText { * Throws if the container is detached. */ getDeepValueJson(): string; - /** - * Get the deep value of the container as JSON text, plus container ids. - * - * `json` parses to the same content as `getDeepValueJson()` (the deep - * value WITHOUT container ids) and `cids` lists the container ids in - * pre-order DFS of the serialized JSON tree, so `cids[0]` is this - * container's own id. A single JS walk can re-attach the ids to - * reconstruct the `getDeepValueWithID()` shape. - * - * Throws if the container is detached. - */ + /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ getDeepValueJsonWithIds(): DeepValueJsonWithIds; insert(pos: number, text: string): void; delete(pos: number, len: number): void; @@ -8208,17 +8085,7 @@ interface LoroTree = Record> * Throws if the container is detached. */ getDeepValueJson(): string; - /** - * Get the deep value of the container as JSON text, plus container ids. - * - * `json` parses to the same content as `getDeepValueJson()` (the deep - * value WITHOUT container ids) and `cids` lists the container ids in - * pre-order DFS of the serialized JSON tree, so `cids[0]` is this - * container's own id. A single JS walk can re-attach the ids to - * reconstruct the `getDeepValueWithID()` shape. - * - * Throws if the container is detached. - */ + /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Create a new tree node as the child of parent and return a `LoroTreeNode` instance. diff --git a/crates/loro-wasm/tests/deep_value.test.ts b/crates/loro-wasm/tests/deep_value.test.ts index 99904a3ba..18d2e1991 100644 --- a/crates/loro-wasm/tests/deep_value.test.ts +++ b/crates/loro-wasm/tests/deep_value.test.ts @@ -135,12 +135,7 @@ describe("range deep reads", () => { it("getRangeValue resolves containers to plain deep values", () => { const { list } = setup(); expect(list.getRangeValue(1, 3)).toStrictEqual(["b", { k: "v" }]); - expect(list.getRangeValue(0, 4)).toStrictEqual([ - "a", - "b", - { k: "v" }, - "d", - ]); + expect(list.getRangeValue(0, 4)).toStrictEqual(["a", "b", { k: "v" }, "d"]); }); it("clamps out-of-range bounds and returns [] for empty ranges", () => { @@ -194,9 +189,7 @@ describe("range deep reads", () => { expect(() => new LoroList().getRangeValue(0, 1)).toThrow(); expect(() => new LoroList().getRangeDeepValueWithID(0, 1)).toThrow(); expect(() => new LoroMovableList().getRangeValue(0, 1)).toThrow(); - expect(() => - new LoroMovableList().getRangeDeepValueWithID(0, 1), - ).toThrow(); + expect(() => new LoroMovableList().getRangeDeepValueWithID(0, 1)).toThrow(); }); }); @@ -262,90 +255,31 @@ describe("getDeepValueJson", () => { }); describe("getDeepValueJsonWithIds", () => { - type ContainerTypeName = - | "Text" - | "Map" - | "List" - | "MovableList" - | "Tree" - | "Counter"; - - const containerTypeOf = (cid: string): ContainerTypeName => - cid.slice(cid.lastIndexOf(":") + 1) as ContainerTypeName; - - /** - * Rebuild the getDeepValueWithID() shape from the parsed `json` and the - * positional `cids` stream, walking against the known Loro grammar: - * root object -> every value is a container; Map -> object entries; - * List/MovableList/Tree -> arrays; Text -> string; Counter -> number. - */ - function reattachContainerIds( - json: unknown, - cids: readonly string[], - isRoot: boolean, - ): unknown { - let i = 0; - const shapeMatches = (type: ContainerTypeName, value: unknown): boolean => { - switch (type) { - case "Text": - return typeof value === "string"; - case "Counter": - return typeof value === "number"; - case "Map": - return ( - typeof value === "object" && value !== null && !Array.isArray(value) - ); - case "List": - case "MovableList": - case "Tree": - return Array.isArray(value); - } - }; - const walkContainerValue = ( - type: ContainerTypeName, - value: any, - ): unknown => { - switch (type) { - case "Text": - case "Counter": - return value; - case "Map": { - const out: Record = {}; - for (const [k, v] of Object.entries(value)) out[k] = attach(v); - return out; - } - case "List": - case "MovableList": - return value.map(attach); - case "Tree": - // Tree node meta maps are plain deep values; no container ids inside - return value; + // Walk every parsed JSON value. Identity is determined only by the sparse + // position index, never by a scalar/object shape or a schema guess. + function reattachContainerIds(result: { + json: string; + cids: readonly string[]; + containerPositions: Uint32Array; + }): unknown { + let position = 0; + let next = 0; + const walk = (value: any): unknown => { + const cid = + result.containerPositions[next] === position++ + ? result.cids[next++] + : undefined; + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) value[i] = walk(value[i]); + } else if (value !== null && typeof value === "object") { + for (const key of Object.keys(value)) value[key] = walk(value[key]); } + return cid === undefined ? value : { cid, value }; }; - // Consume the next cid and wrap `value` as a { cid, value } node. - const attachForced = (value: unknown): ValueWithContainerID => { - const cid = cids[i++]; - return { - cid: cid as ValueWithContainerID["cid"], - value: walkContainerValue(containerTypeOf(cid), value) as never, - }; - }; - // Wrap `value` only if the next pending cid's type matches its shape; - // otherwise it is a plain value and is kept as-is. - const attach = (value: unknown): unknown => { - const cid = cids[i]; - if (cid === undefined || !shapeMatches(containerTypeOf(cid), value)) { - return value; - } - return attachForced(value); - }; - - if (!isRoot) return attachForced(json); - const out: Record = {}; - for (const [k, v] of Object.entries(json as Record)) { - out[k] = attachForced(v); - } - return out; + const value = walk(JSON.parse(result.json)); + expect(next).toBe(result.cids.length); + expect(next).toBe(result.containerPositions.length); + return value; } const setupAll = () => { @@ -374,7 +308,8 @@ describe("getDeepValueJsonWithIds", () => { it("doc-level: json + cids reconstruct getDeepValueWithID", () => { const { doc, map, text, list, sub, movable, tree, counter } = setupAll(); - const { json, cids } = doc.getDeepValueJsonWithIds(); + const result = doc.getDeepValueJsonWithIds(); + const { json, cids } = result; // json parses to the same content as toJSON() expect(JSON.parse(json)).toStrictEqual(doc.toJSON()); @@ -391,7 +326,7 @@ describe("getDeepValueJsonWithIds", () => { tree.id, ]); - expect(reattachContainerIds(JSON.parse(json), cids, true)).toStrictEqual( + expect(reattachContainerIds(result)).toStrictEqual( doc.getDeepValueWithID(), ); }); @@ -407,7 +342,8 @@ describe("getDeepValueJsonWithIds", () => { ["counter", counter], ] as const; for (const [name, container] of containers) { - const { json, cids } = container.getDeepValueJsonWithIds(); + const result = container.getDeepValueJsonWithIds(); + const { json, cids } = result; expect(cids[0], name).toBe(container.id); expect(JSON.parse(json), name).toStrictEqual(container.toJSON()); // LoroCounter has no getDeepValueWithID(); its node shape is trivially @@ -416,16 +352,122 @@ describe("getDeepValueJsonWithIds", () => { expect(cids, name).toStrictEqual([counter.id]); continue; } - expect( - reattachContainerIds(JSON.parse(json), cids, false), - name, - ).toStrictEqual(container.getDeepValueWithID()); + expect(reattachContainerIds(result), name).toStrictEqual( + container.getDeepValueWithID(), + ); } }); + it("distinguishes scalar strings from Text containers at either position", () => { + const results = ["a", "b"].map((textKey) => { + const doc = new LoroDoc(); + doc.setPeerId("1"); + const map = doc.getMap("m"); + map.setContainer(textKey, new LoroText()).insert(0, "same"); + map.set(textKey === "a" ? "b" : "a", "same"); + const result = doc.getDeepValueJsonWithIds(); + expect(reattachContainerIds(result)).toStrictEqual( + doc.getDeepValueWithID(), + ); + return result; + }); + expect(results[0].json).toBe(results[1].json); + expect(results[0].cids).toStrictEqual(results[1].cids); + expect([...results[0].containerPositions]).toStrictEqual([1, 2]); + expect([...results[1].containerPositions]).toStrictEqual([1, 3]); + }); + + it("follows JS integer-key order at document and nested map levels", () => { + const doc = new LoroDoc(); + for (const key of [ + "10", + "2", + "01", + "0", + "4294967294", + "4294967295", + "-1", + ]) { + const map = doc.getMap(key); + map.setContainer("10", new LoroText()).insert(0, "ten"); + map.setContainer("2", new LoroText()).insert(0, "two"); + map.set("0", "plain"); + } + const result = doc.getDeepValueJsonWithIds(); + expect(reattachContainerIds(result)).toStrictEqual( + doc.getDeepValueWithID(), + ); + expect(JSON.parse(result.json)).toStrictEqual(doc.toJSON()); + }); + + it("preserves cid/value lookalikes and counts plain nested data and binary items", () => { + const doc = new LoroDoc(); + const map = doc.getMap("m"); + map.set("a", { cid: "cid:root-fake:Text", value: "ordinary data" }); + map.set("b", { + "10": [1, "literal"], + "2": { cid: "cid:root-fake:Map", value: {} }, + }); + map.set("c", new Uint8Array([3, 4, 5])); + const text = map.setContainer("z", new LoroText()); + text.insert(0, "actual container"); + const result = doc.getDeepValueJsonWithIds(); + expect(result.cids).toStrictEqual([map.id, text.id]); + expect(JSON.parse(result.json)).toStrictEqual( + JSON.parse(doc.getDeepValueJson()), + ); + // Binary's JSON representation is an array rather than toJSON's Uint8Array. + const rebuilt = reattachContainerIds(result) as any; + expect(rebuilt.m.value.a).toStrictEqual({ + cid: "cid:root-fake:Text", + value: "ordinary data", + }); + expect(rebuilt.m.value.z).toStrictEqual({ + cid: text.id, + value: "actual container", + }); + }); + + it("keeps __proto__ as an own JSON property without changing prototypes", () => { + const doc = new LoroDoc(); + doc.getText("__proto__").insert(0, "data"); + const result = doc.getDeepValueJsonWithIds(); + const parsed = JSON.parse(result.json); + const restored = reattachContainerIds(result) as Record; + expect(Object.getPrototypeOf(parsed)).toBe(Object.prototype); + expect(Object.getPrototypeOf(restored)).toBe(Object.prototype); + expect(Object.prototype.hasOwnProperty.call(restored, "__proto__")).toBe( + true, + ); + expect(restored.__proto__).toStrictEqual({ + cid: "cid:root-__proto__:Text", + value: "data", + }); + }); + + it("preserves mixed scalar/container arrays and plain empty values", () => { + const doc = new LoroDoc(); + const list = doc.getList("list"); + list.push("same"); + list.pushContainer(new LoroText()).insert(0, "same"); + list.push({}); + list.pushContainer(new LoroMap()); + list.push([]); + list.pushContainer(new LoroList()); + list.push(null); + list.push(1.5); + list.pushContainer(new LoroCounter()).increment(1.5); + const result = list.getDeepValueJsonWithIds(); + expect(reattachContainerIds(result)).toStrictEqual( + list.getDeepValueWithID(), + ); + expect(result.containerPositions[0]).toBe(0); + }); + it("empty doc yields {} and no cids", () => { const doc = new LoroDoc(); - const { json, cids } = doc.getDeepValueJsonWithIds(); + const result = doc.getDeepValueJsonWithIds(); + const { json, cids } = result; expect(json).toBe("{}"); expect(cids).toStrictEqual([]); }); From e72db6bdc1b57fed65cbecc17e9a3749b3561101 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Sat, 5 Sep 2026 22:25:15 +0800 Subject: [PATCH 3/7] feat(wasm): construct typed read state with fixed JS helpers Replace the unpublished positional JSON proposal with explicit nested container nodes and opaque ordinary values. See context/wasm-bulk-read.md for semantics and measurements. --- .changeset/wasm-deep-value-json.md | 12 - .changeset/wasm-read-state.md | 5 + AGENTS.md | 5 +- context/wasm-bulk-read.md | 239 +++++------ crates/loro-internal/src/handler.rs | 120 ------ crates/loro-internal/src/handler/tree.rs | 24 -- crates/loro-internal/src/lib.rs | 4 +- crates/loro-internal/src/loro.rs | 22 -- crates/loro-internal/src/state.rs | 55 +-- crates/loro-internal/src/state/AGENTS.md | 8 +- .../src/state/deep_value_json.rs | 233 ----------- crates/loro-internal/src/state/read_state.rs | 244 ++++++++++++ crates/loro-internal/tests/deep_value_json.rs | 233 ----------- crates/loro-wasm/AGENTS.md | 9 +- .../scripts/measure-deep-value-json.cjs | 324 --------------- crates/loro-wasm/src/counter.rs | 37 -- crates/loro-wasm/src/lib.rs | 370 +----------------- crates/loro-wasm/src/read_state.rs | 263 +++++++++++++ crates/loro-wasm/tests/deep_value.test.ts | 292 +------------- crates/loro-wasm/tests/read_state.test.ts | 236 +++++++++++ package.json | 1 - 21 files changed, 867 insertions(+), 1869 deletions(-) delete mode 100644 .changeset/wasm-deep-value-json.md create mode 100644 .changeset/wasm-read-state.md delete mode 100644 crates/loro-internal/src/state/deep_value_json.rs create mode 100644 crates/loro-internal/src/state/read_state.rs delete mode 100644 crates/loro-internal/tests/deep_value_json.rs delete mode 100644 crates/loro-wasm/scripts/measure-deep-value-json.cjs create mode 100644 crates/loro-wasm/src/read_state.rs create mode 100644 crates/loro-wasm/tests/read_state.test.ts diff --git a/.changeset/wasm-deep-value-json.md b/.changeset/wasm-deep-value-json.md deleted file mode 100644 index 92231b238..000000000 --- a/.changeset/wasm-deep-value-json.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -"loro-crdt": minor ---- - -Add streaming JSON deep reads on LoroDoc and all container classes. getDeepValueJson() -returns plain JSON; getDeepValueJsonWithIds() returns {json, cids, containerPositions}, -where containerPositions is an owned Uint32Array indexing every JSON value in -JavaScript pre-order traversal. This distinguishes scalars from containers, preserves -ordinary cid/value objects and handles numeric object keys correctly. The unmerged -preview's shape-based {json,cids} reconstruction must be replaced by position lookup. -Tree metadata remains plain deep data, matching getDeepValueWithID. Binary values -serialize as JSON arrays; document reads obey empty/deleted-root visibility. diff --git a/.changeset/wasm-read-state.md b/.changeset/wasm-read-state.md new file mode 100644 index 000000000..d52acd233 --- /dev/null +++ b/.changeset/wasm-read-state.md @@ -0,0 +1,5 @@ +--- +"loro-crdt": minor +--- + +Add `LoroDoc.readState()` for reading nested container snapshots with explicit `type`, `cid`, and `value` fields. Ordinary values are opaque `Value` nodes, so Map/List data cannot be confused with containers. The same API reads an individual container or a clamped list interval. Text can return plain strings or formatting deltas; Tree node metadata preserves its Map ID and nested containers. JavaScript values are built directly with fixed constructors, per-read key/peer reuse, and owned binary buffers. diff --git a/AGENTS.md b/AGENTS.md index 3367b73b9..d1e61fb9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,9 +36,6 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. [context/wasm-error-reporting.md](context/wasm-error-reporting.md). - WASM container id wrapper identity, lazy caching, and benchmark: [context/wasm-container-id-cache.md](context/wasm-container-id-cache.md). -- WASM bulk-read APIs (per-container/range deep reads, JSON text export, cid - format, cids pre-order contract): - [context/wasm-bulk-read.md](context/wasm-bulk-read.md). - Bounded decoded-value cache in `InnerStore` (second-chance FIFO, eviction safety contract, loro-dev/loro#1092): [context/container-value-cache.md](context/container-value-cache.md). @@ -100,3 +97,5 @@ Use narrow checks first. Ask before broad fuzzing or long browser matrices. History uses short imperative commits, often prefixed by `fix:`, `test:`, `chore:`, or `refactor:`. PRs should include summary, rationale, validation, and linked issues or traces when relevant. +- Structured WASM reads, node identity, ranges and performance responsibilities: + [context/wasm-bulk-read.md](context/wasm-bulk-read.md). diff --git a/context/wasm-bulk-read.md b/context/wasm-bulk-read.md index 903abd3c9..37e4faff0 100644 --- a/context/wasm-bulk-read.md +++ b/context/wasm-bulk-read.md @@ -1,154 +1,101 @@ -# WASM bulk reads and the performance stack +# Structured WASM reads and the performance stack Verified against code 2026-09-05. -## Which cost each change removes - -- #1093 bounds the decoded-value cache used by individual handles. This fixes - retained WASM memory growth even for callers that keep their existing reads. -- #1085 caches wrapper `kind()`; #1086 adds subtree/range deep reads. These - reduce per-field JS/WASM calls. A visible history window should use range - reads rather than read an entire document just because a bulk API exists. -- #1087 serializes deep values in Rust and returns JSON text. Both JSON APIs - stream ephemeral container values into the output, without first building a - deep `LoroValue` tree or a `serde_json::Value` tree. The ID variant adds a - sparse position index. Tree metadata still uses its existing deep-value - conversion; the streaming improvement primarily targets Map/List/Text reads. Consumers can build their projection/registry in one - JS walk, without fetching each container again. -- #1090 defines the causal boundary for shallow snapshots. #1091 reduces the - cost of constructing their root state, with replay cost limits. These are - history/bootstrap/export changes, independent of the JSON read format. - -A faster bulk read does not automatically accelerate `new Mirror`: its caller -must adopt it while preserving schema decoding, ignored fields, container -registration, tree normalization and subscription behavior. The benchmark's -projection/registry cases model that read work, not the full Mirror constructor. - -## APIs - -- `getDeepValueWithID()` on documents and Map/List/MovableList/Tree/Text returns - `{ cid, value }` nodes for containers, with bare `ContainerID` strings. -- List/MovableList `getRangeDeepValueWithID(start, end)` and - `getRangeValue(start, end)` read a clamped `[start, end)` slice in one call. -- `getDeepValueJson(): string` returns the plain deep value as JSON text. -- `getDeepValueJsonWithIds(): DeepValueJsonWithIds` returns: +## Responsibility -```ts -type DeepValueJsonWithIds = { - json: string; - cids: ContainerID[]; - containerPositions: Uint32Array; -}; -``` - -These JSON APIs exist on the document and all six container classes. Detached -containers throw, except Counter, which supports detached value reads. JSON -follows Rust's JSON serialization of the deep value: binary becomes an array, -non-finite numbers become null, and arbitrary plain objects stay ordinary data. -Document JSON obeys empty/deleted-root visibility. The older structured -`getDeepValueWithID()` does not apply those display filters. - -## Sparse position contract +- #1093 bounds retained decoded container state; it benefits existing handle reads. +- #1085 caches wrapper kind; #1086 supplies legacy subtree/range reads with IDs. +- #1087 adds `LoroDoc.readState` to construct a consumer-ready JS snapshot directly. +- #1090/#1091 concern shallow snapshot causal boundaries and constructing their + root state. They optimize history import/export, independently of this read API. -`cids[i]` identifies the value at `containerPositions[i]` in a **pre-order walk -of every value** in `JSON.parse(json)`: +## Contract -- Start at zero. Count each object, array and scalar once; do not count keys. -- Visit arrays by increasing index and objects in `Object.keys()` order. -- Binary is a JSON array: its byte values count too. -- A document's root object counts as zero, but is not a container. -- A container-level result marks position zero with its own cid. -- Tree metadata is plain deep data, matching the existing with-id API: count - those JSON values, but do not assign them additional cids. -- Positions increase strictly and have the same length as `cids`. The typed - array owns a copied buffer and survives subsequent WASM calls and doc.free(). - -For example, these values are distinct even when both fields contain `"same"`: - -```text -JSON: {"m":{"a":"same","b":"same"}} -walk: 0 1 2 3 - -Text at a: cids = [mapId, textId], positions = [1, 2] -Text at b: cids = [mapId, textId], positions = [1, 3] +```ts +const roots = doc.readState(); +const subtree = doc.readState({container: map.id}); +const window = doc.readState({container: list.id, range: {start: 20, end: 40}}); +const formatted = doc.readState({container: text.id, text: "delta"}); ``` -The writer discovers identity from actual container edges, including mergeable -markers at map edges. It never strips objects that happen to contain `cid` and -`value`, nor guesses Text/Counter identity from a scalar type. It orders integer -property keys as JavaScript does (`"2"` before `"10"`; `"01"` and `"4294967295"` -are ordinary keys), independent of serde_json's `preserve_order` feature. - -A consumer can wrap known positions in `{cid, value}`, stamp map `$cid` fields, -or register identities directly during its own projection walk. Do not repeat -handle lookups to recover identities that are already present in the result. -The tests and benchmark contain complete reattachment examples; mutation of an -existing JSON.parse object preserves own `__proto__` keys without invoking the -inherited prototype setter. - -## Why positions instead of full paths - -For C containers at average depth D, full paths duplicate O(C*D) key/index -segments and require one path array per container. The sparse sidecar is O(C), -exactly 4*C bytes plus one JS typed-array allocation; it reuses the strings -already present in JSON. It also handles arbitrary keys without path escaping. - -The tradeoff is traversal: positions require visiting all JSON values, including -plain data. Paths allow direct navigation to each container and can have a faster -JS-only attachment step, especially with large plain subtrees. They still cost -path construction, serialization/transfer and extra allocations. Benchmark both -sides of this tradeoff instead of inferring speed from metadata bytes alone. - -The previous `{json,cids}` API in the unmerged PR did not record positions and -could not recover mixed primitive/container layouts. Consumers of that preview -must use the new index; the old shape-based reattachment is not compatible. - -## Reproducible measurements - -Run `pnpm release-wasm`, then `pnpm bench-deep-value-json`. The benchmark uses a -synthetic 70,051-container fixture (15,632 Map / 9,956 List / 44,463 Text) and -validates output before reporting results. No user document is stored. - -Each read case runs in a separate process with a newly imported snapshot. It -reports cold time, warm median (2 warm-ups + 5 measured rounds), and external, -heap and RSS deltas while retaining the first result. External memory includes -WASM linear-memory growth and other ArrayBuffers; it is not an exact Rust -allocation peak. The handle/indexed projection cases both stamp map identities -and register all containers. Path/position attachment timings isolate only JS -consumption, excluding path production/transfer; path payload size is also shown. - -Set `LORO_BENCH_MODULE=/absolute/path/to/nodejs/index.js` to run the same fixture -against another release build. A baseline without position support runs the -existing API cases only. Plain JSON and identity-preserving reads are separate -contracts and must not be advertised as interchangeable speed comparisons. - -## Measurement on this revision - -Release WASM, Node 22.23.1, macOS arm64, 2026-09-05; one run of the command above. Timings -are machine-dependent. The two identity-preserving rows reconstruct the same -with-id value; the two projection rows produce the same plain value and cid -registry (without Mirror schema/lifecycle work). - -| Read | Cold ms | Warm median ms | External delta MiB | -| --- | ---: | ---: | ---: | -| Structured getDeepValueWithID | 279.5 | 236.4 | 41.19 | -| Indexed JSON + parse + reattach | 248.1 | 204.4 | 19.39 | -| Per-handle projection + registry | 315.5 | 270.3 | 17.88 | -| Indexed JSON projection + registry | 249.1 | 207.2 | 19.13 | -| Plain streaming JSON + parse | 196.1 | 163.2 | 18.00 | - -Position metadata: 280,204 bytes. Full-path JSON metadata for the same -containers: 3,584,299 bytes (716,208 repeated key/index segments), excluding -cid strings common to both designs. Prebuilt-sidecar JS parse/attachment took -23.2 ms with positions vs 12.5 ms with paths. The latter excludes generating, -copying and decoding path metadata, so it is not an end-to-end path benchmark. -The position design trades a modest full-value JS walk for a 12.8x smaller -location payload and no per-container path arrays. It is not a claimed 5x -end-to-end speedup. - -The same benchmark against the original PR head (`7e99105a`, using -`LORO_BENCH_MODULE`) measured its legacy ID JSON producer at 365.3 ms cold / -234.3 ms warm and 96.06 MiB external growth. The streaming indexed producer -measured 226.8 ms / 174.3 ms and 19.39 MiB. These producer-only rows exclude -parse/reattachment, and the legacy result cannot represent all container layouts; -they are separate from the valid identity-preserving consumer comparison above. +Containers are `{type, cid, value}`. Map values and List/MovableList items are +nodes. An ordinary value is `{type: "Value", value}`: its contents are opaque, +including plain objects shaped exactly like container nodes. Text contains a +string by default, or its formatting delta when requested. Tree contains the +existing nested tree layout, but each `meta` is an ID-bearing Map node; its +fields follow the same node contract. Counter contains a number. + +Identity is emitted only at real CRDT edges, including mergeable Map markers +and Tree metadata edges. No keys, scalar values, paths, traversal positions, or +user object shapes are used to guess identity. Full-document reads obey root +visibility. Explicit root IDs can read empty implicit roots. Missing normal or +mergeable containers and unknown container types return errors. Reads do not +commit; caller mutations of returned objects/buffers do not mutate the document. + +Ranges require List/MovableList container IDs and nonnegative u32 integer bounds; +end is exclusive, bounds clamp, inverted ranges are empty. Only selected child +subtrees are traversed, though obtaining the parent shallow list is still O(N). +Read traversal rejects nesting above 256 levels. JS number conversion follows +existing reads (including i64 rounding, NaN and infinity); Binary is Uint8Array. +Own `__proto__` and integer keys are ordinary data properties and do not invoke +inherited setters. Consumers must preserve this when projecting nodes themselves. + +## Implementation + +`state/read_state.rs` emits a traversal to a sink without constructing a deep +whole-document LoroValue tree. Each container's shallow value is ephemeral; +values fetched to determine root visibility are reused. `loro-wasm/src/read_state.rs` +owns the JS construction stack and uses fixed imported functions for stable +wrapper shapes, IDs and own-property writes. Keys and peer decimal strings are +cached only for one read; complete CIDs are constructed in JS. Binary buffers +are copied once into an owned JS Uint8Array. There is no public transport format, +codec, path index, JS callback, or document-lifetime cache. + +Inductive correctness: each ordinary edge emits one opaque Value node preserving +its value; each container edge emits its actual ID and the recursively transformed +children; the document enumerates exactly its visible roots. Thus projecting +wrappers away preserves visible values and walking only node children enumerates +container occurrences without confusing embedded Map/List data. The proof relies +on the existing shallow-value/mergeable-edge semantics. Tests separately check +Tree metadata, deltas, binary ownership, special keys, IDs, ranges, errors and +root visibility; this is a design argument, not a machine-checked proof. + +## Mirror integration + +Mirror can consume the node tree in its existing schema/registry walk. On a Value +node it decodes the opaque value; on a container node it registers `cid` and +recurses by `type`. This removes shallow reads previously needed to distinguish +legacy `{cid,value}` nodes from user objects. Schema decoding, Ignore fields, +lazy hydration, Tree normalization, and incremental events remain Mirror's work. +A bulk API should not force eager traversal of lazy history: use subtree/range +reads where the schema allows it. Legacy `getDeepValueWithID` remains unchanged. + +## Measured end-to-end cost + +Final Node 22.23.1 and Chromium 152 comparisons used the same newly built Loro +package and Mirror adapter, an imported local 61.85 MiB snapshot, 10,921 reachable +containers (10,925 after schema-created roots), and the actual application schema. +Each fresh process/page measured its first constructor separately, then two warmups +and five samples; cases ran forward and backward. Import was outside the timed +constructor. Full state and sorted registered IDs agreed across methods. Private +snapshots/transcripts are not fixtures and are not committed. + +| Full Mirror initialization | Node warm median | Chromium warm median | +| --- | --- | --- | +| Per-container handles | 64.5–66.3 ms | 58.8–59.4 ms | +| Legacy bulk + shallow disambiguation | 79.4–80.5 ms | 73.0–74.2 ms | +| Fixed constructors + explicit nodes | 51.9–53.1 ms | 47.2–47.3 ms | + +First constructor: new path 80–83 ms in Node, 71–73 ms in Chromium; handle path +89–91 ms and 85–88 ms respectively. JS retained heap was similar (about 37 MiB); +reuse of already allocated WASM memory must not be described as zero memory cost. +A synthetic 70,051-container sample measured about 256 ms versus 321 ms for legacy +bulk; such small-string workloads do not predict the ranking of string transports +on the large-text application document. + +Moving the entire builder stack to JS did not consistently win and was rejected. +Pre-creating dense array slots likewise gave no material improvement. The shipped +builder retains fixed wrapper constructors, per-read peer/key reuse, and safe own +property writes. These timings describe the tested machines/workload, not a general +speed guarantee. Mirror's Tree normalization still uses its existing handle path. diff --git a/crates/loro-internal/src/handler.rs b/crates/loro-internal/src/handler.rs index 628f1c9e3..402b3d366 100644 --- a/crates/loro-internal/src/handler.rs +++ b/crates/loro-internal/src/handler.rs @@ -2698,30 +2698,6 @@ impl TextHandler { })) } - /// Get the deep value of the text (its content string) as JSON text. - /// - /// The content is identical to serializing the deep value, but the JSON - /// text is produced in one pass. - pub fn get_deep_value_json(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| { - state.get_container_deep_value_json_with_ids(inner.container_idx) - }) - } - pub fn get_cursor(&self, event_index: usize, side: Side) -> Option { self.get_cursor_internal(event_index, side, true) } @@ -3392,30 +3368,6 @@ impl ListHandler { })) } - /// Get the deep value of this list as JSON text. - /// - /// The content is identical to serializing the deep value, but the JSON - /// text is produced in one pass. - pub fn get_deep_value_json(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| { - state.get_container_deep_value_json_with_ids(inner.container_idx) - }) - } - /// Get the deep value of the elements in the range `[start, end)`. /// /// Child containers in the range are recursively resolved to `{ cid, value }` @@ -4127,30 +4079,6 @@ impl MovableListHandler { })) } - /// Get the deep value of this list as JSON text. - /// - /// The content is identical to serializing the deep value, but the JSON - /// text is produced in one pass. - pub fn get_deep_value_json(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| { - state.get_container_deep_value_json_with_ids(inner.container_idx) - }) - } - /// Get the deep value of the elements in the range `[start, end)`. /// /// Child containers in the range are recursively resolved to `{ cid, value }` @@ -4590,30 +4518,6 @@ impl MapHandler { } } - /// Get the deep value of the map as JSON text. - /// - /// The content is identical to serializing the deep value, but the JSON - /// text is produced in one pass. - pub fn get_deep_value_json(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| { - state.get_container_deep_value_json_with_ids(inner.container_idx) - }) - } - pub fn get(&self, key: &str) -> Option { match &self.inner { MaybeDetached::Detached(m) => { @@ -5002,30 +4906,6 @@ pub mod counter { pub fn clear(&self) -> LoroResult<()> { self.decrement(self.get_value().into_double().unwrap()) } - - /// Get the counter value as JSON text (a JSON number). - /// - /// Unlike the other container types this also works on a detached - /// counter, mirroring `get_value`. - pub fn get_deep_value_json(&self) -> LoroResult { - crate::state::deep_value_to_json(&self.get_value()) - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult { - Ok(crate::DeepValueJsonWithIds { - json: self.get_deep_value_json()?, - cids: vec![self.id().to_string()], - container_positions: vec![0], - }) - } } impl std::fmt::Debug for CounterHandler { diff --git a/crates/loro-internal/src/handler/tree.rs b/crates/loro-internal/src/handler/tree.rs index dd24d68fa..52fad6a31 100644 --- a/crates/loro-internal/src/handler/tree.rs +++ b/crates/loro-internal/src/handler/tree.rs @@ -363,30 +363,6 @@ impl TreeHandler { })) } - /// Get the deep value of the tree as JSON text. - /// - /// The content is identical to serializing the deep value, but the JSON - /// text is produced in one pass. - pub fn get_deep_value_json(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| state.get_container_deep_value_json(inner.container_idx)) - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - pub fn get_deep_value_json_with_ids(&self) -> LoroResult { - let inner = self.inner.try_attached_state()?; - inner.with_doc_state(|state| { - state.get_container_deep_value_json_with_ids(inner.container_idx) - }) - } - pub fn delete(&self, target: TreeID) -> LoroResult<()> { match &self.inner { MaybeDetached::Detached(t) => { diff --git a/crates/loro-internal/src/lib.rs b/crates/loro-internal/src/lib.rs index 4691661ce..b27a069bc 100644 --- a/crates/loro-internal/src/lib.rs +++ b/crates/loro-internal/src/lib.rs @@ -14,6 +14,8 @@ pub mod diff; pub mod diff_calc; pub mod handler; pub mod sync; +pub use state::read_state; + use crate::sync::{AtomicBool, AtomicUsize}; use std::sync::Arc; mod change_meta; @@ -36,7 +38,7 @@ use pre_commit::{ PreCommitCallbackPayload, }; pub use rustc_hash::FxHashMap; -pub use state::{DeepValueJsonWithIds, DocState}; +pub use state::DocState; pub use state::{TreeNode, TreeNodeWithChildren, TreeParentId}; use subscription::{LocalUpdateCallback, Observer, PeerIdUpdateCallback}; use txn::Transaction; diff --git a/crates/loro-internal/src/loro.rs b/crates/loro-internal/src/loro.rs index 8b08ef7b2..6c4c532d4 100644 --- a/crates/loro-internal/src/loro.rs +++ b/crates/loro-internal/src/loro.rs @@ -1609,28 +1609,6 @@ impl LoroDoc { self.state.lock().get_deep_value_with_id() } - /// JSON text of the document's deep value. - /// - /// The content is identical to serializing [`Self::get_deep_value`], but - /// the JSON text is produced in one pass. - #[inline] - pub fn get_deep_value_json(&self) -> LoroResult { - self.state.lock().get_deep_value_json() - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - #[inline] - pub fn get_deep_value_json_with_ids(&self) -> LoroResult { - self.state.lock().get_deep_value_json_with_ids() - } - pub fn checkout_to_latest(&self) { let (options, _guard) = self.implicit_commit_then_stop(); if !self.is_detached() { diff --git a/crates/loro-internal/src/state.rs b/crates/loro-internal/src/state.rs index d045b067f..9fcbd35e4 100644 --- a/crates/loro-internal/src/state.rs +++ b/crates/loro-internal/src/state.rs @@ -1,3 +1,5 @@ +pub mod read_state; + use crate::sync::{AtomicU64, Mutex, RwLock}; #[cfg(test)] use std::cell::Cell; @@ -35,8 +37,6 @@ pub(crate) mod container_store; #[cfg(feature = "counter")] mod counter_state; mod dead_containers_cache; -mod deep_value_json; -pub use deep_value_json::DeepValueJsonWithIds; mod list_state; mod map_state; mod mergeable; @@ -96,13 +96,6 @@ fn state_decode_error(message: impl Into>) -> LoroError { LoroError::DecodeError(message.into()) } -/// Serialize a scalar counter value to JSON text. -#[cfg(feature = "counter")] -pub(crate) fn deep_value_to_json(value: &LoroValue) -> LoroResult { - serde_json::to_string(value) - .map_err(|e| state_decode_error(format!("Failed to serialize deep value to JSON: {e}"))) -} - fn decode_peer_table(bytes: &mut &[u8], context: &str) -> LoroResult> { let peer_num = leb128::read::unsigned(bytes) .map_err(|_| state_decode_error(format!("{context}: invalid peer table length")))?; @@ -1364,27 +1357,6 @@ impl DocState { LoroValue::Map(ans.into()) } - /// JSON text of [`Self::get_deep_value`]. - /// - /// The content is identical to serializing `get_deep_value()`, but the JSON - /// text is produced in one pass so callers (e.g. the WASM bindings) can - /// avoid a structured-clone round trip. - pub fn get_deep_value_json(&mut self) -> LoroResult { - Ok(self.write_deep_value_json(false)?.json) - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - pub fn get_deep_value_json_with_ids(&mut self) -> LoroResult { - self.write_deep_value_json(true) - } - pub(crate) fn preferred_root_containers(&mut self) -> Vec { let flag = self.store.load_root_containers(); // Mergeable cids live in a private namespace and are logically children of a regular @@ -1586,29 +1558,6 @@ impl DocState { } } - /// JSON text of [`Self::get_container_deep_value`]. - pub(crate) fn get_container_deep_value_json( - &mut self, - container: ContainerIdx, - ) -> LoroResult { - Ok(self.write_container_deep_value_json(container, false)?.json) - } - - /// Read plain JSON with a sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `container_positions[i]`. - /// Count every JSON value in pre-order, starting at zero, using JavaScript - /// `Object.keys` order for objects. Plain data never acquires a container id. - /// A document object counts as value zero but has no id; a container-level - /// result marks position zero. Tree metadata is plain deep data, as in - /// `get_deep_value_with_id`. Map/list edges stream directly without intermediate deep trees. - pub(crate) fn get_container_deep_value_json_with_ids( - &mut self, - container: ContainerIdx, - ) -> LoroResult { - self.write_container_deep_value_json(container, true) - } - pub fn get_container_deep_value(&mut self, container: ContainerIdx) -> LoroValue { let Some(value) = self.store.get_value_ephemeral(container) else { return container.get_type().default_value(); diff --git a/crates/loro-internal/src/state/AGENTS.md b/crates/loro-internal/src/state/AGENTS.md index 6f34e8af5..e79a0a209 100644 --- a/crates/loro-internal/src/state/AGENTS.md +++ b/crates/loro-internal/src/state/AGENTS.md @@ -15,10 +15,6 @@ before changing mergeable child behavior. bounded and evicted wrappers must stay re-creatable from KV; read [../../../../context/container-value-cache.md](../../../../context/container-value-cache.md) before changing read/caching paths there. -- `deep_value_json.rs`: streaming JSON reads and sparse container positions. - Identity comes from CRDT edges; positions count every JSON value in JavaScript - property order. Never infer containers from scalar types or cid/value objects. - See [../../../../context/wasm-bulk-read.md](../../../../context/wasm-bulk-read.md). - `map_state.rs`, `list_state.rs`, `richtext_state.rs`, `tree_state.rs`, `movable_list_state.rs`, `counter_state.rs`: per-container state and snapshot codecs. `richtext_state.rs` also hosts `redact_dead_style_values`, used by @@ -48,3 +44,7 @@ before changing mergeable child behavior. - `cargo test -p loro-internal --test mergeable_cid_encoding` - `cargo test -p loro-internal --test mergeable_container` - `cargo test -p loro-internal import_atomicity` if import or rollback is involved. + +`read_state.rs` traverses ephemeral shallow values into a sink. Container identity +comes from CRDT edges, including mergeable markers and Tree metadata; ordinary +values remain opaque. See [bulk reads](../../../../context/wasm-bulk-read.md). diff --git a/crates/loro-internal/src/state/deep_value_json.rs b/crates/loro-internal/src/state/deep_value_json.rs deleted file mode 100644 index 9e89253b0..000000000 --- a/crates/loro-internal/src/state/deep_value_json.rs +++ /dev/null @@ -1,233 +0,0 @@ -//! Streaming bulk JSON reads. Container identity comes from CRDT edges, never -//! from the shape of user values. The sparse sidecar indexes *all* JSON values. -use super::{ - deleted_root_container_value_is_cleared, get_meta_value, state_decode_error, - visible_container_value_is_empty, DocState, -}; -use crate::{container::idx::ContainerIdx, ContainerType, LoroValue}; -use loro_common::{ContainerID, LoroResult}; -use std::sync::atomic::Ordering; - -/// Plain JSON and a sparse index of container positions in its parsed value. -#[derive(Debug)] -pub struct DeepValueJsonWithIds { - pub json: String, - pub cids: Vec, - /// Zero-based pre-order positions: count every JSON value, including the - /// document object, scalars, plain objects/arrays, and binary array items. - /// Object children follow JavaScript's `Object.keys` order. - pub container_positions: Vec, -} - -#[derive(Default)] -struct JsonWriter { - bytes: Vec, - cids: Vec, - positions: Vec, - next: u32, - with_ids: bool, -} - -impl JsonWriter { - fn count(&mut self, n: u32) -> LoroResult<()> { - if !self.with_ids { - return Ok(()); - } - self.next = self.next.checked_add(n).ok_or_else(|| { - state_decode_error("Bulk JSON has too many values for its container position index") - })?; - Ok(()) - } - - fn json(&mut self, value: &impl serde::Serialize) -> LoroResult<()> { - serde_json::to_writer(&mut self.bytes, value) - .map_err(|e| state_decode_error(format!("Failed to serialize deep value: {e}"))) - } - - fn finish(self) -> LoroResult { - Ok(DeepValueJsonWithIds { - json: String::from_utf8(self.bytes) - .map_err(|e| state_decode_error(format!("Invalid bulk JSON UTF-8: {e}")))?, - cids: self.cids, - container_positions: self.positions, - }) - } - - fn container(&mut self, state: &mut DocState, idx: ContainerIdx) -> LoroResult<()> { - let id = state.arena.idx_to_id(idx).unwrap(); - let value = state - .store - .get_value_ephemeral(idx) - .unwrap_or_else(|| idx.get_type().default_value()); - self.container_value(state, &id, value) - } - - fn container_value( - &mut self, - state: &mut DocState, - id: &ContainerID, - mut value: LoroValue, - ) -> LoroResult<()> { - if self.with_ids { - self.positions.push(self.next); - self.cids.push(id.to_string()); - } - // Match getDeepValueWithID: tree metadata is plain deep data, not - // additional with-id nodes. Its JSON values still count as positions. - if id.container_type() == ContainerType::Tree { - if let LoroValue::List(list) = &mut value { - get_meta_value(list.make_mut(), state); - } - return self.value(state, &value, None); - } - self.value(state, &value, Some(id)) - } - - fn child(&mut self, state: &mut DocState, value: &LoroValue, resolve: bool) -> LoroResult<()> { - if resolve { - if let LoroValue::Container(id) = value { - let idx = state.arena.register_container(id); - return self.container(state, idx); - } - } - self.value(state, value, None) - } - - fn value( - &mut self, - state: &mut DocState, - value: &LoroValue, - parent: Option<&ContainerID>, - ) -> LoroResult<()> { - self.count(1)?; - match value { - LoroValue::Map(map) => { - let mut entries: Vec<_> = map.iter().collect(); - entries.sort_unstable_by(|(a, _), (b, _)| json_key_cmp(a, b)); - self.bytes.push(b'{'); - for (i, (key, value)) in entries.into_iter().enumerate() { - if i > 0 { - self.bytes.push(b','); - } - self.json(key)?; - self.bytes.push(b':'); - // Resolve compact mergeable markers only at actual map - // edges. Identical bytes in plain user data stay data. - let mergeable = parent.and_then(|id| { - loro_common::parse_mergeable_marker(id, key, value) - .map(|kind| ContainerID::new_mergeable(id, key, kind)) - }); - if let Some(id) = mergeable { - let idx = state.arena.register_container(&id); - self.container(state, idx)?; - } else { - self.child(state, value, parent.is_some())?; - } - } - self.bytes.push(b'}'); - } - LoroValue::List(list) => { - self.bytes.push(b'['); - for (i, value) in list.iter().enumerate() { - if i > 0 { - self.bytes.push(b','); - } - self.child(state, value, parent.is_some())?; - } - self.bytes.push(b']'); - } - LoroValue::Binary(bytes) => { - self.count( - u32::try_from(bytes.len()) - .map_err(|_| state_decode_error("Bulk JSON binary is too large"))?, - )?; - self.json(value)?; - } - _ => self.json(value)?, - } - Ok(()) - } -} - -// ECMA-262 array-index property keys precede other keys in Object.keys, even -// after JSON.parse. 2^32-1, leading-zero spellings and negative keys are NOT -// array indices. Sorting other keys gives deterministic output on every target. -fn array_index(key: &str) -> Option { - if key.is_empty() - || key.len() > 10 - || (key.len() > 1 && key.starts_with('0')) - || !key.bytes().all(|b| b.is_ascii_digit()) - { - return None; - } - key.parse::().ok().filter(|&n| n != u32::MAX) -} - -fn json_key_cmp(a: &str, b: &str) -> std::cmp::Ordering { - match (array_index(a), array_index(b)) { - (Some(a), Some(b)) => a.cmp(&b), - (Some(_), None) => std::cmp::Ordering::Less, - (None, Some(_)) => std::cmp::Ordering::Greater, - (None, None) => a.cmp(b), - } -} - -impl DocState { - pub(super) fn write_deep_value_json( - &mut self, - with_ids: bool, - ) -> LoroResult { - let mut writer = JsonWriter { - with_ids, - ..Default::default() - }; - writer.count(1)?; // The document object is value zero, not a container. - writer.bytes.push(b'{'); - let mut roots: Vec<_> = self - .preferred_root_containers() - .into_iter() - .map(|idx| (self.root_container_name(idx).unwrap(), idx)) - .collect(); - roots.sort_unstable_by(|(a, _), (b, _)| json_key_cmp(a, b)); - let hide_empty = self - .config - .hide_empty_root_containers - .load(Ordering::Relaxed); - let mut first = true; - for (key, idx) in roots { - let id = self.arena.idx_to_id(idx).unwrap(); - let value = self - .store - .get_value_ephemeral(idx) - .unwrap_or_else(|| idx.get_type().default_value()); - if (hide_empty && visible_container_value_is_empty(idx.get_type(), &value)) - || (self.config.deleted_root_containers.lock().contains(&id) - && deleted_root_container_value_is_cleared(idx.get_type(), &value)) - { - continue; - } - if !first { - writer.bytes.push(b','); - } - first = false; - writer.json(&key)?; - writer.bytes.push(b':'); - writer.container_value(self, &id, value)?; - } - writer.bytes.push(b'}'); - writer.finish() - } - - pub(super) fn write_container_deep_value_json( - &mut self, - idx: ContainerIdx, - with_ids: bool, - ) -> LoroResult { - let mut writer = JsonWriter { - with_ids, - ..Default::default() - }; - writer.container(self, idx)?; - writer.finish() - } -} diff --git a/crates/loro-internal/src/state/read_state.rs b/crates/loro-internal/src/state/read_state.rs new file mode 100644 index 000000000..88edebf5f --- /dev/null +++ b/crates/loro-internal/src/state/read_state.rs @@ -0,0 +1,244 @@ +//! Structured state traversal. Container identity comes from CRDT edges, never value shape. +//! A sink constructs the target representation without a whole-document intermediate tree. +use super::{deleted_root_container_value_is_cleared, visible_container_value_is_empty, DocState}; +use crate::{container::idx::ContainerIdx, ContainerType, LoroValue}; +use loro_common::{ContainerID, LoroError, LoroResult}; +use std::sync::atomic::Ordering; +#[derive(Debug)] +pub enum Event<'a> { + Object(usize), + Array(usize), + End, + Key(&'a str), + String(&'a str), + Number(f64), + Bool(bool), + Null, + Binary(&'a [u8]), +} +pub trait Sink { + fn emit(&mut self, e: Event<'_>) -> LoroResult<()>; + fn container(&mut self, id: &ContainerID) -> LoroResult<()>; + fn container_end(&mut self) -> LoroResult<()> { + self.emit(Event::End) + } + fn value_start(&mut self) -> LoroResult<()>; + fn value_end(&mut self) -> LoroResult<()>; +} +impl DocState { + pub fn read_state( + &mut self, + s: &mut S, + cid: Option<&ContainerID>, + rich: bool, + range: Option<(usize, usize)>, + ) -> LoroResult<()> { + if let Some(id) = cid { + let idx = self.arena.register_container(id); + if range.is_some() + && !matches!( + idx.get_type(), + ContainerType::List | ContainerType::MovableList + ) + { + return Err(err("readState range requires a List or MovableList")); + } + return self.read_state_container(s, idx, rich, range, None, 0); + } + if range.is_some() { + return Err(err("readState range requires a container")); + } + let roots = self.preferred_root_containers(); + let mut visible = Vec::new(); + for idx in roots { + if matches!(idx.get_type(), ContainerType::Unknown(_)) { + return Err(err("Unsupported container type")); + } + let id = self + .arena + .idx_to_id(idx) + .ok_or_else(|| err("Missing container ID"))?; + let hidden = self + .config + .hide_empty_root_containers + .load(Ordering::Relaxed); + let deleted = self.config.deleted_root_containers.lock().contains(&id); + let v = if hidden || deleted { + self.store + .try_get_value_ephemeral(idx)? + .or_else(|| Some(idx.get_type().default_value())) + } else { + None + }; + if v.as_ref().is_some_and(|v| { + (hidden && visible_container_value_is_empty(idx.get_type(), v)) + || (deleted && deleted_root_container_value_is_cleared(idx.get_type(), v)) + }) { + continue; + } + visible.push(( + self.root_container_name(idx) + .ok_or_else(|| err("Missing root name"))?, + idx, + v, + )); + } + s.emit(Event::Object(visible.len()))?; + for (key, idx, value) in visible { + s.emit(Event::Key(&key))?; + self.read_state_container(s, idx, rich, None, value, 0)?; + } + s.emit(Event::End) + } + fn read_state_container( + &mut self, + s: &mut S, + idx: ContainerIdx, + rich: bool, + range: Option<(usize, usize)>, + value: Option, + depth: usize, + ) -> LoroResult<()> { + check_depth(depth)?; + let id = self + .arena + .idx_to_id(idx) + .ok_or_else(|| err("Missing container ID"))?; + let kind = idx.get_type(); + if matches!(kind, ContainerType::Unknown(_)) { + return Err(err("Unsupported container type")); + } + let v = if rich && kind == ContainerType::Text { + self.store + .get_or_create_mut(idx) + .as_richtext_state_mut() + .ok_or_else(|| err("Missing text state"))? + .get_richtext_value() + } else { + match value { + Some(value) => value, + None => self + .store + .try_get_value_ephemeral(idx)? + .unwrap_or_else(|| kind.default_value()), + } + }; + s.container(&id)?; + + match (&v, kind) { + (LoroValue::Map(m), ContainerType::Map) => { + s.emit(Event::Object(m.len()))?; + for (k, v) in m.iter() { + s.emit(Event::Key(k))?; + let merge = loro_common::parse_mergeable_marker(&id, k, v) + .map(|t| ContainerID::new_mergeable(&id, k, t)); + if let Some(c) = merge { + let i = self.arena.register_container(&c); + self.read_state_container(s, i, rich, None, None, depth + 1)?; + } else { + self.read_state_edge(s, v, rich, depth + 1)?; + } + } + s.emit(Event::End)?; + } + (LoroValue::List(l), ContainerType::List | ContainerType::MovableList) => { + let (start, end) = range.unwrap_or((0, l.len())); + let start = start.min(l.len()); + let end = end.min(l.len()).max(start); + s.emit(Event::Array(end - start))?; + for v in &l[start..end] { + self.read_state_edge(s, v, rich, depth + 1)?; + } + s.emit(Event::End)?; + } + (_, ContainerType::Tree) => self.read_state_tree(s, &v, rich, depth + 1)?, + _ => raw(s, &v, depth + 1)?, + }; + s.container_end() + } + fn read_state_edge( + &mut self, + s: &mut S, + v: &LoroValue, + rich: bool, + depth: usize, + ) -> LoroResult<()> { + if let LoroValue::Container(cid) = v { + let idx = self.arena.register_container(cid); + return self.read_state_container(s, idx, rich, None, None, depth); + } + s.value_start()?; + raw(s, v, depth + 1)?; + s.value_end() + } + fn read_state_tree( + &mut self, + s: &mut S, + v: &LoroValue, + rich: bool, + depth: usize, + ) -> LoroResult<()> { + check_depth(depth)?; + match v { + LoroValue::List(l) => { + s.emit(Event::Array(l.len()))?; + for node in l.iter() { + let m = node.as_map().ok_or_else(|| err("Invalid tree node"))?; + s.emit(Event::Object(m.len()))?; + for (k, v) in m.iter() { + s.emit(Event::Key(k))?; + if k == "meta" { + self.read_state_edge(s, v, rich, depth + 1)?; + } else if k == "children" { + self.read_state_tree(s, v, rich, depth + 1)?; + } else { + raw(s, v, depth + 1)?; + } + } + s.emit(Event::End)?; + } + s.emit(Event::End) + } + _ => Err(err("Invalid tree value")), + } + } +} +pub fn err(s: &str) -> LoroError { + LoroError::JsError(s.to_string().into_boxed_str()) +} +fn raw(s: &mut S, v: &LoroValue, depth: usize) -> LoroResult<()> { + check_depth(depth)?; + match v { + LoroValue::Null => s.emit(Event::Null), + LoroValue::Bool(b) => s.emit(Event::Bool(*b)), + // Match the existing JavaScript value conversion, including i64 -> number. + LoroValue::I64(n) => s.emit(Event::Number(*n as f64)), + LoroValue::Double(n) => s.emit(Event::Number(*n)), + LoroValue::Binary(v) => s.emit(Event::Binary(v)), + LoroValue::Container(id) => s.emit(Event::String(&id.to_string())), + LoroValue::String(v) => s.emit(Event::String(v)), + LoroValue::List(l) => { + s.emit(Event::Array(l.len()))?; + for v in l.iter() { + raw(s, v, depth + 1)?; + } + s.emit(Event::End) + } + LoroValue::Map(m) => { + s.emit(Event::Object(m.len()))?; + for (k, v) in m.iter() { + s.emit(Event::Key(k))?; + raw(s, v, depth + 1)?; + } + s.emit(Event::End) + } + } +} + +fn check_depth(depth: usize) -> LoroResult<()> { + if depth > 256 { + Err(err("readState nesting exceeds 256 levels")) + } else { + Ok(()) + } +} diff --git a/crates/loro-internal/tests/deep_value_json.rs b/crates/loro-internal/tests/deep_value_json.rs deleted file mode 100644 index 887299829..000000000 --- a/crates/loro-internal/tests/deep_value_json.rs +++ /dev/null @@ -1,233 +0,0 @@ -use loro_common::LoroResult; -use loro_internal::{ - handler::{ListHandler, TextHandler}, - HandlerTrait, LoroDoc, TreeParentId, -}; - -/// Build a doc whose root map contains a list containing a text, plus a tree -/// with meta and a root counter. -fn build_doc() -> LoroResult<(LoroDoc, Vec)> { - let doc = LoroDoc::new_auto_commit(); - let map = doc.get_map("map"); - map.insert("flag", true)?; - let list = map.insert_container("list", ListHandler::new_detached())?; - list.insert(0, "item")?; - let text = list.insert_container(1, TextHandler::new_detached())?; - text.insert_unicode(0, "Hello")?; - let tree = doc.get_tree("tree"); - let root = tree.create(TreeParentId::Root)?; - tree.get_meta(root)?.insert("name", "root")?; - #[cfg(feature = "counter")] - let counter = doc.get_counter("counter"); - #[cfg(feature = "counter")] - counter.increment(2.5)?; - doc.commit_then_renew(); - - // serde_json serializes maps in sorted key order (no `preserve_order` - // feature in this workspace), so the pre-order cids are deterministic: - // root keys sorted: counter < map < tree. Inside map: flag (plain) then - // list; inside list: "item" (plain) then text. - #[cfg(feature = "counter")] - let cids = vec![ - counter.id().to_string(), - map.id().to_string(), - list.id().to_string(), - text.id().to_string(), - tree.id().to_string(), - ]; - #[cfg(not(feature = "counter"))] - let cids = vec![ - map.id().to_string(), - list.id().to_string(), - text.id().to_string(), - tree.id().to_string(), - ]; - Ok((doc, cids)) -} - -#[test] -fn deep_value_json_matches_plain_deep_value_serialization() -> LoroResult<()> { - let (doc, _) = build_doc()?; - let expected = serde_json::to_string(&doc.get_deep_value()).unwrap(); - assert_eq!( - serde_json::from_str::(&doc.get_deep_value_json()?).unwrap(), - serde_json::from_str::(&expected).unwrap(), - ); - Ok(()) -} - -#[test] -fn deep_value_json_with_ids_doc_level() -> LoroResult<()> { - let (doc, expected_cids) = build_doc()?; - let loro_internal::DeepValueJsonWithIds { json, cids, .. } = - doc.get_deep_value_json_with_ids()?; - - // json parses to the same content as the plain deep value JSON (object - // key order may differ between the two strings; see the API docs) - assert_eq!( - serde_json::from_str::(&json).unwrap(), - serde_json::from_str::(&doc.get_deep_value_json()?).unwrap(), - ); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - serde_json::to_value(doc.get_deep_value()).unwrap(), - ); - - // cids are in pre-order DFS of the serialized tree - assert_eq!(cids, expected_cids); - - // tree meta maps are plain deep values: the meta container id does not - // appear in cids - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - let nodes = parsed["tree"].as_array().unwrap(); - assert_eq!(nodes.len(), 1); - assert_eq!(nodes[0]["meta"], serde_json::json!({ "name": "root" })); - #[cfg(feature = "counter")] - assert_eq!(parsed["counter"], serde_json::json!(2.5)); - Ok(()) -} - -#[test] -fn deep_value_json_with_ids_per_container() -> LoroResult<()> { - let (doc, _) = build_doc()?; - - let map = doc.get_map("map"); - let loro_internal::DeepValueJsonWithIds { json, cids, .. } = - map.get_deep_value_json_with_ids()?; - // cids[0] is the container's own id (pre-order includes the root container) - assert_eq!(cids[0], map.id().to_string()); - assert_eq!(cids.len(), 3, "map, list, text"); - // json equals the container's plain deep value - assert_eq!( - serde_json::from_str::(&json).unwrap(), - serde_json::to_value(map.get_deep_value()).unwrap(), - ); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - serde_json::from_str::(&map.get_deep_value_json()?).unwrap(), - ); - - let text = doc.get_text("text"); - text.insert_unicode(0, "abc")?; - let loro_internal::DeepValueJsonWithIds { json, cids, .. } = - text.get_deep_value_json_with_ids()?; - assert_eq!(json, "\"abc\""); - assert_eq!(cids, vec![text.id().to_string()]); - - let tree = doc.get_tree("tree"); - let loro_internal::DeepValueJsonWithIds { json, cids, .. } = - tree.get_deep_value_json_with_ids()?; - assert_eq!(cids, vec![tree.id().to_string()]); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - serde_json::to_value(tree.get_deep_value()).unwrap(), - ); - - #[cfg(feature = "counter")] - { - let counter = doc.get_counter("counter"); - let loro_internal::DeepValueJsonWithIds { json, cids, .. } = - counter.get_deep_value_json_with_ids()?; - assert_eq!(json, "2.5"); - assert_eq!(cids, vec![counter.id().to_string()]); - } - Ok(()) -} - -#[test] -fn deep_value_json_empty_doc() -> LoroResult<()> { - let doc = LoroDoc::new_auto_commit(); - assert_eq!(doc.get_deep_value_json()?, "{}"); - let loro_internal::DeepValueJsonWithIds { json, cids, .. } = - doc.get_deep_value_json_with_ids()?; - assert_eq!(json, "{}"); - assert!(cids.is_empty()); - Ok(()) -} - -#[test] -fn deep_value_json_detached_container_errors() { - let text = TextHandler::new_detached(); - assert!(text.get_deep_value_json().is_err()); - assert!(text.get_deep_value_json_with_ids().is_err()); - let list = ListHandler::new_detached(); - assert!(list.get_deep_value_json().is_err()); - assert!(list.get_deep_value_json_with_ids().is_err()); -} - -#[test] -fn positions_distinguish_identical_values_with_different_container_layouts() -> LoroResult<()> { - let build = |text_key: &str, plain_key: &str| -> LoroResult<_> { - let doc = LoroDoc::new_auto_commit(); - doc.set_peer_id(1)?; - let map = doc.get_map("m"); - map.insert_container(text_key, TextHandler::new_detached())? - .insert_unicode(0, "same")?; - map.insert(plain_key, "same")?; - doc.get_deep_value_json_with_ids() - }; - let a = build("a", "b")?; - let b = build("b", "a")?; - assert_eq!(a.json, b.json); - assert_eq!(a.cids, b.cids); - assert_eq!(a.container_positions, vec![1, 2]); - assert_eq!(b.container_positions, vec![1, 3]); - Ok(()) -} - -#[test] -fn positions_survive_numeric_keys_plain_lookalikes_and_snapshot_import() -> LoroResult<()> { - use loro_common::LoroValue; - use loro_internal::encoding::ExportMode; - let doc = LoroDoc::new_auto_commit(); - doc.set_peer_id(1)?; - let map = doc.get_map("m"); - // No shape-based stripping: all of this is ordinary user data. - map.insert("0", LoroValue::from(vec![1u8, 2, 3]))?; - let plain: LoroValue = - serde_json::from_str(r#"{"cid":"cid:root-fake:Map","value":{"n":1}}"#).unwrap(); - map.insert("1", plain)?; - map.insert_container("10", TextHandler::new_detached())? - .insert_unicode(0, "ten")?; - map.insert_container("2", TextHandler::new_detached())? - .insert_unicode(0, "two")?; - doc.commit_then_renew(); - let imported = LoroDoc::new(); - imported.import(&doc.export(ExportMode::Snapshot)?)?; - for d in [&doc, &imported] { - let out = d.get_deep_value_json_with_ids()?; - assert_eq!(out.container_positions, vec![1, 10, 11]); - assert_eq!(out.cids.len(), 3); - assert_eq!( - serde_json::from_str::(&out.json).unwrap(), - serde_json::to_value(d.get_deep_value()).unwrap() - ); - assert!(out.json.contains("cid:root-fake:Map")); - assert!(out.json.find("two").unwrap() < out.json.find("ten").unwrap()); - } - Ok(()) -} - -#[test] -fn streaming_read_resolves_mergeable_children_and_respects_root_visibility() -> LoroResult<()> { - let doc = LoroDoc::new_auto_commit(); - let map = doc.get_map("m"); - let child = map.ensure_mergeable_map("child")?; - child.insert("number", 42)?; - let out = doc.get_deep_value_json_with_ids()?; - assert_eq!(out.cids, vec![map.id().to_string(), child.id().to_string()]); - assert_eq!(out.container_positions, vec![1, 2]); - assert_eq!( - serde_json::from_str::(&out.json).unwrap(), - serde_json::to_value(doc.get_deep_value()).unwrap() - ); - doc.get_text("empty"); - doc.config().set_hide_empty_root_containers(true); - let out = doc.get_deep_value_json_with_ids()?; - assert_eq!(out.cids, vec![map.id().to_string(), child.id().to_string()]); - assert_eq!( - serde_json::from_str::(&out.json).unwrap(), - serde_json::to_value(doc.get_deep_value()).unwrap() - ); - Ok(()) -} diff --git a/crates/loro-wasm/AGENTS.md b/crates/loro-wasm/AGENTS.md index f30ca4e3e..8f3951a0a 100644 --- a/crates/loro-wasm/AGENTS.md +++ b/crates/loro-wasm/AGENTS.md @@ -75,11 +75,6 @@ Container `id` wrapper identity, lazy-cache lifetime, and the focused benchmark are documented in [context/wasm-container-id-cache.md](../../context/wasm-container-id-cache.md). -Bulk-read APIs (per-container/range deep reads with ids, `getDeepValueJson`, -`getDeepValueJsonWithIds`, the cid format, and the cids pre-order contract) -are documented in -[context/wasm-bulk-read.md](../../context/wasm-bulk-read.md). - ## Packaging Rules - Preserve the public `loro-crdt` API names and package export paths used by @@ -92,3 +87,7 @@ are documented in and Rollup need either the `base64` entry or an explicit asset copy. Keep the bundler smoke tests aligned with these expectations. - If package output or published behavior changes, add a changeset. + +`LoroDoc.readState` constructs typed nested snapshots with fixed JS helpers. See +[context/wasm-bulk-read.md](../../context/wasm-bulk-read.md) for identity, ownership, +range and Mirror integration contracts. diff --git a/crates/loro-wasm/scripts/measure-deep-value-json.cjs b/crates/loro-wasm/scripts/measure-deep-value-json.cjs deleted file mode 100644 index e549e7c53..000000000 --- a/crates/loro-wasm/scripts/measure-deep-value-json.cjs +++ /dev/null @@ -1,324 +0,0 @@ -const { performance } = require("node:perf_hooks"); -const assert = require("node:assert/strict"); -const fs = require("node:fs"); -const os = require("node:os"); -const path = require("node:path"); -const { spawnSync } = require("node:child_process"); -const modulePath = - process.env.LORO_BENCH_MODULE || - path.resolve(__dirname, "../nodejs/index.js"); -const { LoroDoc, LoroList, LoroMap, LoroText } = require(modulePath); - -const MAP_COUNT = 15_632; -const LIST_COUNT = 9_956; -const TEXT_COUNT = 44_463; -const PARENT_POOL_SIZE = 2_048; -const PARAGRAPH = "The quick brown fox jumps over the lazy dog. Pack my box. "; - -const WARMUP_ROUNDS = 2; -const ROUNDS = 5; -let blackhole = 0; - -function buildDocument() { - const doc = new LoroDoc(); - doc.setPeerId("1"); - const root = doc.getMap("root"); - const parents = [root]; - let mi = 1; - let li = 0; - let ti = 0; - let seq = 0; - const addChild = (child, isParent) => { - const parent = parents[seq % parents.length]; - seq++; - let attached; - if (parent.kind() === "Map") { - attached = parent.setContainer(`k${seq}`, child); - } else { - attached = parent.pushContainer(child); - } - if (isParent && parents.length < PARENT_POOL_SIZE) parents.push(attached); - return attached; - }; - while (mi < MAP_COUNT || li < LIST_COUNT || ti < TEXT_COUNT) { - if (mi < MAP_COUNT) { - const map = addChild(new LoroMap(), true); - map.set("id", mi); - map.set("flag", true); - map.set("name", `record-${mi}`); - mi++; - } - if (li < LIST_COUNT) { - const list = addChild(new LoroList(), true); - list.insert(0, li); - list.insert(1, `item-${li}`); - li++; - } - if (ti < TEXT_COUNT) { - const text = addChild(new LoroText(), false); - text.insert(0, `${PARAGRAPH}${ti}`); - ti++; - } - } - doc.commit(); - return doc; -} - -function decode(result, project = false) { - let next = 0, - position = 0; - const registry = new Set(); - function walk(value) { - const cid = - result.containerPositions[next] === position++ - ? result.cids[next++] - : undefined; - if (Array.isArray(value)) { - for (let i = 0; i < value.length; i++) value[i] = walk(value[i]); - } else if (value !== null && typeof value === "object") { - for (const key of Object.keys(value)) value[key] = walk(value[key]); - } - if (cid === undefined) return value; - if (!project) return { cid, value }; - registry.add(cid); - if (cid.endsWith(":Map")) - Object.defineProperty(value, "$cid", { value: cid }); - return value; - } - const value = walk(JSON.parse(result.json)); - assert.equal(next, result.cids.length); - return project ? { value, registry } : value; -} - -function pathsFor(result) { - let next = 0, - position = 0; - const paths = [], - stack = []; - function walk(value) { - if (result.containerPositions[next] === position++) { - paths.push(stack.slice()); - next++; - } - if (value !== null && typeof value === "object") { - for (const key of Object.keys(value)) { - stack.push(Array.isArray(value) ? Number(key) : key); - walk(value[key]); - stack.pop(); - } - } - } - walk(JSON.parse(result.json)); - assert.equal(next, result.cids.length); - return paths; -} - -function attachPaths(json, cids, paths) { - let value = JSON.parse(json); - // Descendants first so replacing a parent does not invalidate its paths. - for (let i = paths.length - 1; i >= 0; i--) { - const steps = paths[i]; - if (!steps.length) { - value = { cid: cids[i], value }; - continue; - } - let parent = value; - for (let j = 0; j < steps.length - 1; j++) parent = parent[steps[j]]; - const key = steps[steps.length - 1]; - parent[key] = { cid: cids[i], value: parent[key] }; - } - return value; -} - -function handleProjection(container, registry) { - const kind = container.kind(); - const cid = container.id; - registry.add(cid); - let out; - if (kind === "Map") { - out = {}; - Object.defineProperty(out, "$cid", { value: cid }); - for (const key of container.keys()) { - const child = container.get(key); - out[key] = - child && typeof child.kind === "function" - ? handleProjection(child, registry) - : child; - } - } else if (kind === "List" || kind === "MovableList") { - out = []; - for (let i = 0, n = container.length; i < n; i++) { - const child = container.get(i); - out.push( - child && typeof child.kind === "function" - ? handleProjection(child, registry) - : child, - ); - } - } else { - out = container.toJSON(); - } - container.free(); - return out; -} - -const name = process.argv[2]; -if (name) { - // Each case gets its own process and a freshly imported snapshot. Do not - // hide allocations in an already materialized, directly built document. - const snapshot = fs.readFileSync(process.argv[3]); - const doc = new LoroDoc(); - const startImport = performance.now(); - doc.import(snapshot); - const importMs = performance.now() - startImport; - const run = { - toJSON: () => doc.toJSON(), - getDeepValueWithID: () => doc.getDeepValueWithID(), - jsonParse: () => JSON.parse(doc.getDeepValueJson()), - indexedJson: () => doc.getDeepValueJsonWithIds(), - legacyJson: () => doc.getDeepValueJsonWithIds(), - indexedJsonParseReattach: () => decode(doc.getDeepValueJsonWithIds()), - indexedProjection: () => decode(doc.getDeepValueJsonWithIds(), true), - handleProjection: () => { - const registry = new Set(); - return { - value: { root: handleProjection(doc.getMap("root"), registry) }, - registry, - }; - }, - }[name]; - global.gc?.(); - const before = process.memoryUsage(); - const start = performance.now(); - let held = run(); - const coldMs = performance.now() - start; - global.gc?.(); - const after = process.memoryUsage(); - const memory = Object.fromEntries( - ["external", "heapUsed", "rss"].map((k) => [ - k + "DeltaBytes", - after[k] - before[k], - ]), - ); - // Correctness outside timed regions; a faster read of the wrong shape is - // not an optimization. Projection includes cid registration and map $cid. - if (name === "indexedJson" || name === "indexedJsonParseReattach") { - assert.deepStrictEqual( - name === "indexedJson" ? decode(held) : held, - doc.getDeepValueWithID(), - ); - } else if (name === "indexedProjection" || name === "handleProjection") { - assert.deepStrictEqual(held.value, doc.toJSON()); - assert.equal(held.registry.size, MAP_COUNT + LIST_COUNT + TEXT_COUNT); - assert.equal(held.value.root.$cid, doc.getMap("root").id); - } else if (name === "legacyJson") { - assert.deepStrictEqual( - JSON.parse(held.json), - JSON.parse(doc.getDeepValueJson()), - ); - } else if (name !== "getDeepValueWithID") { - assert.deepStrictEqual(held, doc.toJSON()); - } - held = null; - const samples = []; - for (let round = 0; round < WARMUP_ROUNDS + ROUNDS; round++) { - global.gc?.(); - const t = performance.now(); - held = run(); - const elapsed = performance.now() - t; - blackhole += Object.keys(held).length; - held = null; - if (round >= WARMUP_ROUNDS) samples.push(elapsed); - } - samples.sort((a, b) => a - b); - console.log( - JSON.stringify({ - name, - importMs, - coldMs, - warmMedianMs: samples[Math.floor(samples.length / 2)], - ...memory, - blackhole, - }), - ); - doc.free(); -} else { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "loro-bulk-bench-")); - try { - const doc = buildDocument(); - const file = path.join(dir, "fixture.bin"); - fs.writeFileSync(file, doc.export({ mode: "snapshot" })); - const result = doc.getDeepValueJsonWithIds(); - const supportsPositions = result.containerPositions instanceof Uint32Array; - const cases = [ - "toJSON", - "getDeepValueWithID", - "jsonParse", - "handleProjection", - ]; - let metadata; - if (supportsPositions) { - cases.push( - "indexedJson", - "indexedJsonParseReattach", - "indexedProjection", - ); - const paths = pathsFor(result); - assert.deepStrictEqual( - attachPaths(result.json, result.cids, paths), - decode(result), - ); - const positionSamples = [], - pathSamples = []; - for (let i = 0; i < WARMUP_ROUNDS + ROUNDS; i++) { - global.gc?.(); - let t = performance.now(); - decode(result); - const posMs = performance.now() - t; - t = performance.now(); - attachPaths(result.json, result.cids, paths); - const pathMs = performance.now() - t; - if (i >= WARMUP_ROUNDS) { - positionSamples.push(posMs); - pathSamples.push(pathMs); - } - } - const median = (xs) => - xs.sort((a, b) => a - b)[Math.floor(xs.length / 2)]; - metadata = { - positionsBytes: result.containerPositions.byteLength, - pathsJsonBytes: Buffer.byteLength(JSON.stringify(paths)), - pathSegments: paths.reduce((n, p) => n + p.length, 0), - cidsJsonBytes: Buffer.byteLength(JSON.stringify(result.cids)), - // These isolate JS consumption; path generation/transfer is excluded. - positionsParseAttachMs: median(positionSamples), - pathsParseAttachMs: median(pathSamples), - }; - } - if (!supportsPositions) cases.push("legacyJson"); - doc.free(); - const measurements = cases.map((name) => { - const child = spawnSync( - process.execPath, - ["--expose-gc", __filename, name, file], - { encoding: "utf8", env: process.env }, - ); - if (child.status !== 0) throw new Error(child.stderr || child.stdout); - return JSON.parse(child.stdout); - }); - console.log( - JSON.stringify( - { - containers: MAP_COUNT + LIST_COUNT + TEXT_COUNT, - jsonBytes: Buffer.byteLength(result.json), - metadata, - measurements, - }, - null, - 2, - ), - ); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -} diff --git a/crates/loro-wasm/src/counter.rs b/crates/loro-wasm/src/counter.rs index f78a4d879..7920a88ca 100644 --- a/crates/loro-wasm/src/counter.rs +++ b/crates/loro-wasm/src/counter.rs @@ -136,41 +136,4 @@ impl LoroCounter { .into_double() .map_err(|_| JsValue::from_str("Counter value is not a number")) } - - /// Get the counter value as JSON text (a JSON number). - /// - /// The content is identical to `JSON.stringify(counter.toJSON())`, but the - /// JSON text is produced inside WASM. Unlike the other container types, - /// this also works on a detached counter, mirroring `toJSON()`. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; - /// - /// const doc = new LoroDoc(); - /// const counter = doc.getCounter("counter"); - /// counter.increment(1.5); - /// console.log(counter.getDeepValueJson()); // "1.5" - /// ``` - #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] - pub fn get_deep_value_json(&self) -> JsResult { - Ok(self.handler.get_deep_value_json()?) - } - - /// Read plain JSON plus an exact, sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. - /// Count every value in pre-order from zero, including scalars and plain - /// objects/arrays; visit object children in `Object.keys` order. - /// The document object counts as zero but has no id; for a container call, - /// position zero identifies that container. Tree metadata remains plain - /// deep data, matching `getDeepValueWithID()`. - /// - /// No schema or value-shape guesses are needed. The positions are a copied - /// Uint32Array (four bytes per container), not a view into WASM memory. - /// Detached containers throw, except counters which also support reads. - #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] - pub fn get_deep_value_json_with_ids(&self) -> JsResult { - crate::deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) - } } diff --git a/crates/loro-wasm/src/lib.rs b/crates/loro-wasm/src/lib.rs index ced878011..9fe8ee2fb 100644 --- a/crates/loro-wasm/src/lib.rs +++ b/crates/loro-wasm/src/lib.rs @@ -5,6 +5,8 @@ #![allow(clippy::doc_lazy_continuation)] // #![warn(missing_docs)] +mod read_state; + use convert::{ import_blob_metadata_to_js, import_status_to_js_value, js_diff_to_inner_diff, js_json_schema_to_loro_json_schema, js_to_id_span, js_to_version_vector, @@ -525,26 +527,6 @@ fn id_to_js(id: &ID) -> JsResult { Ok(obj.into()) } -pub(crate) fn deep_value_json_with_ids_to_js( - result: loro_internal::DeepValueJsonWithIds, -) -> JsResult { - let loro_internal::DeepValueJsonWithIds { - json, - cids, - container_positions, - } = result; - let obj = Object::new(); - Reflect::set(&obj, &"json".into(), &json.into())?; - let arr = Array::new(); - for cid in cids { - arr.push(&cid.into()); - } - Reflect::set(&obj, &"cids".into(), &arr)?; - let positions = js_sys::Uint32Array::from(container_positions.as_slice()); - Reflect::set(&obj, &"containerPositions".into(), &positions)?; - Ok(obj.into()) -} - fn peer_id_to_js(peer: PeerID) -> JsStrPeerID { let v: JsValue = peer.to_string().into(); v.into() @@ -1488,42 +1470,6 @@ impl LoroDoc { self.doc.get_deep_value_with_id().into() } - /// Get the deep value of the document as JSON text. - /// - /// The content is identical to `JSON.stringify(doc.toJSON())`, but the - /// JSON text is produced inside WASM in a single call, avoiding the cost - /// of crossing the WASM/JS boundary with a large structured value. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; - /// - /// const doc = new LoroDoc(); - /// doc.getText("text").insert(0, "Hello"); - /// console.log(doc.getDeepValueJson()); // {"text":"Hello"} - /// ``` - #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] - pub fn get_deep_value_json(&self) -> JsResult { - Ok(self.doc.get_deep_value_json()?) - } - - /// Read plain JSON plus an exact, sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. - /// Count every value in pre-order from zero, including scalars and plain - /// objects/arrays; visit object children in `Object.keys` order. - /// The document object counts as zero but has no id; for a container call, - /// position zero identifies that container. Tree metadata remains plain - /// deep data, matching `getDeepValueWithID()`. - /// - /// No schema or value-shape guesses are needed. The positions are a copied - /// Uint32Array (four bytes per container), not a view into WASM memory. - /// Detached containers throw, except counters which also support reads. - #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] - pub fn get_deep_value_json_with_ids(&self) -> JsResult { - deep_value_json_with_ids_to_js(self.doc.get_deep_value_json_with_ids()?) - } - /// Get the path from the root to the container #[wasm_bindgen(js_name = "getPathToContainer")] pub fn get_path_to_container(&self, id: JsContainerID) -> JsResult> { @@ -3309,47 +3255,6 @@ impl LoroText { pub fn get_deep_value_with_id(&self) -> JsResult { Ok(self.handler.get_deep_value_with_id()?.into()) } - - /// Get the deep value of the text as JSON text. - /// - /// The content is identical to `JSON.stringify(text.toJSON())`, but - /// the JSON text is produced inside WASM in a single call, avoiding the - /// cost of crossing the WASM/JS boundary with a large structured value. - /// - /// For a text container the JSON text is a JSON string of the text - /// content. - /// Throws if the container is detached. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; - /// - /// const doc = new LoroDoc(); - /// const text = doc.getText("text"); - /// text.insert(0, "Hello"); - /// console.log(text.getDeepValueJson()); // \"Hello\" - /// ``` - #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] - pub fn get_deep_value_json(&self) -> JsResult { - Ok(self.handler.get_deep_value_json()?) - } - - /// Read plain JSON plus an exact, sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. - /// Count every value in pre-order from zero, including scalars and plain - /// objects/arrays; visit object children in `Object.keys` order. - /// The document object counts as zero but has no id; for a container call, - /// position zero identifies that container. Tree metadata remains plain - /// deep data, matching `getDeepValueWithID()`. - /// - /// No schema or value-shape guesses are needed. The positions are a copied - /// Uint32Array (four bytes per container), not a view into WASM memory. - /// Detached containers throw, except counters which also support reads. - #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] - pub fn get_deep_value_json_with_ids(&self) -> JsResult { - deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) - } } impl Default for LoroText { @@ -3639,45 +3544,6 @@ impl LoroMap { Ok(self.handler.get_deep_value_with_id()?.into()) } - /// Get the deep value of the map as JSON text. - /// - /// The content is identical to `JSON.stringify(map.toJSON())`, but - /// the JSON text is produced inside WASM in a single call, avoiding the - /// cost of crossing the WASM/JS boundary with a large structured value. - /// - /// Throws if the container is detached. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; - /// - /// const doc = new LoroDoc(); - /// const map = doc.getMap("map"); - /// map.set("foo", "bar"); - /// console.log(map.getDeepValueJson()); // {\"foo\":\"bar\"} - /// ``` - #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] - pub fn get_deep_value_json(&self) -> JsResult { - Ok(self.handler.get_deep_value_json()?) - } - - /// Read plain JSON plus an exact, sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. - /// Count every value in pre-order from zero, including scalars and plain - /// objects/arrays; visit object children in `Object.keys` order. - /// The document object counts as zero but has no id; for a container call, - /// position zero identifies that container. Tree metadata remains plain - /// deep data, matching `getDeepValueWithID()`. - /// - /// No schema or value-shape guesses are needed. The positions are a copied - /// Uint32Array (four bytes per container), not a view into WASM memory. - /// Detached containers throw, except counters which also support reads. - #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] - pub fn get_deep_value_json_with_ids(&self) -> JsResult { - deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) - } - /// Set the key with a regular child container. /// /// The inserted child receives a regular op-created container id. Use this @@ -4109,45 +3975,6 @@ impl LoroList { Ok(self.handler.get_deep_value_with_id()?.into()) } - /// Get the deep value of the list as JSON text. - /// - /// The content is identical to `JSON.stringify(list.toJSON())`, but - /// the JSON text is produced inside WASM in a single call, avoiding the - /// cost of crossing the WASM/JS boundary with a large structured value. - /// - /// Throws if the container is detached. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; - /// - /// const doc = new LoroDoc(); - /// const list = doc.getList("list"); - /// list.insert(0, 100); - /// console.log(list.getDeepValueJson()); // [100] - /// ``` - #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] - pub fn get_deep_value_json(&self) -> JsResult { - Ok(self.handler.get_deep_value_json()?) - } - - /// Read plain JSON plus an exact, sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. - /// Count every value in pre-order from zero, including scalars and plain - /// objects/arrays; visit object children in `Object.keys` order. - /// The document object counts as zero but has no id; for a container call, - /// position zero identifies that container. Tree metadata remains plain - /// deep data, matching `getDeepValueWithID()`. - /// - /// No schema or value-shape guesses are needed. The positions are a copied - /// Uint32Array (four bytes per container), not a view into WASM memory. - /// Detached containers throw, except counters which also support reads. - #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] - pub fn get_deep_value_json_with_ids(&self) -> JsResult { - deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) - } - /// Get the deep value of the elements in the range `[start, end)`, with /// container ids. /// @@ -4590,45 +4417,6 @@ impl LoroMovableList { Ok(self.handler.get_deep_value_with_id()?.into()) } - /// Get the deep value of the movableList as JSON text. - /// - /// The content is identical to `JSON.stringify(movableList.toJSON())`, but - /// the JSON text is produced inside WASM in a single call, avoiding the - /// cost of crossing the WASM/JS boundary with a large structured value. - /// - /// Throws if the container is detached. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; - /// - /// const doc = new LoroDoc(); - /// const movableList = doc.getMovableList("list"); - /// movableList.insert(0, 100); - /// console.log(movableList.getDeepValueJson()); // [100] - /// ``` - #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] - pub fn get_deep_value_json(&self) -> JsResult { - Ok(self.handler.get_deep_value_json()?) - } - - /// Read plain JSON plus an exact, sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. - /// Count every value in pre-order from zero, including scalars and plain - /// objects/arrays; visit object children in `Object.keys` order. - /// The document object counts as zero but has no id; for a container call, - /// position zero identifies that container. Tree metadata remains plain - /// deep data, matching `getDeepValueWithID()`. - /// - /// No schema or value-shape guesses are needed. The positions are a copied - /// Uint32Array (four bytes per container), not a view into WASM memory. - /// Detached containers throw, except counters which also support reads. - #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] - pub fn get_deep_value_json_with_ids(&self) -> JsResult { - deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) - } - /// Get the deep value of the elements in the range `[start, end)`, with /// container ids. /// @@ -5452,47 +5240,6 @@ impl LoroTree { Ok(self.handler.get_deep_value_with_id()?.into()) } - /// Get the deep value of the tree as JSON text. - /// - /// The content is identical to `JSON.stringify(tree.toJSON())`, but - /// the JSON text is produced inside WASM in a single call, avoiding the - /// cost of crossing the WASM/JS boundary with a large structured value. - /// - /// Tree node meta maps are plain deep values, so their container ids - /// do not appear in `cids`. - /// Throws if the container is detached. - /// - /// @example - /// ```ts - /// import { LoroDoc } from "loro-crdt"; - /// - /// const doc = new LoroDoc(); - /// const tree = doc.getTree("tree"); - /// tree.createNode(); - /// console.log(tree.getDeepValueJson()); // [{\"id\":\"...\",\"parent\":null,\"meta\":{},\"index\":0,\"fractional_index\":\"...\",\"children\":[]}] - /// ``` - #[wasm_bindgen(js_name = "getDeepValueJson", skip_typescript)] - pub fn get_deep_value_json(&self) -> JsResult { - Ok(self.handler.get_deep_value_json()?) - } - - /// Read plain JSON plus an exact, sparse container-position index. - /// - /// `cids[i]` belongs to the JSON value at `containerPositions[i]`. - /// Count every value in pre-order from zero, including scalars and plain - /// objects/arrays; visit object children in `Object.keys` order. - /// The document object counts as zero but has no id; for a container call, - /// position zero identifies that container. Tree metadata remains plain - /// deep data, matching `getDeepValueWithID()`. - /// - /// No schema or value-shape guesses are needed. The positions are a copied - /// Uint32Array (four bytes per container), not a view into WASM memory. - /// Detached containers throw, except counters which also support reads. - #[wasm_bindgen(js_name = "getDeepValueJsonWithIds", skip_typescript)] - pub fn get_deep_value_json_with_ids(&self) -> JsResult { - deep_value_json_with_ids_to_js(self.handler.get_deep_value_json_with_ids()?) - } - /// Get all tree nodes of the forest, including deleted nodes. /// /// @example @@ -6891,27 +6638,6 @@ export type ValueWithContainerID = { value: Value, } -/** - * Plain JSON plus a sparse index of container identities. - * - * `cids[i]` belongs to value `containerPositions[i]` in a pre-order walk of - * JSON.parse(json). Count EVERY value (objects, arrays and scalars), starting - * at zero; visit arrays in index order and objects in Object.keys order. - * Binary data serializes as an array, so each byte counts as another value. - * Document calls count the root object but do not assign it a cid; container - * calls mark position zero. Tree metadata is plain deep data, matching the - * existing getDeepValueWithID format. Ordinary cid/value objects stay data. - * - * This JSON view follows getDeepValueJson's root visibility and JSON scalar - * representation (e.g. binary arrays and non-finite numbers as null). - * containerPositions owns its buffer and remains valid after further WASM calls. - */ -export type DeepValueJsonWithIds = { - json: string, - cids: ContainerID[], - containerPositions: Uint32Array, -} - export type IdSpan = { peer: PeerID, counter: number, @@ -7397,28 +7123,6 @@ interface LoroDoc { * You can debounce/throttle the callback before running `JSONPath(...)` to optimize heavy reads. */ subscribeJsonpath(path: string, callback: () => void): Subscription; - /** - * Get the deep value of the document as JSON text. - * - * The content is identical to `JSON.stringify(doc.toJSON())`, but the - * JSON text is produced inside WASM in a single call, avoiding the cost - * of crossing the WASM/JS boundary with a large structured value. - */ - getDeepValueJson(): string; - /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ - getDeepValueJsonWithIds(): DeepValueJsonWithIds; -} - -interface LoroCounter { - /** - * Get the counter value as JSON text (a JSON number). - * - * Unlike the other container types, this also works on a detached - * counter, mirroring `toJSON()`. - */ - getDeepValueJson(): string; - /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ - getDeepValueJsonWithIds(): DeepValueJsonWithIds; } interface UndoManager { @@ -7574,20 +7278,6 @@ interface LoroList { * own `{ cid, value }` nodes. */ getDeepValueWithID(): ValueWithContainerID; - - /** - * Get the deep value of the container as JSON text. - * - * The content is identical to `JSON.stringify` of the container's - * `toJSON()`, but the JSON text is produced inside WASM in a single call, - * avoiding the cost of crossing the WASM/JS boundary with a large - * structured value. - * - * Throws if the container is detached. - */ - getDeepValueJson(): string; - /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ - getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get the deep value of the elements in the range `[start, end)`, with * container ids. @@ -7691,20 +7381,6 @@ interface LoroMovableList { * own `{ cid, value }` nodes. */ getDeepValueWithID(): ValueWithContainerID; - - /** - * Get the deep value of the container as JSON text. - * - * The content is identical to `JSON.stringify` of the container's - * `toJSON()`, but the JSON text is produced inside WASM in a single call, - * avoiding the cost of crossing the WASM/JS boundary with a large - * structured value. - * - * Throws if the container is detached. - */ - getDeepValueJson(): string; - /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ - getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get the deep value of the elements in the range `[start, end)`, with * container ids. @@ -7874,20 +7550,6 @@ interface LoroMap = Record> { * `{ cid, value }` nodes. */ getDeepValueWithID(): ValueWithContainerID; - - /** - * Get the deep value of the container as JSON text. - * - * The content is identical to `JSON.stringify` of the container's - * `toJSON()`, but the JSON text is produced inside WASM in a single call, - * avoiding the cost of crossing the WASM/JS boundary with a large - * structured value. - * - * Throws if the container is detached. - */ - getDeepValueJson(): string; - /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ - getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Get or create a regular child container at the given key. * @@ -8014,20 +7676,6 @@ interface LoroText { * string (the same as `text.id`) and `value` is the text content. */ getDeepValueWithID(): { cid: ContainerID, value: string }; - - /** - * Get the deep value of the container as JSON text. - * - * The content is identical to `JSON.stringify` of the container's - * `toJSON()`, but the JSON text is produced inside WASM in a single call, - * avoiding the cost of crossing the WASM/JS boundary with a large - * structured value. - * - * Throws if the container is detached. - */ - getDeepValueJson(): string; - /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ - getDeepValueJsonWithIds(): DeepValueJsonWithIds; insert(pos: number, text: string): void; delete(pos: number, len: number): void; subscribe(listener: Listener): Subscription; @@ -8073,20 +7721,6 @@ interface LoroTree = Record> * emits for every container. */ getDeepValueWithID(): ValueWithContainerID; - - /** - * Get the deep value of the container as JSON text. - * - * The content is identical to `JSON.stringify` of the container's - * `toJSON()`, but the JSON text is produced inside WASM in a single call, - * avoiding the cost of crossing the WASM/JS boundary with a large - * structured value. - * - * Throws if the container is detached. - */ - getDeepValueJson(): string; - /** Read plain JSON and its sparse identity index; see DeepValueJsonWithIds. */ - getDeepValueJsonWithIds(): DeepValueJsonWithIds; /** * Create a new tree node as the child of parent and return a `LoroTreeNode` instance. * If the parent is undefined, the tree node will be a root node. diff --git a/crates/loro-wasm/src/read_state.rs b/crates/loro-wasm/src/read_state.rs new file mode 100644 index 000000000..f7074a26a --- /dev/null +++ b/crates/loro-wasm/src/read_state.rs @@ -0,0 +1,263 @@ +//! Build JS state with fixed constructors; no callbacks or document-wide intermediary. +use super::*; +use loro_internal::read_state::{err, Event, Sink}; +use std::collections::HashMap; +#[wasm_bindgen(inline_js = " +const kinds = ['Map', 'List', 'MovableList', 'Text', 'Tree', 'Counter']; +export function stateContainer(kind, cid, value) { return {type: kinds[kind], cid, value}; } +export function stateCid(peer,counter,kind) { return 'cid:'+counter+'@'+peer+':'+kinds[kind]; } +export function stateRootCid(name,kind) { return 'cid:root-'+name+':'+kinds[kind]; } +export function stateValue(value) { return {type: 'Value', value}; } +export function stateBinary(bytes) { return bytes.slice(); } +export function stateIndex(array, index, value) { Object.defineProperty(array, index, {value, enumerable: true, writable: true, configurable: true}); } +export function stateSet(object, key, value) { Object.defineProperty(object, key, {value, enumerable: true, writable: true, configurable: true}); } +")] +extern "C" { + fn stateContainer(kind: u8, cid: &JsValue, value: JsValue) -> JsValue; + fn stateValue(value: JsValue) -> JsValue; + fn stateBinary(bytes: &[u8]) -> JsValue; + #[wasm_bindgen(catch)] + fn stateIndex(array: &JsValue, index: u32, value: &JsValue) -> Result<(), JsValue>; + fn stateCid(peer: &JsValue, counter: i32, kind: u8) -> JsValue; + fn stateRootCid(name: &str, kind: u8) -> JsValue; + #[wasm_bindgen(catch)] + fn stateSet(object: &JsValue, key: &JsValue, value: &JsValue) -> Result<(), JsValue>; +} +struct Frame { + wrapper: u8, + cid: JsValue, + v: JsValue, + array: bool, + next: u32, + key: JsValue, +} +#[derive(Default)] +struct JsSink { + stack: Vec, + root: JsValue, + keys: HashMap, +} +impl JsSink { + fn add(&mut self, v: JsValue) -> LoroResult<()> { + if let Some(f) = self.stack.last_mut() { + if f.wrapper != 0 { + f.v = v; + return Ok(()); + } + if f.array { + stateIndex(&f.v, f.next, &v).map_err(|_| err("JS array property failed"))?; + f.next += 1; + } else { + stateSet(&f.v, &f.key, &v).map_err(|_| err("JS define property failed"))?; + } + } else { + self.root = v; + } + Ok(()) + } +} +impl JsSink { + fn emit(&mut self, e: Event<'_>) -> LoroResult<()> { + match e { + Event::Key(k) => { + let key = if let Some(v) = self.keys.get(k) { + v.clone() + } else { + let v: JsValue = k.into(); + self.keys.insert(k.to_owned(), v.clone()); + v + }; + let f = self + .stack + .last_mut() + .ok_or_else(|| err("Missing object frame"))?; + f.key = key; + } + Event::Object(_) => self.stack.push(Frame { + wrapper: 0, + cid: JsValue::NULL, + v: Object::new().into(), + array: false, + next: 0, + key: JsValue::NULL, + }), + Event::Array(n) => self.stack.push(Frame { + wrapper: 0, + cid: JsValue::NULL, + v: Array::new_with_length(n as u32).into(), + array: true, + next: 0, + key: JsValue::NULL, + }), + Event::End => { + let f = self.stack.pop().ok_or_else(|| err("Missing state frame"))?; + let v = if f.wrapper == 16 { + stateValue(f.v) + } else if f.wrapper != 0 { + stateContainer(f.wrapper - 10, &f.cid, f.v) + } else { + f.v + }; + self.add(v)?; + } + Event::String(s) => self.add(s.into())?, + Event::Number(n) => self.add(n.into())?, + Event::Bool(b) => self.add(b.into())?, + Event::Binary(bytes) => self.add(stateBinary(bytes))?, + Event::Null => self.add(JsValue::NULL)?, + }; + Ok(()) + } +} +#[derive(Default)] +struct FixedSink(JsSink, HashMap); +impl Sink for FixedSink { + fn container(&mut self, id: &ContainerID) -> LoroResult<()> { + let kind = match id.container_type() { + ContainerType::Map => 0, + ContainerType::List => 1, + ContainerType::MovableList => 2, + ContainerType::Text => 3, + ContainerType::Tree => 4, + ContainerType::Counter => 5, + _ => return Err(err("Unsupported container type")), + }; + let cid = match id { + ContainerID::Root { name, .. } => stateRootCid(name, kind), + ContainerID::Normal { peer, counter, .. } => { + let p = self + .1 + .entry(*peer) + .or_insert_with(|| JsValue::from_str(&peer.to_string())); + stateCid(p, *counter, kind) + } + }; + self.0.stack.push(Frame { + wrapper: kind + 10, + cid, + v: JsValue::NULL, + array: false, + next: 0, + key: JsValue::NULL, + }); + Ok(()) + } + + fn emit(&mut self, e: Event<'_>) -> LoroResult<()> { + self.0.emit(e) + } + fn container_end(&mut self) -> LoroResult<()> { + self.emit(Event::End) + } + fn value_start(&mut self) -> LoroResult<()> { + self.0.stack.push(Frame { + wrapper: 16, + cid: JsValue::NULL, + v: JsValue::NULL, + array: false, + next: 0, + key: JsValue::NULL, + }); + Ok(()) + } + fn value_end(&mut self) -> LoroResult<()> { + self.emit(Event::End) + } +} + +#[derive(Default, serde::Deserialize)] +struct Options { + container: Option, + #[serde(default)] + text: TextMode, + range: Option, +} +#[derive(Default, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +enum TextMode { + #[default] + Plain, + Delta, +} +#[derive(serde::Deserialize)] +struct Range { + start: u32, + end: u32, +} + +#[wasm_bindgen] +impl LoroDoc { + /// Read nested container state with explicit IDs and opaque ordinary values. + /// Reads current state without committing. Text defaults to plain strings. + /// A list range is end-exclusive and clamped; inverted ranges are empty. + /// Throws for invalid options, missing containers, unsupported container types, + /// or nesting exceeding 256 levels. Returned objects are independent snapshots. + #[wasm_bindgen(js_name = readState, skip_typescript)] + pub fn read_state(&self, options: Option) -> JsResult { + let opts: Options = match options { + None => Options::default(), + Some(v) => serde_wasm_bindgen::from_value(v.into()) + .map_err(|e| JsValue::from_str(&format!("Invalid readState options: {e}")))?, + }; + let cid = opts + .container + .as_deref() + .map(ContainerID::try_from) + .transpose() + .map_err(|_| JsValue::from_str("Invalid container ID"))?; + if cid.as_ref().is_some_and(|id| !self.doc.has_container(id)) { + return Err(JsValue::from_str("The container does not exist in the doc")); + } + let mut sink = FixedSink::default(); + self.doc.app_state().lock().read_state( + &mut sink, + cid.as_ref(), + matches!(opts.text, TextMode::Delta), + opts.range.map(|r| (r.start as usize, r.end as usize)), + )?; + Ok(sink.0.root.unchecked_into()) + } +} +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(typescript_type = "ReadStateOptions")] + pub type JsReadStateOptions; + #[wasm_bindgen(typescript_type = "ContainerState | Record")] + pub type JsReadState; +} +#[wasm_bindgen(typescript_custom_section)] +const TYPES: &str = r#" +interface LoroDoc { + /** Read nested container snapshots without committing. Ordinary data stays inside Value nodes. + * Text defaults to strings; use text: "delta" to preserve formatting. + * Throws for invalid options, unsupported containers, or nesting over 256 levels. + */ + readState(options?: ReadStateOptions & { container?: undefined; range?: never }): Record; + /** Read one container; list ranges are clamped and end-exclusive. */ + readState(options: ReadStateOptions & { container: ContainerID }): ContainerState; + readState(options: ReadStateOptions): ContainerState | Record; +} +/** Options for reading the whole document, a container, or a list interval. */ +export interface ReadStateOptions { + container?: ContainerID; + text?: "plain" | "delta"; + range?: { start: number; end: number }; +} +/** Ordinary data is opaque: never interpret objects inside value as containers. */ +export type StateValue = { type: "Value"; value: Value }; +export type StateNode = ContainerState | StateValue; +export type ContainerState = + | { type: "Map"; cid: ContainerID; value: Record } + | { type: "List" | "MovableList"; cid: ContainerID; value: StateNode[] } + | { type: "Text"; cid: ContainerID; value: string | Delta[] } + | { type: "Tree"; cid: ContainerID; value: StateTreeNode[] } + | { type: "Counter"; cid: ContainerID; value: number }; +export interface StateTreeNode { + id: TreeID; + parent: TreeID | null; + index: number; + fractional_index: string; + meta: Extract; + children: StateTreeNode[]; +} +"#; diff --git a/crates/loro-wasm/tests/deep_value.test.ts b/crates/loro-wasm/tests/deep_value.test.ts index 18d2e1991..4fa55ee32 100644 --- a/crates/loro-wasm/tests/deep_value.test.ts +++ b/crates/loro-wasm/tests/deep_value.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from "vitest"; import { - LoroCounter, LoroDoc, LoroList, LoroMap, @@ -135,7 +134,12 @@ describe("range deep reads", () => { it("getRangeValue resolves containers to plain deep values", () => { const { list } = setup(); expect(list.getRangeValue(1, 3)).toStrictEqual(["b", { k: "v" }]); - expect(list.getRangeValue(0, 4)).toStrictEqual(["a", "b", { k: "v" }, "d"]); + expect(list.getRangeValue(0, 4)).toStrictEqual([ + "a", + "b", + { k: "v" }, + "d", + ]); }); it("clamps out-of-range bounds and returns [] for empty ranges", () => { @@ -189,286 +193,8 @@ describe("range deep reads", () => { expect(() => new LoroList().getRangeValue(0, 1)).toThrow(); expect(() => new LoroList().getRangeDeepValueWithID(0, 1)).toThrow(); expect(() => new LoroMovableList().getRangeValue(0, 1)).toThrow(); - expect(() => new LoroMovableList().getRangeDeepValueWithID(0, 1)).toThrow(); - }); -}); - -describe("getDeepValueJson", () => { - const setupAll = () => { - const doc = new LoroDoc(); - const map = doc.getMap("map"); - map.set("flag", true); - map.set("n", 42); - const text = map.setContainer("text", new LoroText()); - text.insert(0, "Hello"); - const list = doc.getList("list"); - list.insert(0, "a"); - const sub = list.insertContainer(1, new LoroMap()); - sub.set("k", "v"); - const movable = doc.getMovableList("movable"); - movable.insert(0, "x"); - const tree = doc.getTree("tree"); - const root = tree.createNode(); - root.data.set("name", "root"); - const child = root.createNode(); - child.data.set("name", "child"); - const counter = doc.getCounter("counter"); - counter.increment(2.5); - doc.commit(); - return { doc, map, text, list, sub, movable, tree, counter }; - }; - - it("JSON.parse matches toJSON for a doc with every container type", () => { - const { doc, map, text, list, movable, tree, counter } = setupAll(); - expect(JSON.parse(doc.getDeepValueJson())).toStrictEqual(doc.toJSON()); - expect(JSON.parse(map.getDeepValueJson())).toStrictEqual(map.toJSON()); - expect(JSON.parse(text.getDeepValueJson())).toStrictEqual(text.toJSON()); - expect(JSON.parse(list.getDeepValueJson())).toStrictEqual(list.toJSON()); - expect(JSON.parse(movable.getDeepValueJson())).toStrictEqual( - movable.toJSON(), - ); - expect(JSON.parse(tree.getDeepValueJson())).toStrictEqual(tree.toJSON()); - expect(JSON.parse(counter.getDeepValueJson())).toStrictEqual( - counter.toJSON(), - ); - }); - - it("empty doc serializes to {}", () => { - const doc = new LoroDoc(); - expect(doc.getDeepValueJson()).toBe("{}"); - }); - - it("throws on detached containers instead of trapping", () => { - expect(() => new LoroMap().getDeepValueJson()).toThrow(); - expect(() => new LoroList().getDeepValueJson()).toThrow(); - expect(() => new LoroMovableList().getDeepValueJson()).toThrow(); - expect(() => new LoroTree().getDeepValueJson()).toThrow(); - expect(() => new LoroText().getDeepValueJson()).toThrow(); - expect(() => new LoroMap().getDeepValueJsonWithIds()).toThrow(); - expect(() => new LoroList().getDeepValueJsonWithIds()).toThrow(); - expect(() => new LoroMovableList().getDeepValueJsonWithIds()).toThrow(); - expect(() => new LoroTree().getDeepValueJsonWithIds()).toThrow(); - expect(() => new LoroText().getDeepValueJsonWithIds()).toThrow(); - // Counter mirrors toJSON(): it also works on a detached counter - expect(new LoroCounter().getDeepValueJson()).toBe("0.0"); - }); -}); - -describe("getDeepValueJsonWithIds", () => { - // Walk every parsed JSON value. Identity is determined only by the sparse - // position index, never by a scalar/object shape or a schema guess. - function reattachContainerIds(result: { - json: string; - cids: readonly string[]; - containerPositions: Uint32Array; - }): unknown { - let position = 0; - let next = 0; - const walk = (value: any): unknown => { - const cid = - result.containerPositions[next] === position++ - ? result.cids[next++] - : undefined; - if (Array.isArray(value)) { - for (let i = 0; i < value.length; i++) value[i] = walk(value[i]); - } else if (value !== null && typeof value === "object") { - for (const key of Object.keys(value)) value[key] = walk(value[key]); - } - return cid === undefined ? value : { cid, value }; - }; - const value = walk(JSON.parse(result.json)); - expect(next).toBe(result.cids.length); - expect(next).toBe(result.containerPositions.length); - return value; - } - - const setupAll = () => { - const doc = new LoroDoc(); - const map = doc.getMap("map"); - map.set("flag", true); - map.set("n", 42); - const text = map.setContainer("text", new LoroText()); - text.insert(0, "Hello"); - const list = doc.getList("list"); - list.insert(0, "a"); - const sub = list.insertContainer(1, new LoroMap()); - sub.set("k", "v"); - const movable = doc.getMovableList("movable"); - movable.insert(0, "x"); - const tree = doc.getTree("tree"); - const root = tree.createNode(); - root.data.set("name", "root"); - const child = root.createNode(); - child.data.set("name", "child"); - const counter = doc.getCounter("counter"); - counter.increment(2.5); - doc.commit(); - return { doc, map, text, list, sub, movable, tree, counter }; - }; - - it("doc-level: json + cids reconstruct getDeepValueWithID", () => { - const { doc, map, text, list, sub, movable, tree, counter } = setupAll(); - const result = doc.getDeepValueJsonWithIds(); - const { json, cids } = result; - - // json parses to the same content as toJSON() - expect(JSON.parse(json)).toStrictEqual(doc.toJSON()); - - // cids are the pre-order DFS of the serialized tree (root keys are - // serialized in sorted order: counter, list, map, movable, tree) - expect(cids).toStrictEqual([ - counter.id, - list.id, - sub.id, - map.id, - text.id, - movable.id, - tree.id, - ]); - - expect(reattachContainerIds(result)).toStrictEqual( - doc.getDeepValueWithID(), - ); - }); - - it("per-container: cids[0] is the container id and the walk round-trips", () => { - const { map, text, list, movable, tree, counter } = setupAll(); - const containers = [ - ["map", map], - ["text", text], - ["list", list], - ["movable", movable], - ["tree", tree], - ["counter", counter], - ] as const; - for (const [name, container] of containers) { - const result = container.getDeepValueJsonWithIds(); - const { json, cids } = result; - expect(cids[0], name).toBe(container.id); - expect(JSON.parse(json), name).toStrictEqual(container.toJSON()); - // LoroCounter has no getDeepValueWithID(); its node shape is trivially - // { cid: counter.id, value: number } - if (name === "counter") { - expect(cids, name).toStrictEqual([counter.id]); - continue; - } - expect(reattachContainerIds(result), name).toStrictEqual( - container.getDeepValueWithID(), - ); - } - }); - - it("distinguishes scalar strings from Text containers at either position", () => { - const results = ["a", "b"].map((textKey) => { - const doc = new LoroDoc(); - doc.setPeerId("1"); - const map = doc.getMap("m"); - map.setContainer(textKey, new LoroText()).insert(0, "same"); - map.set(textKey === "a" ? "b" : "a", "same"); - const result = doc.getDeepValueJsonWithIds(); - expect(reattachContainerIds(result)).toStrictEqual( - doc.getDeepValueWithID(), - ); - return result; - }); - expect(results[0].json).toBe(results[1].json); - expect(results[0].cids).toStrictEqual(results[1].cids); - expect([...results[0].containerPositions]).toStrictEqual([1, 2]); - expect([...results[1].containerPositions]).toStrictEqual([1, 3]); - }); - - it("follows JS integer-key order at document and nested map levels", () => { - const doc = new LoroDoc(); - for (const key of [ - "10", - "2", - "01", - "0", - "4294967294", - "4294967295", - "-1", - ]) { - const map = doc.getMap(key); - map.setContainer("10", new LoroText()).insert(0, "ten"); - map.setContainer("2", new LoroText()).insert(0, "two"); - map.set("0", "plain"); - } - const result = doc.getDeepValueJsonWithIds(); - expect(reattachContainerIds(result)).toStrictEqual( - doc.getDeepValueWithID(), - ); - expect(JSON.parse(result.json)).toStrictEqual(doc.toJSON()); - }); - - it("preserves cid/value lookalikes and counts plain nested data and binary items", () => { - const doc = new LoroDoc(); - const map = doc.getMap("m"); - map.set("a", { cid: "cid:root-fake:Text", value: "ordinary data" }); - map.set("b", { - "10": [1, "literal"], - "2": { cid: "cid:root-fake:Map", value: {} }, - }); - map.set("c", new Uint8Array([3, 4, 5])); - const text = map.setContainer("z", new LoroText()); - text.insert(0, "actual container"); - const result = doc.getDeepValueJsonWithIds(); - expect(result.cids).toStrictEqual([map.id, text.id]); - expect(JSON.parse(result.json)).toStrictEqual( - JSON.parse(doc.getDeepValueJson()), - ); - // Binary's JSON representation is an array rather than toJSON's Uint8Array. - const rebuilt = reattachContainerIds(result) as any; - expect(rebuilt.m.value.a).toStrictEqual({ - cid: "cid:root-fake:Text", - value: "ordinary data", - }); - expect(rebuilt.m.value.z).toStrictEqual({ - cid: text.id, - value: "actual container", - }); - }); - - it("keeps __proto__ as an own JSON property without changing prototypes", () => { - const doc = new LoroDoc(); - doc.getText("__proto__").insert(0, "data"); - const result = doc.getDeepValueJsonWithIds(); - const parsed = JSON.parse(result.json); - const restored = reattachContainerIds(result) as Record; - expect(Object.getPrototypeOf(parsed)).toBe(Object.prototype); - expect(Object.getPrototypeOf(restored)).toBe(Object.prototype); - expect(Object.prototype.hasOwnProperty.call(restored, "__proto__")).toBe( - true, - ); - expect(restored.__proto__).toStrictEqual({ - cid: "cid:root-__proto__:Text", - value: "data", - }); - }); - - it("preserves mixed scalar/container arrays and plain empty values", () => { - const doc = new LoroDoc(); - const list = doc.getList("list"); - list.push("same"); - list.pushContainer(new LoroText()).insert(0, "same"); - list.push({}); - list.pushContainer(new LoroMap()); - list.push([]); - list.pushContainer(new LoroList()); - list.push(null); - list.push(1.5); - list.pushContainer(new LoroCounter()).increment(1.5); - const result = list.getDeepValueJsonWithIds(); - expect(reattachContainerIds(result)).toStrictEqual( - list.getDeepValueWithID(), - ); - expect(result.containerPositions[0]).toBe(0); - }); - - it("empty doc yields {} and no cids", () => { - const doc = new LoroDoc(); - const result = doc.getDeepValueJsonWithIds(); - const { json, cids } = result; - expect(json).toBe("{}"); - expect(cids).toStrictEqual([]); + expect(() => + new LoroMovableList().getRangeDeepValueWithID(0, 1), + ).toThrow(); }); }); diff --git a/crates/loro-wasm/tests/read_state.test.ts b/crates/loro-wasm/tests/read_state.test.ts new file mode 100644 index 000000000..a5f180edb --- /dev/null +++ b/crates/loro-wasm/tests/read_state.test.ts @@ -0,0 +1,236 @@ +import { describe, expect, it } from "vitest"; +import { + LoroDoc, + LoroMap, + LoroText, + LoroList, + LoroMovableList, + type ContainerState, + type StateNode, +} from "../bundler/index"; + +function project(node: StateNode): unknown { + switch (node.type) { + case "Value": + return node.value; + case "Map": + return Object.fromEntries( + Object.entries(node.value).map(([k, v]) => [k, project(v)]), + ); + case "List": + case "MovableList": + return node.value.map(project); + case "Tree": { + const visit = (nodes: typeof node.value): unknown[] => + nodes.map((n) => ({ + ...n, + meta: project(n.meta), + children: visit(n.children), + })); + return visit(node.value); + } + default: + return node.value; + } +} +function ids(node: StateNode): string[] { + if (node.type === "Value") return []; + if (node.type === "Map") + return [node.cid, ...Object.values(node.value).flatMap(ids)]; + if (node.type === "List" || node.type === "MovableList") + return [node.cid, ...node.value.flatMap(ids)]; + return [node.cid]; +} + +describe("readState", () => { + it("distinguishes containers from opaque values and preserves exact IDs", () => { + const d = new LoroDoc(); + d.setPeerId("18446744073709551614"); + const m = d.getMap("root"); + const fake = { + type: "Map", + cid: "cid:root-fake:Map", + value: { nested: [null, true, "🙂"] }, + }; + m.set("fake", fake); + m.set("__proto__", { safe: true }); + m.set("bytes", new Uint8Array([0, 255, 128])); + m.set("number", Infinity); + m.set("nan", NaN); + const l = m.setContainer("list", new LoroMovableList()); + const t = l.pushContainer(new LoroText()); + t.insert(0, "\uFEFFhello中🙂"); + const merged = m.ensureMergeableMap("merged"); + merged.set("ok", true); + const read = d.readState(); + expect(project(read.root)).toStrictEqual({ + ...m.toJSON(), + ["__proto__"]: { safe: true }, + }); + expect(ids(read.root).sort()).toEqual([m.id, l.id, t.id, merged.id].sort()); + const root = read.root as Extract; + expect(root.value.fake).toEqual({ type: "Value", value: fake }); + expect(Object.prototype.hasOwnProperty.call(root.value, "__proto__")).toBe( + true, + ); + expect(Object.getPrototypeOf(root.value)).toBe(Object.prototype); + expect(d.readState({ container: m.id })).toStrictEqual(root); + const bytes = root.value.bytes as { type: "Value"; value: Uint8Array }; + bytes.value[0] = 42; + expect(m.get("bytes")).toEqual(new Uint8Array([0, 255, 128])); + d.free(); + expect(bytes.value[1]).toBe(255); + }); + + it("includes Tree metadata IDs and rich text formatting", () => { + const d = new LoroDoc(); + const tree = d.getTree("tree"); + const node = tree.createNode(); + const text = node.data.setContainer("body", new LoroText()); + text.insert(0, "hello"); + text.mark({ start: 0, end: 5 }, "bold", true); + node.createNode().data.set("title", "child"); + const plain = d.readState().tree; + expect(project(plain)).toEqual(d.toJSON().tree); + const rich = d.readState({ container: tree.id, text: "delta" }); + if (rich.type !== "Tree") throw new Error("expected Tree"); + expect(rich.value[0].meta.cid).toBe(node.data.id); + expect(rich.value[0].meta.value.body).toEqual({ + type: "Text", + cid: text.id, + value: text.toDelta(), + }); + }); + + it("reads only the selected list subtrees and clamps ranges", () => { + for (const list of [new LoroList(), new LoroMovableList()]) { + const d = new LoroDoc(); + const l = d.getMap("root").setContainer("list", list); + l.push(1); + l.pushContainer(new LoroMap()).set("x", 2); + l.push("end"); + const full = d.readState({ container: l.id }); + if (full.type !== "List" && full.type !== "MovableList") + throw new Error("expected list"); + expect( + d.readState({ container: l.id, range: { start: 1, end: 2 } }), + ).toEqual({ ...full, value: full.value.slice(1, 2) }); + for (const range of [ + { start: 3, end: 99 }, + { start: 2, end: 1 }, + { start: 99, end: 100 }, + ]) { + expect(d.readState({ container: l.id, range })).toEqual({ + ...full, + value: [], + }); + } + } + }); + + it("rejects invalid inputs and remains usable", () => { + const d = new LoroDoc(); + d.getMap("root").set("ok", true); + const read = d.readState.bind(d) as (o: unknown) => unknown; + for (const options of [ + { container: "bad" }, + { container: "cid:99@88:Map" }, + { text: "html" }, + { range: { start: 0, end: 1 } }, + { container: "cid:root-root:Map", range: { start: 0, end: 1 } }, + { container: "cid:root-l:List", range: { start: -1, end: 1 } }, + ]) { + expect(() => read(options)).toThrow(); + expect(project(d.readState().root)).toEqual({ ok: true }); + } + }); + + it("does not invoke inherited setters or commit pending changes", () => { + const d = new LoroDoc(); + const m = d.getMap("root"); + m.set("__read_state_probe", "data"); + m.set("array", [1, 2]); + let calls = 0; + const vv = d.version().encode(); + Object.defineProperty(Object.prototype, "__read_state_probe", { + configurable: true, + set() { + calls++; + }, + }); + let result: ReturnType; + try { + result = d.readState(); + } finally { + delete (Object.prototype as Record).__read_state_probe; + } + expect(calls).toBe(0); + expect(d.version().encode()).toEqual(vv); + expect(project((result as Record).root)).toEqual( + m.toJSON(), + ); + }); + + it("bounds recursive traversal and leaves the document usable after an error", () => { + const d = new LoroDoc(); + let m = d.getMap("deep"); + for (let i = 0; i < 270; i++) m = m.setContainer("child", new LoroMap()); + expect(() => d.readState()).toThrow("nesting"); + expect(d.readState({ container: m.id })).toEqual({ + type: "Map", + cid: m.id, + value: {}, + }); + }); + + it("constructs dense arrays without invoking inherited index setters", () => { + const d = new LoroDoc(); + d.getList("list").push([1, 2]); + let calls = 0; + let result: unknown; + Object.defineProperty(Array.prototype, "0", { + configurable: true, + set() { + calls++; + }, + }); + try { + result = d.readState(); + } finally { + delete (Array.prototype as unknown as Record)["0"]; + } + expect(calls).toBe(0); + expect(result).toEqual({ + list: { + type: "List", + cid: "cid:root-list:List", + value: [{ type: "Value", value: [1, 2] }], + }, + }); + }); + + it("honors root visibility and returns empty implicit roots", () => { + const d = new LoroDoc(); + expect(d.readState({ container: "cid:root-empty:Map" })).toEqual({ + type: "Map", + cid: "cid:root-empty:Map", + value: {}, + }); + expect( + Object.fromEntries( + Object.entries(d.readState()).map(([k, v]) => [k, project(v)]), + ), + ).toEqual(d.toJSON()); + d.getMap("empty"); + d.setHideEmptyRootContainers(true); + expect(d.readState()).toEqual({}); + d.getCounter("counter").increment(3); + expect(d.readState().counter).toEqual({ + type: "Counter", + cid: "cid:root-counter:Counter", + value: 3, + }); + d.deleteRootContainer("cid:root-counter:Counter"); + expect(d.readState()).toEqual({}); + }); +}); diff --git a/package.json b/package.json index 70a66a94e..f7aae33fe 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "test-bundlers-next": "pnpm --dir examples/bundler-smoke-tests run test:next", "run-fuzz-corpus": "node ./scripts/cargo-fuzz-run.mjs all -- -max_total_time=1", "bench-wasm-container-id": "node --expose-gc ./crates/loro-wasm/scripts/measure-container-id.cjs", - "bench-deep-value-json": "node --expose-gc ./crates/loro-wasm/scripts/measure-deep-value-json.cjs", "fix": "cargo clippy --fix --features=test_utils", "vet": "cargo vet", "release-rust": "deno run -A ./scripts/cargo-release.ts" From 83cd47d72a38f87fe62243e90d9f06e29681dad7 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Sun, 6 Sep 2026 01:05:27 +0800 Subject: [PATCH 4/7] feat(wasm): expose recursive container tree API --- .changeset/wasm-read-state.md | 2 +- context/wasm-bulk-read.md | 21 +- crates/loro-internal/src/state/read_state.rs | 53 ++- crates/loro-wasm/AGENTS.md | 7 +- crates/loro-wasm/src/read_state.rs | 325 ++++++++++++++---- ...d_state.test.ts => container_tree.test.ts} | 165 +++++++-- 6 files changed, 453 insertions(+), 120 deletions(-) rename crates/loro-wasm/tests/{read_state.test.ts => container_tree.test.ts} (54%) diff --git a/.changeset/wasm-read-state.md b/.changeset/wasm-read-state.md index d52acd233..8baacd4bf 100644 --- a/.changeset/wasm-read-state.md +++ b/.changeset/wasm-read-state.md @@ -2,4 +2,4 @@ "loro-crdt": minor --- -Add `LoroDoc.readState()` for reading nested container snapshots with explicit `type`, `cid`, and `value` fields. Ordinary values are opaque `Value` nodes, so Map/List data cannot be confused with containers. The same API reads an individual container or a clamped list interval. Text can return plain strings or formatting deltas; Tree node metadata preserves its Map ID and nested containers. JavaScript values are built directly with fixed constructors, per-read key/peer reuse, and owned binary buffers. +Add `toContainerTree()` to documents and attached containers. It returns independent recursive `{type, cid, value}` nodes and opaque ordinary `Value` nodes. Documents can select visible roots without creating missing roots. Text formatting applies to all descendants and is inferred in TypeScript. List and MovableList expose `toContainerTreeSlice(start, end)` with explicit `start`, `totalLength`, and `items`. Fixed JS construction, per-read key/peer reuse, and owned binary buffers avoid a public transport protocol. diff --git a/context/wasm-bulk-read.md b/context/wasm-bulk-read.md index 37e4faff0..c410ff01a 100644 --- a/context/wasm-bulk-read.md +++ b/context/wasm-bulk-read.md @@ -1,22 +1,22 @@ # Structured WASM reads and the performance stack -Verified against code 2026-09-05. +Verified against code 2026-09-06. ## Responsibility - #1093 bounds retained decoded container state; it benefits existing handle reads. - #1085 caches wrapper kind; #1086 supplies legacy subtree/range reads with IDs. -- #1087 adds `LoroDoc.readState` to construct a consumer-ready JS snapshot directly. +- #1087 adds `LoroDoc.toContainerTree` to construct a consumer-ready JS snapshot directly. - #1090/#1091 concern shallow snapshot causal boundaries and constructing their root state. They optimize history import/export, independently of this read API. ## Contract ```ts -const roots = doc.readState(); -const subtree = doc.readState({container: map.id}); -const window = doc.readState({container: list.id, range: {start: 20, end: 40}}); -const formatted = doc.readState({container: text.id, text: "delta"}); +const roots = doc.toContainerTree(); +const subtree = map.toContainerTree(); +const window = list.toContainerTreeSlice(20, 40); +const formatted = text.toContainerTree({text: "delta"}); ``` Containers are `{type, cid, value}`. Map values and List/MovableList items are @@ -29,12 +29,11 @@ fields follow the same node contract. Counter contains a number. Identity is emitted only at real CRDT edges, including mergeable Map markers and Tree metadata edges. No keys, scalar values, paths, traversal positions, or user object shapes are used to guess identity. Full-document reads obey root -visibility. Explicit root IDs can read empty implicit roots. Missing normal or -mergeable containers and unknown container types return errors. Reads do not +visibility. An attached root handle can read its empty root. Detached handles and unknown container types return errors. Document roots selection uses visible root names; missing names are omitted without creating roots, duplicates are ignored, and an empty selection returns {}. Reads do not commit; caller mutations of returned objects/buffers do not mutate the document. -Ranges require List/MovableList container IDs and nonnegative u32 integer bounds; -end is exclusive, bounds clamp, inverted ranges are empty. Only selected child +List/MovableList.toContainerTreeSlice requires nonnegative u32 integer bounds; +end is exclusive, bounds clamp, inverted ranges are empty. It returns {cid, start, totalLength, items}, not a full container node; metadata and items are read under one state lock. Only selected child subtrees are traversed, though obtaining the parent shallow list is still O(N). Read traversal rejects nesting above 256 levels. JS number conversion follows existing reads (including i64 rounding, NaN and infinity); Binary is Uint8Array. @@ -99,3 +98,5 @@ Pre-creating dense array slots likewise gave no material improvement. The shippe builder retains fixed wrapper constructors, per-read peer/key reuse, and safe own property writes. These timings describe the tested machines/workload, not a general speed guarantee. Mirror's Tree normalization still uses its existing handle path. + +Document toContainerTree({roots}) filters before reading root values; the optional text format applies recursively. TypeScript infers both the receiver kind and Text value format. Mirror selects schema roots and, when preserving unknown roots, includes them while excluding explicit Ignore roots. diff --git a/crates/loro-internal/src/state/read_state.rs b/crates/loro-internal/src/state/read_state.rs index 88edebf5f..ba12b00ff 100644 --- a/crates/loro-internal/src/state/read_state.rs +++ b/crates/loro-internal/src/state/read_state.rs @@ -31,26 +31,25 @@ impl DocState { s: &mut S, cid: Option<&ContainerID>, rich: bool, - range: Option<(usize, usize)>, + selected_roots: Option<&[String]>, ) -> LoroResult<()> { if let Some(id) = cid { let idx = self.arena.register_container(id); - if range.is_some() - && !matches!( - idx.get_type(), - ContainerType::List | ContainerType::MovableList - ) - { - return Err(err("readState range requires a List or MovableList")); - } - return self.read_state_container(s, idx, rich, range, None, 0); - } - if range.is_some() { - return Err(err("readState range requires a container")); + return self.read_state_container(s, idx, rich, None, None, 0); } let roots = self.preferred_root_containers(); let mut visible = Vec::new(); + let selected: Option> = + selected_roots.map(|names| names.iter().map(String::as_str).collect()); for idx in roots { + if let Some(names) = &selected { + let name = self + .root_container_name(idx) + .ok_or_else(|| err("Missing root name"))?; + if !names.contains(name.as_str()) { + continue; + } + } if matches!(idx.get_type(), ContainerType::Unknown(_)) { return Err(err("Unsupported container type")); } @@ -90,6 +89,32 @@ impl DocState { } s.emit(Event::End) } + /// Read a list window and its coordinates under the same state lock. + pub fn read_state_slice( + &mut self, + sink: &mut S, + cid: &ContainerID, + rich: bool, + start: usize, + end: usize, + ) -> LoroResult<(usize, usize)> { + let idx = self.arena.register_container(cid); + let kind = idx.get_type(); + if !matches!(kind, ContainerType::List | ContainerType::MovableList) { + return Err(err("Expected list container")); + } + let value = self + .store + .try_get_value_ephemeral(idx)? + .unwrap_or_else(|| kind.default_value()); + let total = value + .as_list() + .ok_or_else(|| err("Expected list value"))? + .len(); + let start = start.min(total); + self.read_state_container(sink, idx, rich, Some((start, end)), Some(value), 0)?; + Ok((start, total)) + } fn read_state_container( &mut self, s: &mut S, @@ -237,7 +262,7 @@ fn raw(s: &mut S, v: &LoroValue, depth: usize) -> LoroResult<()> { fn check_depth(depth: usize) -> LoroResult<()> { if depth > 256 { - Err(err("readState nesting exceeds 256 levels")) + Err(err("toContainerTree nesting exceeds 256 levels")) } else { Ok(()) } diff --git a/crates/loro-wasm/AGENTS.md b/crates/loro-wasm/AGENTS.md index 8f3951a0a..e83746ccb 100644 --- a/crates/loro-wasm/AGENTS.md +++ b/crates/loro-wasm/AGENTS.md @@ -88,6 +88,11 @@ are documented in bundler smoke tests aligned with these expectations. - If package output or published behavior changes, add a changeset. -`LoroDoc.readState` constructs typed nested snapshots with fixed JS helpers. See +`LoroDoc.toContainerTree` constructs typed nested snapshots with fixed JS helpers. See [context/wasm-bulk-read.md](../../context/wasm-bulk-read.md) for identity, ownership, range and Mirror integration contracts. + +`toContainerTree` on attached containers recursively applies its text format. +Document `roots` filters before reading values; missing roots are omitted. List +`toContainerTreeSlice` returns coordinates and items under one state lock, never +a partial ContainerNode. Keep receiver and text-format inference in TypeScript. diff --git a/crates/loro-wasm/src/read_state.rs b/crates/loro-wasm/src/read_state.rs index f7074a26a..a80336bfb 100644 --- a/crates/loro-wasm/src/read_state.rs +++ b/crates/loro-wasm/src/read_state.rs @@ -4,6 +4,15 @@ use loro_internal::read_state::{err, Event, Sink}; use std::collections::HashMap; #[wasm_bindgen(inline_js = " const kinds = ['Map', 'List', 'MovableList', 'Text', 'Tree', 'Counter']; +export function stateOptions(options, document) { + if (typeof options !== 'object' || options === null || Array.isArray(options)) throw new Error('Invalid toContainerTree options'); + for (const key of Object.keys(options)) { + if (key !== 'text' && !(document && key === 'roots')) throw new Error('Unknown toContainerTree option: ' + key); + } + if (document && options.roots !== undefined && !Array.isArray(options.roots)) throw new Error('roots must be an array'); + return options; +} +export function stateSlice(node, start, totalLength) { return {cid:node.cid, start, totalLength, items:node.value}; } export function stateContainer(kind, cid, value) { return {type: kinds[kind], cid, value}; } export function stateCid(peer,counter,kind) { return 'cid:'+counter+'@'+peer+':'+kinds[kind]; } export function stateRootCid(name,kind) { return 'cid:root-'+name+':'+kinds[kind]; } @@ -13,6 +22,9 @@ export function stateIndex(array, index, value) { Object.defineProperty(array, i export function stateSet(object, key, value) { Object.defineProperty(object, key, {value, enumerable: true, writable: true, configurable: true}); } ")] extern "C" { + #[wasm_bindgen(catch)] + fn stateOptions(options: JsValue, document: bool) -> Result; + fn stateSlice(node: JsValue, start: u32, total_length: u32) -> JsValue; fn stateContainer(kind: u8, cid: &JsValue, value: JsValue) -> JsValue; fn stateValue(value: JsValue) -> JsValue; fn stateBinary(bytes: &[u8]) -> JsValue; @@ -166,11 +178,17 @@ impl Sink for FixedSink { } #[derive(Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] struct Options { - container: Option, #[serde(default)] text: TextMode, - range: Option, +} +#[derive(Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct DocumentOptions { + #[serde(default)] + text: TextMode, + roots: Option>, } #[derive(Default, serde::Deserialize)] #[serde(rename_all = "lowercase")] @@ -179,85 +197,264 @@ enum TextMode { Plain, Delta, } -#[derive(serde::Deserialize)] -struct Range { - start: u32, - end: u32, +fn options( + value: Option, + document: bool, +) -> JsResult { + match value { + None => Ok(T::default()), + Some(value) => serde_wasm_bindgen::from_value(stateOptions(value, document)?) + .map_err(|e| JsValue::from_str(&format!("Invalid toContainerTree options: {e}"))), + } +} +fn container_tree( + handler: &H, + opts: Option, +) -> JsResult { + let opts: Options = options(opts, false)?; + let doc = handler + .doc() + .ok_or_else(|| JsValue::from_str("toContainerTree requires an attached container"))?; + let mut sink = FixedSink::default(); + doc.app_state().lock().read_state( + &mut sink, + Some(&handler.id()), + matches!(opts.text, TextMode::Delta), + None, + )?; + Ok(sink.0.root.unchecked_into()) +} +fn list_slice( + handler: &H, + start: f64, + end: f64, + opts: Option, +) -> JsResult { + if [start, end] + .iter() + .any(|n| !n.is_finite() || *n < 0.0 || *n > u32::MAX as f64 || n.fract() != 0.0) + { + return Err(JsValue::from_str( + "Slice bounds must be nonnegative u32 integers", + )); + } + let opts: Options = options(opts, false)?; + let doc = handler + .doc() + .ok_or_else(|| JsValue::from_str("toContainerTreeSlice requires an attached container"))?; + let mut sink = FixedSink::default(); + let (start, total) = doc.app_state().lock().read_state_slice( + &mut sink, + &handler.id(), + matches!(opts.text, TextMode::Delta), + start as usize, + end as usize, + )?; + Ok(stateSlice(sink.0.root, start as u32, total as u32).unchecked_into()) } - #[wasm_bindgen] impl LoroDoc { - /// Read nested container state with explicit IDs and opaque ordinary values. - /// Reads current state without committing. Text defaults to plain strings. - /// A list range is end-exclusive and clamped; inverted ranges are empty. - /// Throws for invalid options, missing containers, unsupported container types, - /// or nesting exceeding 256 levels. Returned objects are independent snapshots. - #[wasm_bindgen(js_name = readState, skip_typescript)] - pub fn read_state(&self, options: Option) -> JsResult { - let opts: Options = match options { - None => Options::default(), - Some(v) => serde_wasm_bindgen::from_value(v.into()) - .map_err(|e| JsValue::from_str(&format!("Invalid readState options: {e}")))?, - }; - let cid = opts - .container - .as_deref() - .map(ContainerID::try_from) - .transpose() - .map_err(|_| JsValue::from_str("Invalid container ID"))?; - if cid.as_ref().is_some_and(|id| !self.doc.has_container(id)) { - return Err(JsValue::from_str("The container does not exist in the doc")); - } + /// Convert visible roots to independent container trees without committing. + /// The text format applies recursively, including Tree metadata. Unknown or + /// hidden roots are omitted; roots: [] returns an empty object. Reads never + /// create roots. Binary values are owned Uint8Arrays. Not a CRDT export. + #[wasm_bindgen(js_name = toContainerTree, skip_typescript)] + pub fn to_container_tree( + &self, + opts: Option, + ) -> JsResult { + let opts: DocumentOptions = options(opts.map(Into::into), true)?; let mut sink = FixedSink::default(); self.doc.app_state().lock().read_state( &mut sink, - cid.as_ref(), + None, matches!(opts.text, TextMode::Delta), - opts.range.map(|r| (r.start as usize, r.end as usize)), + opts.roots.as_deref(), )?; Ok(sink.0.root.unchecked_into()) } } #[wasm_bindgen] +impl LoroMap { + /// Convert this attached container and all descendants to an independent tree. + /// Text format applies recursively. Does not commit; throws for detached + /// containers, unsupported types, invalid options, or nesting above 256. + #[wasm_bindgen(js_name = toContainerTree, skip_typescript)] + pub fn to_container_tree( + &self, + opts: Option, + ) -> JsResult { + container_tree(&self.handler, opts.map(Into::into)) + } +} +#[wasm_bindgen] +impl LoroList { + /// Convert this attached container and all descendants to an independent tree. + /// Text format applies recursively. Does not commit; throws for detached + /// containers, unsupported types, invalid options, or nesting above 256. + #[wasm_bindgen(js_name = toContainerTree, skip_typescript)] + pub fn to_container_tree( + &self, + opts: Option, + ) -> JsResult { + container_tree(&self.handler, opts.map(Into::into)) + } +} +#[wasm_bindgen] +impl LoroList { + /// Read [start,end), clamped to list length. Inverted bounds return no items. + /// Returns actual start, totalLength and items; this is not a complete container. + /// Only selected descendants are traversed; parent shallow list access is O(N). + #[wasm_bindgen(js_name = toContainerTreeSlice, skip_typescript)] + pub fn to_container_tree_slice( + &self, + start: f64, + end: f64, + opts: Option, + ) -> JsResult { + list_slice(&self.handler, start, end, opts.map(Into::into)) + } +} +#[wasm_bindgen] +impl LoroMovableList { + /// Convert this attached container and all descendants to an independent tree. + /// Text format applies recursively. Does not commit; throws for detached + /// containers, unsupported types, invalid options, or nesting above 256. + #[wasm_bindgen(js_name = toContainerTree, skip_typescript)] + pub fn to_container_tree( + &self, + opts: Option, + ) -> JsResult { + container_tree(&self.handler, opts.map(Into::into)) + } +} +#[wasm_bindgen] +impl LoroMovableList { + /// Read [start,end), clamped to list length. Inverted bounds return no items. + /// Returns actual start, totalLength and items; this is not a complete container. + /// Only selected descendants are traversed; parent shallow list access is O(N). + #[wasm_bindgen(js_name = toContainerTreeSlice, skip_typescript)] + pub fn to_container_tree_slice( + &self, + start: f64, + end: f64, + opts: Option, + ) -> JsResult { + list_slice(&self.handler, start, end, opts.map(Into::into)) + } +} +#[wasm_bindgen] +impl LoroText { + /// Convert this attached container and all descendants to an independent tree. + /// Text format applies recursively. Does not commit; throws for detached + /// containers, unsupported types, invalid options, or nesting above 256. + #[wasm_bindgen(js_name = toContainerTree, skip_typescript)] + pub fn to_container_tree( + &self, + opts: Option, + ) -> JsResult { + container_tree(&self.handler, opts.map(Into::into)) + } +} +#[wasm_bindgen] +impl LoroTree { + /// Convert this attached container and all descendants to an independent tree. + /// Text format applies recursively. Does not commit; throws for detached + /// containers, unsupported types, invalid options, or nesting above 256. + #[wasm_bindgen(js_name = toContainerTree, skip_typescript)] + pub fn to_container_tree( + &self, + opts: Option, + ) -> JsResult { + container_tree(&self.handler, opts.map(Into::into)) + } +} +#[wasm_bindgen] +impl LoroCounter { + /// Convert this attached container and all descendants to an independent tree. + /// Text format applies recursively. Does not commit; throws for detached + /// containers, unsupported types, invalid options, or nesting above 256. + #[wasm_bindgen(js_name = toContainerTree, skip_typescript)] + pub fn to_container_tree( + &self, + opts: Option, + ) -> JsResult { + container_tree(&self.handler, opts.map(Into::into)) + } +} +#[wasm_bindgen] extern "C" { - #[wasm_bindgen(typescript_type = "ReadStateOptions")] - pub type JsReadStateOptions; - #[wasm_bindgen(typescript_type = "ContainerState | Record")] - pub type JsReadState; + #[wasm_bindgen(typescript_type = "ContainerTreeOptions")] + pub type JsContainerTreeOptions; + #[wasm_bindgen(typescript_type = "DocumentContainerTreeOptions")] + pub type JsDocumentContainerTreeOptions; + #[wasm_bindgen(typescript_type = "ContainerNode")] + pub type JsContainerNode; + #[wasm_bindgen(typescript_type = "Record")] + pub type JsDocumentContainerTree; + #[wasm_bindgen(typescript_type = "ContainerTreeSlice")] + pub type JsContainerTreeSlice; } #[wasm_bindgen(typescript_custom_section)] const TYPES: &str = r#" +export type ContainerTreeTextFormat = "plain" | "delta"; +/** Text format applies to every descendant Text, including Tree metadata. */ +export interface ContainerTreeOptions { text?: T } +export interface DocumentContainerTreeOptions extends ContainerTreeOptions { + /** Select visible roots by name. Missing roots are omitted, never created. [] selects none. */ + roots?: readonly string[]; +} +/** Opaque ordinary data: never interpret objects inside value as container nodes. */ +export type ValueNode = { type: "Value"; value: Value }; +export type ContainerTreeNode = ContainerNode | ValueNode; +export type ContainerNode = + | { type: "Map"; cid: ContainerID; value: Record> } + | { type: "List"; cid: ContainerID; value: ContainerTreeNode[] } + | { type: "MovableList"; cid: ContainerID; value: ContainerTreeNode[] } + | { type: "Text"; cid: ContainerID; value: T extends "delta" ? Delta[] : string } + | { type: "Tree"; cid: ContainerID; value: TreeNodeSnapshot[] } + | { type: "Counter"; cid: ContainerID; value: number }; +export interface TreeNodeSnapshot { + id: TreeID; parent: TreeID | null; index: number; fractional_index: string; + meta: Extract, {type:"Map"}>; + children: TreeNodeSnapshot[]; +} +/** A partial list, not a complete ContainerNode. start is clamped to totalLength. */ +export interface ContainerTreeSlice { + cid: ContainerID; start: number; totalLength: number; items: ContainerTreeNode[]; +} interface LoroDoc { - /** Read nested container snapshots without committing. Ordinary data stays inside Value nodes. - * Text defaults to strings; use text: "delta" to preserve formatting. - * Throws for invalid options, unsupported containers, or nesting over 256 levels. + /** Convert visible roots to independent container trees. No commit, live handles or CRDT history. + * Text defaults to plain strings. Throws for invalid options, unsupported types or nesting >256. */ - readState(options?: ReadStateOptions & { container?: undefined; range?: never }): Record; - /** Read one container; list ranges are clamped and end-exclusive. */ - readState(options: ReadStateOptions & { container: ContainerID }): ContainerState; - readState(options: ReadStateOptions): ContainerState | Record; -} -/** Options for reading the whole document, a container, or a list interval. */ -export interface ReadStateOptions { - container?: ContainerID; - text?: "plain" | "delta"; - range?: { start: number; end: number }; -} -/** Ordinary data is opaque: never interpret objects inside value as containers. */ -export type StateValue = { type: "Value"; value: Value }; -export type StateNode = ContainerState | StateValue; -export type ContainerState = - | { type: "Map"; cid: ContainerID; value: Record } - | { type: "List" | "MovableList"; cid: ContainerID; value: StateNode[] } - | { type: "Text"; cid: ContainerID; value: string | Delta[] } - | { type: "Tree"; cid: ContainerID; value: StateTreeNode[] } - | { type: "Counter"; cid: ContainerID; value: number }; -export interface StateTreeNode { - id: TreeID; - parent: TreeID | null; - index: number; - fractional_index: string; - meta: Extract; - children: StateTreeNode[]; + toContainerTree(options?: DocumentContainerTreeOptions): Record>; +} +interface LoroMap { + /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ + toContainerTree(options?: ContainerTreeOptions): Extract, {type:"Map"}>; +} +interface LoroList { + /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ + toContainerTree(options?: ContainerTreeOptions): Extract, {type:"List"}>; + /** Read [start,end), clamped, with source coordinates; parent shallow list access remains O(N). */ + toContainerTreeSlice(start:number,end:number,options?:ContainerTreeOptions): ContainerTreeSlice; +} +interface LoroMovableList { + /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ + toContainerTree(options?: ContainerTreeOptions): Extract, {type:"MovableList"}>; + /** Read [start,end), clamped, with source coordinates; parent shallow list access remains O(N). */ + toContainerTreeSlice(start:number,end:number,options?:ContainerTreeOptions): ContainerTreeSlice; +} +interface LoroText { + /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ + toContainerTree(options?: ContainerTreeOptions): Extract, {type:"Text"}>; +} +interface LoroTree { + /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ + toContainerTree(options?: ContainerTreeOptions): Extract, {type:"Tree"}>; +} +interface LoroCounter { + /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ + toContainerTree(options?: ContainerTreeOptions): Extract, {type:"Counter"}>; } "#; diff --git a/crates/loro-wasm/tests/read_state.test.ts b/crates/loro-wasm/tests/container_tree.test.ts similarity index 54% rename from crates/loro-wasm/tests/read_state.test.ts rename to crates/loro-wasm/tests/container_tree.test.ts index a5f180edb..627231fad 100644 --- a/crates/loro-wasm/tests/read_state.test.ts +++ b/crates/loro-wasm/tests/container_tree.test.ts @@ -5,11 +5,13 @@ import { LoroText, LoroList, LoroMovableList, - type ContainerState, - type StateNode, + LoroTree, + LoroCounter, + type ContainerNode, + type ContainerTreeNode, } from "../bundler/index"; -function project(node: StateNode): unknown { +function project(node: ContainerTreeNode): unknown { switch (node.type) { case "Value": return node.value; @@ -33,7 +35,7 @@ function project(node: StateNode): unknown { return node.value; } } -function ids(node: StateNode): string[] { +function ids(node: ContainerTreeNode): string[] { if (node.type === "Value") return []; if (node.type === "Map") return [node.cid, ...Object.values(node.value).flatMap(ids)]; @@ -42,7 +44,7 @@ function ids(node: StateNode): string[] { return [node.cid]; } -describe("readState", () => { +describe("toContainerTree", () => { it("distinguishes containers from opaque values and preserves exact IDs", () => { const d = new LoroDoc(); d.setPeerId("18446744073709551614"); @@ -62,19 +64,19 @@ describe("readState", () => { t.insert(0, "\uFEFFhello中🙂"); const merged = m.ensureMergeableMap("merged"); merged.set("ok", true); - const read = d.readState(); + const read = d.toContainerTree(); expect(project(read.root)).toStrictEqual({ ...m.toJSON(), ["__proto__"]: { safe: true }, }); expect(ids(read.root).sort()).toEqual([m.id, l.id, t.id, merged.id].sort()); - const root = read.root as Extract; + const root = read.root as Extract; expect(root.value.fake).toEqual({ type: "Value", value: fake }); expect(Object.prototype.hasOwnProperty.call(root.value, "__proto__")).toBe( true, ); expect(Object.getPrototypeOf(root.value)).toBe(Object.prototype); - expect(d.readState({ container: m.id })).toStrictEqual(root); + expect(m.toContainerTree()).toStrictEqual(root); const bytes = root.value.bytes as { type: "Value"; value: Uint8Array }; bytes.value[0] = 42; expect(m.get("bytes")).toEqual(new Uint8Array([0, 255, 128])); @@ -90,9 +92,9 @@ describe("readState", () => { text.insert(0, "hello"); text.mark({ start: 0, end: 5 }, "bold", true); node.createNode().data.set("title", "child"); - const plain = d.readState().tree; + const plain = d.toContainerTree().tree; expect(project(plain)).toEqual(d.toJSON().tree); - const rich = d.readState({ container: tree.id, text: "delta" }); + const rich = tree.toContainerTree({ text: "delta" }); if (rich.type !== "Tree") throw new Error("expected Tree"); expect(rich.value[0].meta.cid).toBe(node.data.id); expect(rich.value[0].meta.value.body).toEqual({ @@ -109,20 +111,25 @@ describe("readState", () => { l.push(1); l.pushContainer(new LoroMap()).set("x", 2); l.push("end"); - const full = d.readState({ container: l.id }); + const full = l.toContainerTree(); if (full.type !== "List" && full.type !== "MovableList") throw new Error("expected list"); - expect( - d.readState({ container: l.id, range: { start: 1, end: 2 } }), - ).toEqual({ ...full, value: full.value.slice(1, 2) }); + expect(l.toContainerTreeSlice(1, 2)).toEqual({ + cid: l.id, + start: 1, + totalLength: 3, + items: full.value.slice(1, 2), + }); for (const range of [ { start: 3, end: 99 }, { start: 2, end: 1 }, { start: 99, end: 100 }, ]) { - expect(d.readState({ container: l.id, range })).toEqual({ - ...full, - value: [], + expect(l.toContainerTreeSlice(range.start, range.end)).toEqual({ + cid: l.id, + start: Math.min(range.start, 3), + totalLength: 3, + items: [], }); } } @@ -131,7 +138,7 @@ describe("readState", () => { it("rejects invalid inputs and remains usable", () => { const d = new LoroDoc(); d.getMap("root").set("ok", true); - const read = d.readState.bind(d) as (o: unknown) => unknown; + const read = d.toContainerTree.bind(d) as (o: unknown) => unknown; for (const options of [ { container: "bad" }, { container: "cid:99@88:Map" }, @@ -141,7 +148,7 @@ describe("readState", () => { { container: "cid:root-l:List", range: { start: -1, end: 1 } }, ]) { expect(() => read(options)).toThrow(); - expect(project(d.readState().root)).toEqual({ ok: true }); + expect(project(d.toContainerTree().root)).toEqual({ ok: true }); } }); @@ -158,15 +165,15 @@ describe("readState", () => { calls++; }, }); - let result: ReturnType; + let result: ReturnType; try { - result = d.readState(); + result = d.toContainerTree(); } finally { delete (Object.prototype as Record).__read_state_probe; } expect(calls).toBe(0); expect(d.version().encode()).toEqual(vv); - expect(project((result as Record).root)).toEqual( + expect(project((result as Record).root)).toEqual( m.toJSON(), ); }); @@ -175,8 +182,8 @@ describe("readState", () => { const d = new LoroDoc(); let m = d.getMap("deep"); for (let i = 0; i < 270; i++) m = m.setContainer("child", new LoroMap()); - expect(() => d.readState()).toThrow("nesting"); - expect(d.readState({ container: m.id })).toEqual({ + expect(() => d.toContainerTree()).toThrow("nesting"); + expect(m.toContainerTree()).toEqual({ type: "Map", cid: m.id, value: {}, @@ -195,7 +202,7 @@ describe("readState", () => { }, }); try { - result = d.readState(); + result = d.toContainerTree(); } finally { delete (Array.prototype as unknown as Record)["0"]; } @@ -211,26 +218,124 @@ describe("readState", () => { it("honors root visibility and returns empty implicit roots", () => { const d = new LoroDoc(); - expect(d.readState({ container: "cid:root-empty:Map" })).toEqual({ + expect(d.getMap("empty").toContainerTree()).toEqual({ type: "Map", cid: "cid:root-empty:Map", value: {}, }); expect( Object.fromEntries( - Object.entries(d.readState()).map(([k, v]) => [k, project(v)]), + Object.entries(d.toContainerTree()).map(([k, v]) => [k, project(v)]), ), ).toEqual(d.toJSON()); d.getMap("empty"); d.setHideEmptyRootContainers(true); - expect(d.readState()).toEqual({}); + expect(d.toContainerTree()).toEqual({}); d.getCounter("counter").increment(3); - expect(d.readState().counter).toEqual({ + expect(d.toContainerTree().counter).toEqual({ type: "Counter", cid: "cid:root-counter:Counter", value: 3, }); d.deleteRootContainer("cid:root-counter:Counter"); - expect(d.readState()).toEqual({}); + expect(d.toContainerTree()).toEqual({}); }); }); + +it("selects roots without traversing excluded subtrees or creating missing roots", () => { + const d = new LoroDoc(); + d.getMap("selected").set("ok", true); + let deep = d.getMap("excluded"); + for (let i = 0; i < 270; i++) + deep = deep.setContainer("child", new LoroMap()); + const vv = d.version().encode(); + const shallow = d.getShallowValue(); + expect( + d.toContainerTree({ roots: ["selected", "missing", "selected"] }), + ).toEqual({ selected: d.getMap("selected").toContainerTree() }); + expect(d.toContainerTree({ roots: [] })).toEqual({}); + expect(d.getShallowValue()).toEqual(shallow); + expect(d.version().encode()).toEqual(vv); + expect(() => d.toContainerTree()).toThrow("nesting"); +}); +it("propagates text format through maps, lists and Tree metadata", () => { + const d = new LoroDoc(); + const map = d.getMap("root"); + const list = map.setContainer("list", new LoroList()); + const text = list + .pushContainer(new LoroMap()) + .setContainer("text", new LoroText()); + text.insert(0, "hello"); + text.mark({ start: 0, end: 5 }, "bold", true); + const tree = d.getTree("tree"); + const node = tree.createNode(); + node.data.setContainer("text", new LoroText()).insert(0, "child"); + const roots = d.toContainerTree({ text: "delta" }); + expect(roots.root).toEqual(map.toContainerTree({ text: "delta" })); + expect(roots.tree).toEqual(tree.toContainerTree({ text: "delta" })); + const nested = map.toContainerTree({ text: "delta" }).value.list; + if (nested.type !== "List") throw Error("list"); + const child = nested.value[0]; + if (child.type !== "Map") throw Error("map"); + expect(child.value.text).toEqual({ + type: "Text", + cid: text.id, + value: text.toDelta(), + }); + expect(list.toContainerTreeSlice(0, 1, { text: "delta" }).items).toEqual( + nested.value, + ); +}); +it("rejects detached containers and invalid slice bounds", () => { + for (const container of [ + new LoroMap(), + new LoroList(), + new LoroMovableList(), + new LoroText(), + new LoroTree(), + new LoroCounter(), + ]) { + expect(() => container.toContainerTree()).toThrow("attached"); + } + const d = new LoroDoc(); + const list = d.getList("list"); + list.push("ok"); + for (const n of [-1, 0.5, NaN, Infinity, 2 ** 32]) { + expect(() => list.toContainerTreeSlice(n, 1)).toThrow("bounds"); + expect(() => list.toContainerTreeSlice(0, n)).toThrow("bounds"); + } + expect(list.toContainerTreeSlice(0, 99)).toEqual({ + cid: list.id, + start: 0, + totalLength: 1, + items: [{ type: "Value", value: "ok" }], + }); + expect(d.getCounter("counter").toContainerTree()).toEqual({ + type: "Counter", + cid: "cid:root-counter:Counter", + value: 0, + }); +}); +// Compile-only API contracts. This function is deliberately never called. +function checkContainerTreeTypes( + d: LoroDoc, + map: LoroMap, + list: LoroList, + text: LoroText, +) { + const plain: string = text.toContainerTree().value; + const delta: ReturnType = text.toContainerTree({ + text: "delta", + }).value; + const mapKind: "Map" = map.toContainerTree().type; + // @ts-expect-error Document selection is by roots, not a container ID. + d.toContainerTree({ container: map.id }); + // @ts-expect-error Complete tree reads never accept a range. + list.toContainerTree({ range: { start: 0, end: 1 } }); + // @ts-expect-error Containers do not select document roots. + map.toContainerTree({ roots: ["root"] }); + // @ts-expect-error Plain text is not a delta. + const invalid: typeof delta = plain; + return [plain, delta, mapKind, invalid]; +} +void checkContainerTreeTypes; From 02b9a32ebcff5ae656a1fbec848aa01d65b9abac Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Sun, 6 Sep 2026 11:33:00 +0800 Subject: [PATCH 5/7] refactor: align container tree module and file names --- ...m-read-state.md => wasm-container-tree.md} | 0 AGENTS.md | 2 +- ...sm-bulk-read.md => wasm-container-tree.md} | 6 ++-- crates/loro-internal/src/lib.rs | 2 +- crates/loro-internal/src/state.rs | 2 +- crates/loro-internal/src/state/AGENTS.md | 4 +-- .../{read_state.rs => container_tree.rs} | 30 +++++++++---------- crates/loro-wasm/AGENTS.md | 2 +- .../src/{read_state.rs => container_tree.rs} | 8 ++--- crates/loro-wasm/src/lib.rs | 2 +- crates/loro-wasm/tests/container_tree.test.ts | 6 ++-- 11 files changed, 32 insertions(+), 32 deletions(-) rename .changeset/{wasm-read-state.md => wasm-container-tree.md} (100%) rename context/{wasm-bulk-read.md => wasm-container-tree.md} (97%) rename crates/loro-internal/src/state/{read_state.rs => container_tree.rs} (90%) rename crates/loro-wasm/src/{read_state.rs => container_tree.rs} (98%) diff --git a/.changeset/wasm-read-state.md b/.changeset/wasm-container-tree.md similarity index 100% rename from .changeset/wasm-read-state.md rename to .changeset/wasm-container-tree.md diff --git a/AGENTS.md b/AGENTS.md index d1e61fb9c..cb00f0dd4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,4 +98,4 @@ History uses short imperative commits, often prefixed by `fix:`, `test:`, `chore:`, or `refactor:`. PRs should include summary, rationale, validation, and linked issues or traces when relevant. - Structured WASM reads, node identity, ranges and performance responsibilities: - [context/wasm-bulk-read.md](context/wasm-bulk-read.md). + [context/wasm-container-tree.md](context/wasm-container-tree.md). diff --git a/context/wasm-bulk-read.md b/context/wasm-container-tree.md similarity index 97% rename from context/wasm-bulk-read.md rename to context/wasm-container-tree.md index c410ff01a..3814d52a5 100644 --- a/context/wasm-bulk-read.md +++ b/context/wasm-container-tree.md @@ -1,4 +1,4 @@ -# Structured WASM reads and the performance stack +# Container tree API and the performance stack Verified against code 2026-09-06. @@ -42,9 +42,9 @@ inherited setters. Consumers must preserve this when projecting nodes themselves ## Implementation -`state/read_state.rs` emits a traversal to a sink without constructing a deep +`state/container_tree.rs` emits a traversal to a sink without constructing a deep whole-document LoroValue tree. Each container's shallow value is ephemeral; -values fetched to determine root visibility are reused. `loro-wasm/src/read_state.rs` +values fetched to determine root visibility are reused. `loro-wasm/src/container_tree.rs` owns the JS construction stack and uses fixed imported functions for stable wrapper shapes, IDs and own-property writes. Keys and peer decimal strings are cached only for one read; complete CIDs are constructed in JS. Binary buffers diff --git a/crates/loro-internal/src/lib.rs b/crates/loro-internal/src/lib.rs index b27a069bc..83d354239 100644 --- a/crates/loro-internal/src/lib.rs +++ b/crates/loro-internal/src/lib.rs @@ -14,7 +14,7 @@ pub mod diff; pub mod diff_calc; pub mod handler; pub mod sync; -pub use state::read_state; +pub use state::container_tree; use crate::sync::{AtomicBool, AtomicUsize}; use std::sync::Arc; diff --git a/crates/loro-internal/src/state.rs b/crates/loro-internal/src/state.rs index 9fcbd35e4..afdd5e225 100644 --- a/crates/loro-internal/src/state.rs +++ b/crates/loro-internal/src/state.rs @@ -1,4 +1,4 @@ -pub mod read_state; +pub mod container_tree; use crate::sync::{AtomicU64, Mutex, RwLock}; #[cfg(test)] diff --git a/crates/loro-internal/src/state/AGENTS.md b/crates/loro-internal/src/state/AGENTS.md index e79a0a209..60e6c9a28 100644 --- a/crates/loro-internal/src/state/AGENTS.md +++ b/crates/loro-internal/src/state/AGENTS.md @@ -45,6 +45,6 @@ before changing mergeable child behavior. - `cargo test -p loro-internal --test mergeable_container` - `cargo test -p loro-internal import_atomicity` if import or rollback is involved. -`read_state.rs` traverses ephemeral shallow values into a sink. Container identity +`container_tree.rs` traverses ephemeral shallow values into a sink. Container identity comes from CRDT edges, including mergeable markers and Tree metadata; ordinary -values remain opaque. See [bulk reads](../../../../context/wasm-bulk-read.md). +values remain opaque. See [bulk reads](../../../../context/wasm-container-tree.md). diff --git a/crates/loro-internal/src/state/read_state.rs b/crates/loro-internal/src/state/container_tree.rs similarity index 90% rename from crates/loro-internal/src/state/read_state.rs rename to crates/loro-internal/src/state/container_tree.rs index ba12b00ff..2c771dc48 100644 --- a/crates/loro-internal/src/state/read_state.rs +++ b/crates/loro-internal/src/state/container_tree.rs @@ -26,7 +26,7 @@ pub trait Sink { fn value_end(&mut self) -> LoroResult<()>; } impl DocState { - pub fn read_state( + pub fn read_container_tree( &mut self, s: &mut S, cid: Option<&ContainerID>, @@ -35,7 +35,7 @@ impl DocState { ) -> LoroResult<()> { if let Some(id) = cid { let idx = self.arena.register_container(id); - return self.read_state_container(s, idx, rich, None, None, 0); + return self.read_container_node(s, idx, rich, None, None, 0); } let roots = self.preferred_root_containers(); let mut visible = Vec::new(); @@ -85,12 +85,12 @@ impl DocState { s.emit(Event::Object(visible.len()))?; for (key, idx, value) in visible { s.emit(Event::Key(&key))?; - self.read_state_container(s, idx, rich, None, value, 0)?; + self.read_container_node(s, idx, rich, None, value, 0)?; } s.emit(Event::End) } /// Read a list window and its coordinates under the same state lock. - pub fn read_state_slice( + pub fn read_container_tree_slice( &mut self, sink: &mut S, cid: &ContainerID, @@ -112,10 +112,10 @@ impl DocState { .ok_or_else(|| err("Expected list value"))? .len(); let start = start.min(total); - self.read_state_container(sink, idx, rich, Some((start, end)), Some(value), 0)?; + self.read_container_node(sink, idx, rich, Some((start, end)), Some(value), 0)?; Ok((start, total)) } - fn read_state_container( + fn read_container_node( &mut self, s: &mut S, idx: ContainerIdx, @@ -159,9 +159,9 @@ impl DocState { .map(|t| ContainerID::new_mergeable(&id, k, t)); if let Some(c) = merge { let i = self.arena.register_container(&c); - self.read_state_container(s, i, rich, None, None, depth + 1)?; + self.read_container_node(s, i, rich, None, None, depth + 1)?; } else { - self.read_state_edge(s, v, rich, depth + 1)?; + self.read_container_edge(s, v, rich, depth + 1)?; } } s.emit(Event::End)?; @@ -172,16 +172,16 @@ impl DocState { let end = end.min(l.len()).max(start); s.emit(Event::Array(end - start))?; for v in &l[start..end] { - self.read_state_edge(s, v, rich, depth + 1)?; + self.read_container_edge(s, v, rich, depth + 1)?; } s.emit(Event::End)?; } - (_, ContainerType::Tree) => self.read_state_tree(s, &v, rich, depth + 1)?, + (_, ContainerType::Tree) => self.read_tree_nodes(s, &v, rich, depth + 1)?, _ => raw(s, &v, depth + 1)?, }; s.container_end() } - fn read_state_edge( + fn read_container_edge( &mut self, s: &mut S, v: &LoroValue, @@ -190,13 +190,13 @@ impl DocState { ) -> LoroResult<()> { if let LoroValue::Container(cid) = v { let idx = self.arena.register_container(cid); - return self.read_state_container(s, idx, rich, None, None, depth); + return self.read_container_node(s, idx, rich, None, None, depth); } s.value_start()?; raw(s, v, depth + 1)?; s.value_end() } - fn read_state_tree( + fn read_tree_nodes( &mut self, s: &mut S, v: &LoroValue, @@ -213,9 +213,9 @@ impl DocState { for (k, v) in m.iter() { s.emit(Event::Key(k))?; if k == "meta" { - self.read_state_edge(s, v, rich, depth + 1)?; + self.read_container_edge(s, v, rich, depth + 1)?; } else if k == "children" { - self.read_state_tree(s, v, rich, depth + 1)?; + self.read_tree_nodes(s, v, rich, depth + 1)?; } else { raw(s, v, depth + 1)?; } diff --git a/crates/loro-wasm/AGENTS.md b/crates/loro-wasm/AGENTS.md index e83746ccb..ca607ad1b 100644 --- a/crates/loro-wasm/AGENTS.md +++ b/crates/loro-wasm/AGENTS.md @@ -89,7 +89,7 @@ are documented in - If package output or published behavior changes, add a changeset. `LoroDoc.toContainerTree` constructs typed nested snapshots with fixed JS helpers. See -[context/wasm-bulk-read.md](../../context/wasm-bulk-read.md) for identity, ownership, +[context/wasm-container-tree.md](../../context/wasm-container-tree.md) for identity, ownership, range and Mirror integration contracts. `toContainerTree` on attached containers recursively applies its text format. diff --git a/crates/loro-wasm/src/read_state.rs b/crates/loro-wasm/src/container_tree.rs similarity index 98% rename from crates/loro-wasm/src/read_state.rs rename to crates/loro-wasm/src/container_tree.rs index a80336bfb..d80965e8f 100644 --- a/crates/loro-wasm/src/read_state.rs +++ b/crates/loro-wasm/src/container_tree.rs @@ -1,6 +1,6 @@ //! Build JS state with fixed constructors; no callbacks or document-wide intermediary. use super::*; -use loro_internal::read_state::{err, Event, Sink}; +use loro_internal::container_tree::{err, Event, Sink}; use std::collections::HashMap; #[wasm_bindgen(inline_js = " const kinds = ['Map', 'List', 'MovableList', 'Text', 'Tree', 'Counter']; @@ -216,7 +216,7 @@ fn container_tree( .doc() .ok_or_else(|| JsValue::from_str("toContainerTree requires an attached container"))?; let mut sink = FixedSink::default(); - doc.app_state().lock().read_state( + doc.app_state().lock().read_container_tree( &mut sink, Some(&handler.id()), matches!(opts.text, TextMode::Delta), @@ -243,7 +243,7 @@ fn list_slice( .doc() .ok_or_else(|| JsValue::from_str("toContainerTreeSlice requires an attached container"))?; let mut sink = FixedSink::default(); - let (start, total) = doc.app_state().lock().read_state_slice( + let (start, total) = doc.app_state().lock().read_container_tree_slice( &mut sink, &handler.id(), matches!(opts.text, TextMode::Delta), @@ -265,7 +265,7 @@ impl LoroDoc { ) -> JsResult { let opts: DocumentOptions = options(opts.map(Into::into), true)?; let mut sink = FixedSink::default(); - self.doc.app_state().lock().read_state( + self.doc.app_state().lock().read_container_tree( &mut sink, None, matches!(opts.text, TextMode::Delta), diff --git a/crates/loro-wasm/src/lib.rs b/crates/loro-wasm/src/lib.rs index 9fe8ee2fb..7bd4e822b 100644 --- a/crates/loro-wasm/src/lib.rs +++ b/crates/loro-wasm/src/lib.rs @@ -5,7 +5,7 @@ #![allow(clippy::doc_lazy_continuation)] // #![warn(missing_docs)] -mod read_state; +mod container_tree; use convert::{ import_blob_metadata_to_js, import_status_to_js_value, js_diff_to_inner_diff, diff --git a/crates/loro-wasm/tests/container_tree.test.ts b/crates/loro-wasm/tests/container_tree.test.ts index 627231fad..ed056f1a7 100644 --- a/crates/loro-wasm/tests/container_tree.test.ts +++ b/crates/loro-wasm/tests/container_tree.test.ts @@ -155,11 +155,11 @@ describe("toContainerTree", () => { it("does not invoke inherited setters or commit pending changes", () => { const d = new LoroDoc(); const m = d.getMap("root"); - m.set("__read_state_probe", "data"); + m.set("__container_tree_probe", "data"); m.set("array", [1, 2]); let calls = 0; const vv = d.version().encode(); - Object.defineProperty(Object.prototype, "__read_state_probe", { + Object.defineProperty(Object.prototype, "__container_tree_probe", { configurable: true, set() { calls++; @@ -169,7 +169,7 @@ describe("toContainerTree", () => { try { result = d.toContainerTree(); } finally { - delete (Object.prototype as Record).__read_state_probe; + delete (Object.prototype as Record).__container_tree_probe; } expect(calls).toBe(0); expect(d.version().encode()).toEqual(vv); From c453bd2d59a960f10c9542a1f1fd851b33499037 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Sun, 6 Sep 2026 12:30:13 +0800 Subject: [PATCH 6/7] fix(wasm): infer container tree types from actual options --- .changeset/wasm-container-tree.md | 2 + context/wasm-container-tree.md | 8 ++ crates/loro-wasm/AGENTS.md | 4 + crates/loro-wasm/src/container_tree.rs | 24 +++--- crates/loro-wasm/tests/container_tree.test.ts | 78 ++++++++++++++++++- 5 files changed, 106 insertions(+), 10 deletions(-) diff --git a/.changeset/wasm-container-tree.md b/.changeset/wasm-container-tree.md index 8baacd4bf..950969dd0 100644 --- a/.changeset/wasm-container-tree.md +++ b/.changeset/wasm-container-tree.md @@ -3,3 +3,5 @@ --- Add `toContainerTree()` to documents and attached containers. It returns independent recursive `{type, cid, value}` nodes and opaque ordinary `Value` nodes. Documents can select visible roots without creating missing roots. Text formatting applies to all descendants and is inferred in TypeScript. List and MovableList expose `toContainerTreeSlice(start, end)` with explicit `start`, `totalLength`, and `items`. Fixed JS construction, per-read key/peer reuse, and owned binary buffers avoid a public transport protocol. + +Optional text configurations retain the default plain-text possibility in TypeScript. Literal root selections preserve their names as optional result properties. diff --git a/context/wasm-container-tree.md b/context/wasm-container-tree.md index 3814d52a5..cd5d5e48d 100644 --- a/context/wasm-container-tree.md +++ b/context/wasm-container-tree.md @@ -100,3 +100,11 @@ property writes. These timings describe the tested machines/workload, not a gene speed guarantee. Mirror's Tree normalization still uses its existing handle path. Document toContainerTree({roots}) filters before reading root values; the optional text format applies recursively. TypeScript infers both the receiver kind and Text value format. Mirror selects schema roots and, when preserving unknown roots, includes them while excluding explicit Ignore roots. + +TypeScript conditional types distinguish a required `text` from an optional format: only +`{ text: "delta" }` guarantees delta arrays. Optional delta options (or omitted +options) retain `"plain"` in the result type. Explicit required option types also +require the argument at the call site. +Required literal root selections return `Partial>`, +because selected roots may be missing or hidden; optional roots can select all +roots when omitted and therefore do not restrict the result's key names. diff --git a/crates/loro-wasm/AGENTS.md b/crates/loro-wasm/AGENTS.md index ca607ad1b..fcba36691 100644 --- a/crates/loro-wasm/AGENTS.md +++ b/crates/loro-wasm/AGENTS.md @@ -96,3 +96,7 @@ range and Mirror integration contracts. Document `roots` filters before reading values; missing roots are omitted. List `toContainerTreeSlice` returns coordinates and items under one state lock, never a partial ContainerNode. Keep receiver and text-format inference in TypeScript. + +Only a required `text` option can exclude the default plain format from the +return type. Optional options/text must retain plain, including explicit generic +arguments. Required root selections preserve literal keys as optional properties. diff --git a/crates/loro-wasm/src/container_tree.rs b/crates/loro-wasm/src/container_tree.rs index d80965e8f..aab27c99b 100644 --- a/crates/loro-wasm/src/container_tree.rs +++ b/crates/loro-wasm/src/container_tree.rs @@ -423,38 +423,44 @@ export interface TreeNodeSnapshot { cid: ContainerID; start: number; totalLength: number; items: ContainerTreeNode[]; } +/** Resolve runtime defaults, distributing over optional/union configurations. */ +export type ContainerTreeText = O extends { text: infer T extends ContainerTreeTextFormat } + ? T : O extends { text?: infer T } ? Extract | "plain" : "plain"; +export type DocumentContainerTree = O extends { roots: readonly (infer R extends string)[] } + ? Partial>>> + : Record>>; interface LoroDoc { /** Convert visible roots to independent container trees. No commit, live handles or CRDT history. * Text defaults to plain strings. Throws for invalid options, unsupported types or nesting >256. */ - toContainerTree(options?: DocumentContainerTreeOptions): Record>; + toContainerTree(...args: A): DocumentContainerTree; } interface LoroMap { /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ - toContainerTree(options?: ContainerTreeOptions): Extract, {type:"Map"}>; + toContainerTree(...args: A): Extract>, {type:"Map"}>; } interface LoroList { /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ - toContainerTree(options?: ContainerTreeOptions): Extract, {type:"List"}>; + toContainerTree(...args: A): Extract>, {type:"List"}>; /** Read [start,end), clamped, with source coordinates; parent shallow list access remains O(N). */ - toContainerTreeSlice(start:number,end:number,options?:ContainerTreeOptions): ContainerTreeSlice; + toContainerTreeSlice(start:number,end:number,...args:A): ContainerTreeSlice>; } interface LoroMovableList { /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ - toContainerTree(options?: ContainerTreeOptions): Extract, {type:"MovableList"}>; + toContainerTree(...args: A): Extract>, {type:"MovableList"}>; /** Read [start,end), clamped, with source coordinates; parent shallow list access remains O(N). */ - toContainerTreeSlice(start:number,end:number,options?:ContainerTreeOptions): ContainerTreeSlice; + toContainerTreeSlice(start:number,end:number,...args:A): ContainerTreeSlice>; } interface LoroText { /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ - toContainerTree(options?: ContainerTreeOptions): Extract, {type:"Text"}>; + toContainerTree(...args: A): Extract>, {type:"Text"}>; } interface LoroTree { /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ - toContainerTree(options?: ContainerTreeOptions): Extract, {type:"Tree"}>; + toContainerTree(...args: A): Extract>, {type:"Tree"}>; } interface LoroCounter { /** Read this attached container and descendants; text format applies recursively. Throws if detached. */ - toContainerTree(options?: ContainerTreeOptions): Extract, {type:"Counter"}>; + toContainerTree(...args: A): Extract>, {type:"Counter"}>; } "#; diff --git a/crates/loro-wasm/tests/container_tree.test.ts b/crates/loro-wasm/tests/container_tree.test.ts index ed056f1a7..804fbc854 100644 --- a/crates/loro-wasm/tests/container_tree.test.ts +++ b/crates/loro-wasm/tests/container_tree.test.ts @@ -169,7 +169,8 @@ describe("toContainerTree", () => { try { result = d.toContainerTree(); } finally { - delete (Object.prototype as Record).__container_tree_probe; + delete (Object.prototype as Record) + .__container_tree_probe; } expect(calls).toBe(0); expect(d.version().encode()).toEqual(vv); @@ -339,3 +340,78 @@ function checkContainerTreeTypes( return [plain, delta, mapKind, invalid]; } void checkContainerTreeTypes; + +// Optional options must not promise delta when runtime defaults to plain text. +function checkOptionalTreeTypes( + text: LoroText, + doc: LoroDoc, + map: LoroMap, + tree: LoroTree, + list: LoroList, + movable: LoroMovableList, + optional: import("../bundler/index").ContainerTreeOptions<"delta">, + maybe: { text: "delta" } | undefined, + mode: "plain" | "delta", + listUnion: LoroList | LoroMovableList, +) { + type Delta = ReturnType; + const unionList = listUnion.toContainerTree({ text: "delta" }); + const unionSlice = listUnion.toContainerTreeSlice(0, 1, { text: "delta" }); + void [unionList, unionSlice]; + const union: string | Delta = text.toContainerTree(optional).value; + const dynamic: string | Delta = text.toContainerTree({ text: mode }).value; + // @ts-expect-error Optional text can default to a string. + const a: Delta = text.toContainerTree(optional).value; + // @ts-expect-error Missing options default to a string. + const b: Delta = text.toContainerTree(maybe).value; + // @ts-expect-error Explicit type arguments cannot override runtime defaults. + const c: Delta = text.toContainerTree<[{ text: "delta" }]>().value; + const plain: string = text.toContainerTree({}).value; + const delta: Delta = text.toContainerTree({ text: "delta" }).value; + const selected = doc.toContainerTree({ + roots: ["settings"] as const, + text: "delta", + }); + // @ts-expect-error Unselected root names are not in the result type. + selected.other; + // @ts-expect-error A selected root can be missing or hidden. + const required: ContainerNode<"delta"> = selected.settings; + const selectedNode: ContainerNode<"delta"> | undefined = selected.settings; + const absent = doc.toContainerTree({ roots: [] as const }); + // @ts-expect-error Empty selection has no keys. + absent.settings; + const child = map.toContainerTree(optional).value.text; + if (child.type === "Text") { + // @ts-expect-error Optional text applies recursively. + const d: Delta = child.value; + void d; + } + const meta = tree.toContainerTree(optional).value[0].meta.value.text; + if (meta.type === "Text") { + // @ts-expect-error Tree metadata also retains the default plain possibility. + const e: Delta = meta.value; + void e; + } + for (const items of [ + list.toContainerTreeSlice(0, 1, maybe).items, + movable.toContainerTreeSlice(0, 1, maybe).items, + ]) { + const item = items[0]; + if (item.type === "Text") { + // @ts-expect-error Optional slice options can produce strings. + const f: Delta = item.value; + void f; + } + } + return [union, dynamic, a, b, c, plain, delta, required, selectedNode]; +} +void checkOptionalTreeTypes; + +it("defaults optional delta configurations to plain text when omitted", () => { + const text = new LoroDoc().getText("text"); + text.insert(0, "hello"); + const options: import("../bundler/index").ContainerTreeOptions<"delta"> = {}; + expect(text.toContainerTree(options).value).toBe("hello"); + expect(text.toContainerTree(undefined).value).toBe("hello"); + expect(text.toContainerTree({ text: "delta" }).value).toEqual(text.toDelta()); +}); From 90abd52e64f7ca6571e5fdce3fd1dbcfd9dbc5b0 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Sun, 6 Sep 2026 12:49:50 +0800 Subject: [PATCH 7/7] fix(wasm): compile Node snippets to CommonJS --- .changeset/wasm-container-tree.md | 2 ++ context/wasm-container-tree.md | 5 ++-- crates/loro-wasm/AGENTS.md | 4 ++++ crates/loro-wasm/package.json | 2 +- crates/loro-wasm/scripts/build.ts | 13 ++++++++++ crates/loro-wasm/scripts/nodejs-snippets.cjs | 25 ++++++++++++++++++++ crates/loro-wasm/scripts/test-commonjs.cjs | 22 +++++++++++++++++ 7 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 crates/loro-wasm/scripts/nodejs-snippets.cjs create mode 100644 crates/loro-wasm/scripts/test-commonjs.cjs diff --git a/.changeset/wasm-container-tree.md b/.changeset/wasm-container-tree.md index 950969dd0..6a1c4139a 100644 --- a/.changeset/wasm-container-tree.md +++ b/.changeset/wasm-container-tree.md @@ -5,3 +5,5 @@ Add `toContainerTree()` to documents and attached containers. It returns independent recursive `{type, cid, value}` nodes and opaque ordinary `Value` nodes. Documents can select visible roots without creating missing roots. Text formatting applies to all descendants and is inferred in TypeScript. List and MovableList expose `toContainerTreeSlice(start, end)` with explicit `start`, `totalLength`, and `items`. Fixed JS construction, per-read key/peer reuse, and owned binary buffers avoid a public transport protocol. Optional text configurations retain the default plain-text possibility in TypeScript. Literal root selections preserve their names as optional result properties. + +Compile Node snippets to CommonJS so the package does not require `require(esm)` support. diff --git a/context/wasm-container-tree.md b/context/wasm-container-tree.md index cd5d5e48d..fd58a2dc8 100644 --- a/context/wasm-container-tree.md +++ b/context/wasm-container-tree.md @@ -43,8 +43,9 @@ inherited setters. Consumers must preserve this when projecting nodes themselves ## Implementation `state/container_tree.rs` emits a traversal to a sink without constructing a deep -whole-document LoroValue tree. Each container's shallow value is ephemeral; -values fetched to determine root visibility are reused. `loro-wasm/src/container_tree.rs` +whole-document LoroValue tree. Plain-mode shallow values are read ephemerally; values fetched to determine +root visibility are reused. The delta option materializes and retains Text states +through `get_or_create_mut`, matching the existing `toDelta()` path. `loro-wasm/src/container_tree.rs` owns the JS construction stack and uses fixed imported functions for stable wrapper shapes, IDs and own-property writes. Keys and peer decimal strings are cached only for one read; complete CIDs are constructed in JS. Binary buffers diff --git a/crates/loro-wasm/AGENTS.md b/crates/loro-wasm/AGENTS.md index fcba36691..6816756f4 100644 --- a/crates/loro-wasm/AGENTS.md +++ b/crates/loro-wasm/AGENTS.md @@ -100,3 +100,7 @@ a partial ContainerNode. Keep receiver and text-format inference in TypeScript. Only a required `text` option can exclude the default plain format from the return type. Optional options/text must retain plain, including explicit generic arguments. Required root selections preserve literal keys as optional properties. + +The nodejs target must load without Node's `require(esm)` support. Convert +wasm-bindgen snippets to CommonJS during the build and keep the package test's +`--no-experimental-require-module` smoke check. Other targets retain ESM snippets. diff --git a/crates/loro-wasm/package.json b/crates/loro-wasm/package.json index efce2f324..e491f9133 100644 --- a/crates/loro-wasm/package.json +++ b/crates/loro-wasm/package.json @@ -77,7 +77,7 @@ "scripts": { "build-dev": "deno run -A ./scripts/build.ts dev && rollup -c && deno run -A ./scripts/post-rollup.ts && npm run test", "build-release": "deno run -A ./scripts/build.ts release && rollup -c && deno run -A ./scripts/post-rollup.ts && npm run test", - "test": "node --expose-gc ./node_modules/vitest/vitest.mjs run && npx tsc --noEmit && cd ./deno_tests && deno test -A && cd ../bun_tests && bun test", + "test": "node --no-experimental-require-module ./scripts/test-commonjs.cjs && node --expose-gc ./node_modules/vitest/vitest.mjs run && npx tsc --noEmit && cd ./deno_tests && deno test -A && cd ../bun_tests && bun test", "test:snapshot-memory": "node --expose-gc ./scripts/measure-snapshot-round-memory.cjs" }, "homepage": "https://loro.dev", diff --git a/crates/loro-wasm/scripts/build.ts b/crates/loro-wasm/scripts/build.ts index d95c2c2ff..15c7f491e 100644 --- a/crates/loro-wasm/scripts/build.ts +++ b/crates/loro-wasm/scripts/build.ts @@ -237,6 +237,19 @@ async function buildTarget(target: string) { if (target === "nodejs") { console.log("🔨 Patching nodejs target"); + const snippets = await new Deno.Command("node", { + args: [ + path.resolve(__dirname, "nodejs-snippets.cjs"), + path.resolve(targetDirPath, "snippets"), + ], + cwd: LoroWasmDir, + }).output(); + if (!snippets.success) { + throw new Error( + `CommonJS snippet conversion failed: ${textDecoder.decode(snippets.stderr)}`, + ); + } + const patch = await Deno.readTextFile( path.resolve(__dirname, "./nodejs_patch.js"), ); diff --git a/crates/loro-wasm/scripts/nodejs-snippets.cjs b/crates/loro-wasm/scripts/nodejs-snippets.cjs new file mode 100644 index 000000000..ad6d77d7b --- /dev/null +++ b/crates/loro-wasm/scripts/nodejs-snippets.cjs @@ -0,0 +1,25 @@ +// wasm-bindgen emits ES modules for snippets even with --target nodejs. +// Compile them to CommonJS so loading the package never relies on require(esm). +const fs = require("node:fs"); +const path = require("node:path"); +const { transformSync } = require("esbuild"); + +function convert(directory) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name); + if (entry.isDirectory()) { + convert(file); + } else if (entry.isFile() && entry.name.endsWith(".js")) { + const { code } = transformSync(fs.readFileSync(file, "utf8"), { + format: "cjs", + target: "node18", + sourcefile: file, + }); + fs.writeFileSync(file, code); + } + } +} + +const directory = process.argv[2]; +if (!directory) throw new Error("Expected the nodejs snippets directory"); +if (fs.existsSync(directory)) convert(directory); diff --git a/crates/loro-wasm/scripts/test-commonjs.cjs b/crates/loro-wasm/scripts/test-commonjs.cjs new file mode 100644 index 000000000..dcdcec9d2 --- /dev/null +++ b/crates/loro-wasm/scripts/test-commonjs.cjs @@ -0,0 +1,22 @@ +// Run with --no-experimental-require-module to catch ESM leaking into CJS. +const assert = require("node:assert/strict"); +const { LoroDoc, LoroText } = require("../nodejs"); +const doc = new LoroDoc(); +const map = doc.getMap("root"); +const text = map.setContainer("text", new LoroText()); +text.insert(0, "hello"); +text.mark({ start: 0, end: 5 }, "bold", true); +map.set("bytes", new Uint8Array([0, 255])); +assert.deepEqual(doc.toJSON().root.text, "hello"); +const result = doc.toContainerTree({ text: "delta" }).root; +assert.equal(result.cid, map.id); +assert.deepEqual(result.value.text, { + type: "Text", + cid: text.id, + value: text.toDelta(), +}); +assert.deepEqual(result.value.bytes.value, new Uint8Array([0, 255])); +doc.free(); +console.log( + "CommonJS loading and container tree smoke passed without require(esm)", +);