From 350cdd85720b36fe6d09c2d4b696db45bf84b1d2 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Tue, 15 Sep 2026 23:23:24 +0000 Subject: [PATCH 01/73] Fix ACTZipper::val_count counting from the zipper root, not the focus Cherry-pick of e49af0f from archive-bugfix/act-val-count-from-focus. `val_count` opened with `reset()`, so it counted the values below the zipper's root whatever the focus was and was right only at the root; `ZipperValues::val_count` counts at and below the focus. Verified against the model: the class goes from 1416 hits on 5000 ACT inputs at seed 7 to 0, and agreement rises 3581 -> 4733. The crate-mode run is untouched, as it must be. This also unmasks 52 inputs that now diverge on `to_next_step`. They are not a regression: `val_count` was wrong at nearly every focus, so the classifier attributed those lines to it, and the only behaviour this commit changes is `val_count`, which `to_next_step` does not call. They are an ACT iteration defect that was hidden and is now visible, and are left unclassified deliberately. The upstream commit's own test conflicted with the ACT tests master has since gained; both sets are kept. Co-Authored-By: Claude Opus 5 (1M context) --- src/arena_compact.rs | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/arena_compact.rs b/src/arena_compact.rs index 3914ba8b..314e4911 100644 --- a/src/arena_compact.rs +++ b/src/arena_compact.rs @@ -2938,13 +2938,24 @@ where Storage: AsRef<[u8]> /// WARNING: This is not a cheap method. It may have an order-N cost fn val_count(&self) -> usize { timed_span!(ValueCount, COUNTERS); + //`ZipperMoving::val_count` counts the values at and below the *focus*. This used to + // `reset()` first, so it counted the whole subtrie below the zipper's root and returned + // the same number wherever the focus was -- right only when the focus was at the root. + // + //`to_next_val` walks the zipper's entire subtrie, not just the part below the focus, so + // the walk has to stop when it leaves: depth-first order visits everything below the + // focus before anything outside it, so the first path that no longer starts with the + // focus ends the count. let mut zipper = self.clone(); - zipper.reset(); + let focus: Vec = zipper.path().to_vec(); let mut count = 0; if zipper.is_val() { count += 1; } while zipper.to_next_val() { + if !zipper.path().starts_with(&focus) { + break; + } count += 1; } count @@ -4191,4 +4202,31 @@ mod tests { assert_eq!(az.val(), None); assert!(!az.path_exists()); } + /// `ACTZipper::val_count` opened with `reset()`, so it counted the values below + /// the zipper's *root* whatever the focus was, and was right only at the root. + /// `ZipperValues::val_count` counts at and below the focus. + #[test] + fn act_zipper_val_count_counts_from_the_focus() { + use crate::zipper::*; + let mut m = PathMap::::new(); + m.insert(b"aa", 1); + m.insert(b"ab", 2); + m.insert(b"b", 3); + let t = ArenaCompactTree::from_zipper(m.read_zipper(), |&v| v); + for path in [&b""[..], b"a", b"aa", b"ab", b"b", b"zz"] { + let mut az = t.read_zipper_u64(); + let mut pz = m.read_zipper(); + az.descend_to(path); + pz.descend_to(path); + assert_eq!(az.val_count(), pz.val_count(), "focus {path:?}"); + } + //And from a zipper rooted below the map root + for path in [&b""[..], b"a", b"b"] { + let mut az = t.read_zipper_at_path_u64(b"a"); + let mut pz = m.read_zipper_at_path(b"a"); + az.descend_to(path); + pz.descend_to(path); + assert_eq!(az.val_count(), pz.val_count(), "root a, focus {path:?}"); + } + } } From 9420b9e88e7fd017b2d9a45cf6f76320b5b5ab5a Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Tue, 15 Sep 2026 23:26:21 +0000 Subject: [PATCH 02/73] Give ACTZipper::to_sibling an answer for an off-trie focus Cherry-pick of 0ef62ff from archive-bugfix/act-sibling-off-trie-focus. `to_sibling` worked off the node stack, which holds only real nodes, so a focus one byte off the trie had no frame and the method answered `None`. That starved `to_next_step`, which moves by it, and whole subtrees went unvisited. This is the fix for what the previous commit unmasked. On 5000 ACT inputs at seed 7 the 52 `to_next_step` divergences go to 0, and with them the ACT hits of the sibling-after-iteration class (50 -> 0) and the to_next_val class (123 -> 0), which were the same starvation seen through other operations. Agreement goes 4733 -> 4956, and 3581 -> 4956 against the baseline before either ACT fix. Crate mode is untouched. The upstream commit's test conflicted with the test added by the previous commit, and the conflict boundary fell inside that test; both are kept and the truncated one closed. Co-Authored-By: Claude Opus 5 (1M context) --- src/arena_compact.rs | 84 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/src/arena_compact.rs b/src/arena_compact.rs index 314e4911..6212bd34 100644 --- a/src/arena_compact.rs +++ b/src/arena_compact.rs @@ -2772,6 +2772,39 @@ where Storage: AsRef<[u8]> } fn to_sibling(&mut self, next: bool) -> Option { + //An off-trie focus has no stack frame -- the stack holds real nodes, and a byte that is + //not in the trie has no node -- so the index-based path below cannot serve it and used to + //answer `None`. That starves `to_next_step`, which is `ZipperIteration`'s default and + //moves by this method: from a non-existent focus that sorts before an existing sibling it + //would give up and reset rather than step to it. + // + //The sibling of a phantom byte is still well defined, because it is defined by the + //*parent's* children rather than by the focus: the next child byte strictly greater than + //the phantom one. That only makes sense while the parent itself is real, so a focus more + //than one byte off the trie has no siblings -- its parent has no children at all. + if self.invalid > 0 { + if self.invalid > 1 { + return None; + } + let byte = *self.path.last()?; + if self.ascend_invalid(Some(1)) != 1 { + //The phantom byte is the zipper's root, so there is nothing to be a sibling of. + return None; + } + let mask = self.child_mask(); + let target = if next { mask.next_bit(byte) } else { mask.prev_bit(byte) }; + match target { + Some(t) => { + self.descend_to_byte(t); + return Some(t); + } + None => { + //Documented to leave the zipper where it was when it does not move. + self.descend_to_byte(byte); + return None; + } + } + } let top_frame = self.stack.last().unwrap(); if self.stack.len() <= 1 || top_frame.node_depth > 0 { // can't move to sibling at root, or along the path @@ -4229,4 +4262,55 @@ mod tests { assert_eq!(az.val_count(), pz.val_count(), "root a, focus {path:?}"); } } + + /// `ACTZipper::to_sibling` worked entirely off the node stack, which holds real + /// nodes, so a focus one byte off the trie had no frame and the method answered + /// `None`. That starved `to_next_step`, which moves by it: from a non-existent + /// focus sorting before an existing sibling it gave up instead of stepping to it, + /// and whole subtrees went unvisited. The sibling of a phantom byte is defined by + /// the parent's children, so it exists while the parent is real. + #[test] + fn act_zipper_sibling_step_from_an_off_trie_focus() { + use crate::zipper::*; + let mut m = PathMap::::new(); + { let mut w = m.write_zipper(); w.set_val(38); } + m.insert(&[1u8], 5); + m.insert(&[1u8, 0, 2], 22); + m.insert(&[3u8], 7); + let t = ArenaCompactTree::from_zipper(m.read_zipper(), |&v| v); + + //One byte off the trie, with a sibling on either side + let mut az = t.read_zipper_u64(); + az.descend_to(&[2u8]); + assert!(!az.path_exists()); + assert_eq!(az.to_next_sibling_byte(), Some(3)); + assert_eq!(az.path(), &[3u8]); + assert_eq!(az.val(), Some(&7)); + az.ascend(1); + az.descend_to(&[2u8]); + assert_eq!(az.to_prev_sibling_byte(), Some(1)); + assert_eq!(az.path(), &[1u8]); + assert_eq!(az.val(), Some(&5)); + + //No sibling on that side: the zipper stays where it was + let mut az = t.read_zipper_u64(); + az.descend_to(&[0u8]); + assert_eq!(az.to_prev_sibling_byte(), None); + assert_eq!(az.path(), &[0u8]); + assert!(!az.path_exists()); + assert_eq!(az.to_next_sibling_byte(), Some(1)); + + //Two bytes off the trie: the parent is not real, so there is no sibling + let mut az = t.read_zipper_u64(); + az.descend_to(&[2u8, 0]); + assert_eq!(az.to_next_sibling_byte(), None); + assert_eq!(az.path(), &[2u8, 0]); + + //`to_next_step` from an off-trie focus visits what follows it + let mut az = t.read_zipper_u64(); + az.descend_to(&[0u8]); + let mut seen = Vec::new(); + while az.to_next_step() { seen.push(az.path().to_vec()); } + assert_eq!(seen, vec![vec![1u8], vec![1, 0], vec![1, 0, 2], vec![3]]); + } } From 3a98201874c83da51917bfcc3cac199b13900500 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Tue, 15 Sep 2026 23:32:12 +0000 Subject: [PATCH 03/73] Keep a value in subtract when the other side only passes through its byte Cherry-pick of f75fce1 from archive-bugfix/subtract-drops-value: the missing else arm in `ByteNode::psubtract_abstract`. Verified: both corpus reproducers for the class (subtract_into-drops-value-under-source-path{,-2}.bin) go from failing to agreeing, the class goes 1 -> 0 on 20000 crate inputs at seed 7, agreement rises 19738 -> 19739, and no other class moves. The class is not empty afterwards. Seed 12 keeps one hit, on which the model returns Identity with the subtrie intact and the crate returns None having dropped it -- the same shape but a subtrie discarded rather than a value, and present identically before this commit, so it is a residual rather than a regression. Saved as lean/corpus/subtract_into-drops-subtrie-residual.bin. Co-Authored-By: Claude Opus 5 (1M context) --- src/dense_byte_node.rs | 11 +++++++---- src/trie_map.rs | 31 +++++++++++++++++++++++++++++++ src/write_zipper.rs | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 65568d9c..a1419c5e 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -418,10 +418,12 @@ impl> ByteNode let cf = unsafe{ self_node.values.get_unchecked(cf_idx) }; let mut new_cf = Cf::new(None, None); - //If there is a value at this key_byte, and the other node contains a value, subtract them + //If there is a value at this key_byte, and the other node contains a value, subtract them. + // If the other node only passes through this byte on the way to something deeper, it holds + // nothing at this exact location, so the value survives untouched. if let Some(self_val) = cf.val() { - if let Some(other_val) = other.node_get_val(&[key_byte]) { - match self_val.psubtract(other_val) { + match other.node_get_val(&[key_byte]) { + Some(other_val) => match self_val.psubtract(other_val) { AlgebraicResult::None => { is_identity = false; }, AlgebraicResult::Identity(mask) => { debug_assert_eq!(mask, SELF_IDENT); //subtract is not commutative @@ -431,7 +433,8 @@ impl> ByteNode is_identity = false; new_cf.set_val(e); }, - } + }, + None => new_cf.set_val(self_val.clone()), } } diff --git a/src/trie_map.rs b/src/trie_map.rs index efe75ed6..b5e0b611 100644 --- a/src/trie_map.rs +++ b/src/trie_map.rs @@ -876,6 +876,37 @@ mod tests { // c-> List("aaa") } + /// `subtract` must keep a value when the other map only has a *longer* path through its location. + /// Needs a byte-node root on the left (a value-and-child key plus a second branch) and a line-node + /// root on the right; every smaller shape already worked. + #[test] + fn map_subtract_keeps_value_under_other_path() { + let mut a: PathMap = PathMap::new(); + a.insert([0], 0); + a.insert([0, 0], 0); + a.insert([0, 0, 0], 0); + a.insert([1, 0, 0], 0); + let mut b: PathMap = PathMap::new(); + b.insert([0, 0], 0); + b.insert([1, 0, 0], 0); + + let diff = a.subtract(&b); + let paths: Vec<(Vec, u64)> = diff.iter().map(|(k, v)| (k.to_vec(), *v)).collect(); + assert_eq!(paths, vec![(vec![0], 0), (vec![0, 0, 0], 0)]); + + // and with distinct values, so the surviving value is visibly the left one + let mut a: PathMap = PathMap::new(); + a.insert([0], 5); + a.insert([0, 0], 6); + a.insert([0, 0, 0], 7); + a.insert([1, 0, 0], 8); + let diff = a.subtract(&b); + assert_eq!(diff.val_at([0]), Some(&5)); + assert_eq!(diff.val_at([0, 0]), Some(&6)); // 6 - 0 is Element(6) for u64 + assert_eq!(diff.val_at([0, 0, 0]), Some(&7)); + assert_eq!(diff.val_at([1, 0, 0]), Some(&8)); + } + #[test] fn map_insert_test() { let keys = [ diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 0892e475..fc549e6b 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -3754,6 +3754,42 @@ mod tests { assert_eq!(map.iter().count(), 1); } + /// A value must survive `subtract_into` when the source only passes *through* its location on the + /// way to a deeper value. The destination root here is a byte node (three branches worth of + /// payloads) and the source a line node, so the subtraction goes through `psubtract_abstract`, + /// which used to drop the value at `[0]` because the source has no value at that exact byte. + #[test] + fn write_zipper_subtract_into_value_under_source_path() { + let mut map: PathMap = PathMap::new(); + map.insert([0], 0); + map.insert([0, 0], 0); + map.insert([0, 0, 0], 0); + map.insert([1, 0, 0], 0); + let mut src: PathMap = PathMap::new(); + src.insert([0, 0], 0); + src.insert([1, 0, 0], 0); + + assert_eq!(map.write_zipper().subtract_into(&src.read_zipper(), false), AlgebraicStatus::Element); + let remaining: Vec<(Vec, u64)> = map.iter().map(|(k, v)| (k.to_vec(), *v)).collect(); + assert_eq!(remaining, vec![(vec![0], 0), (vec![0, 0, 0], 0)]); + + // The same shape reached the way the fuzzer found it: join the source in first, then take it out again + let mut map: PathMap = PathMap::new(); + map.insert([], 0); + map.insert([0], 0); + map.insert([0, 0, 0], 0); + let mut src: PathMap = PathMap::new(); + src.insert([], 0); + src.insert([0, 0], 0); + src.insert([1, 0, 0], 0); + let mut wz = map.write_zipper(); + wz.join_into(&src.read_zipper()); + wz.subtract_into(&src.read_zipper(), false); + drop(wz); + let remaining: Vec<(Vec, u64)> = map.iter().map(|(k, v)| (k.to_vec(), *v)).collect(); + assert_eq!(remaining, vec![(vec![0], 0), (vec![0, 0, 0], 0)]); + } + /// Tests how `subtract_into` handles dangling paths, including situations with extraneous empty nodes hanging around #[test] fn write_zipper_subtract_into_test2() { From 3a16a579fb4db49095cd0a5ac0c84b87f42f3de2 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Tue, 15 Sep 2026 23:33:52 +0000 Subject: [PATCH 04/73] Fix graft_child_maps and graft_masked_branches (finding 15) Cherry-pick of 4453b07 from archive-bugfix/graft-child-maps: both documented symptoms plus the third inconsistency, and op 54 comes out of quarantine in the harness and the model. Verified: graft_masked_branches creating the focus goes from 177 hits on 20000 crate inputs at seed 7 to 0, and agreement rises 19739 -> 19894. Other classes move up -- join_into 24 -> 36, finding 8 20 -> 29, status 18 -> 19, dangling 11 -> 12 -- because op 54 now executes where it used to be skipped on both sides, so every input that reaches it runs more operations and has more chances to hit a bug that was already there. Total known hits fall 261 -> 106 and nothing new appears. The upstream commit also deleted its own KNOWN entries from differential.py; those are dropped from this cherry-pick, since letting a fix retire the entry that measures it makes the verification circular. Its harness.rs and Fuzz.lean hunks conflicted with the skip-reason vocabulary and were reapplied in terms of it. Co-Authored-By: Claude Opus 5 (1M context) --- differential/src/harness.rs | 19 +-- lean/FINDINGS.md | 8 ++ lean/PathMapModel/Fuzz.lean | 6 +- src/dense_byte_node.rs | 32 +++-- src/write_zipper.rs | 232 +++++++++++++++++++++++++++++++----- 5 files changed, 248 insertions(+), 49 deletions(-) diff --git a/differential/src/harness.rs b/differential/src/harness.rs index 0b0d72d9..160b34be 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -224,9 +224,8 @@ pub trait ReadSource: fn do_graft>(&self, _wz: &mut W) -> bool { false } fn do_graft_masked>(&self, _wz: &mut W, _m: ByteMask, _ru: bool) -> bool { false } - /// Currently unused: op 54 is quarantined (lean/FINDINGS.md #15). Kept so - /// the op can be re-enabled with one line once the method is fixed. - #[allow(dead_code)] + /// `graft_child_maps` fed this zipper's own child subtries (op 54), which must + /// agree with `graft_masked_branches` on the same mask. fn do_graft_child_maps>(&self, _wz: &mut W, _m: ByteMask, _ru: bool) -> bool { false } /// `meet_2` needs two sources; the second is this zipper moved to `path`. fn do_meet_2>(&self, _wz: &mut W, _path: &[u8]) -> Option { None } @@ -926,12 +925,14 @@ pub fn run_ops( let mut canon: Vec = m.clone(); canon.sort_unstable(); canon.dedup(); - // Skipped outright: `graft_child_maps` is broken three ways - // (lean/FINDINGS.md #15) and the node representations it - // leaves behind degrade the AlgebraicStatus that *later* - // operations report, contaminating the rest of the run. - let _ = (mask, ru, &canon); - ("graft_child_maps", SKIP_QUARANTINED.to_string()) + // Fed the source's own child subtries, so it must agree with + // `graft_masked_branches` on the same mask (op 53). + let s = if (*rz).do_graft_child_maps(&mut wz, mask, ru) { + format!("{}:{}", hex_path(&canon), show_bool(ru)) + } else { + SKIP_ACT.to_string() + }; + ("graft_child_maps", s) } 55 => { let p = get!(d.path(6)); diff --git a/lean/FINDINGS.md b/lean/FINDINGS.md index 4d98c548..f68b3d4d 100644 --- a/lean/FINDINGS.md +++ b/lean/FINDINGS.md @@ -469,6 +469,14 @@ it. `case: graft_child_maps_dense` -- **abort, or a silently created path** +*Fixed* (`bugfix/graft-child-maps`): the root-focus lookup no longer hands an +empty key to the dense node, and grafting nothing onto a branch now follows +`graft`: an existing branch is emptied but survives as a dangling path, an +absent one is not created. A third symptom found while fixing -- an empty map, +or one without a root value, left the destination's old branch or old value in +place -- is fixed the same way. Op 54 is no longer skipped by the harness, and +the regression test is `write_zipper::tests::graft_child_maps_dense`. + Two symptoms, one method. **On any dense destination it aborts.** diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index de6c88e7..67e8ad5d 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -527,11 +527,7 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do | 54 => do let (n, d) ← d.mod 4; let (m, d) ← d.pathN n; let (ru, d) ← d.bool -- Fed the source's own child subtries, this must agree with -- `graft_masked_branches` on the same mask. - -- Skipped outright, not just in ACT mode: `graft_child_maps` is - -- broken three ways (FINDINGS.md #15) and the node representations - -- it leaves behind degrade the `AlgebraicStatus` that *later* - -- operations report, which would contaminate the whole run. - if true then some (emit s "graft_child_maps" skipQuarantined, d) else + if s.act then some (emit s "graft_child_maps" skipAct, d) else do let mask := ByteMask.ofList m let maps := mask.map (fun b => ([b], s.rz.trie.subtrie (s.rz.focus ++ [b]))) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index a1419c5e..43d3fa91 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -1578,14 +1578,30 @@ where } } - let src_range_mask = src_node.mask & ByteMask::from_range(range_start..=range_end); - let mut src_ix = src_node.mask.index_of(range_start) as usize; - for child_byte in src_range_mask.iter() { - let cf = unsafe { src_node.values.get_unchecked(src_ix) }; - src_ix += 1; - - if cf.has_rec() || cf.has_val() { - new_values.v.push(CfDst::from_cf(cf.clone())); + let range_mask = ByteMask::from_range(range_start..=range_end); + let src_range_mask = src_node.mask & range_mask; + let dst_range_mask = old_mask & range_mask; + for child_byte in (src_range_mask | dst_range_mask).iter() { + // A source branch counts only if it leads somewhere: a value, or a node with + // contents. A dangling branch in the source is nothing to graft, exactly like an + // absent one, so it must not create a location here. + let mut grafted = false; + if src_range_mask.test_bit(child_byte) { + let cf = unsafe { src_node.values.get_unchecked(src_node.mask.index_of(child_byte) as usize) }; + let rec_has_contents = cf.rec().map_or(false, |rec| !rec.as_tagged().node_is_empty()); + if rec_has_contents || cf.has_val() { + new_values.v.push(CfDst::from_cf(cf.clone())); + new_mask.set_bit(child_byte); + grafted = true; + } + } + if !grafted && !REMOVE_UNSET && dst_range_mask.test_bit(child_byte) { + // Grafting nothing over an existing branch removes its contents and its value, + // but the location itself survives as a dangling path -- the same thing + // `graft` of an empty source does at the focus, and what the model's + // `graftBelow` + `removeVal` specify. With `remove_unset` the branch was + // removed outright instead. + new_values.v.push(CfDst::new(Some(TrieNodeODRc::new_empty()), None)); new_mask.set_bit(child_byte); } } diff --git a/src/write_zipper.rs b/src/write_zipper.rs index fc549e6b..7fc45beb 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -1552,6 +1552,26 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC ; *focus_node = replacement; } + /// One masked branch of [Self::graft_masked_branches]: `self`'s branch at `byte` becomes the + /// source's, value included. + /// + /// When the source has nothing at `byte` -- no value and no node with contents, a dangling + /// source branch included -- grafting nothing removes the branch's contents and value, but + /// the location is neither created nor destroyed: an existing branch survives as a dangling + /// path (as after `graft` of an empty source, or the model's `graftBelow` + `removeVal`), + /// and an absent one stays absent. + fn graft_masked_branch>(&mut self, src: &Z, byte: u8) { + let src_node = src.get_focus_at([byte]); + let src_has_node = !src_node.is_none() && !src_node.as_tagged().node_is_empty(); + self.descend_to_byte(byte); + if src_has_node || src.val_at([byte]).is_some() { + self.graft_src_at(src, [byte]); + } else if self.path_exists() { + self.remove_branches(false); + self.remove_val(false); + } + self.ascend_byte(); + } /// See [ZipperWriting::graft_masked_branches] pub fn graft_masked_branches>(&mut self, src: &Z, child_mask: ByteMask, remove_unset: bool) { // The dense-node merge handles both pieces of the contract directly: it removes @@ -1570,9 +1590,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } // SAFETY: this arm is selected only when `child_mask` has one bit. let byte = unsafe { child_mask.indexed_bit::(0).unwrap_unchecked() }; - self.descend_to_byte(byte); - self.graft_src_at(src, &[byte]); - self.ascend_byte(); + self.graft_masked_branch(src, byte); } 2 => { if remove_unset { @@ -1580,15 +1598,11 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } // SAFETY: this arm is selected only when `child_mask` has two bits. let first_byte = unsafe { child_mask.indexed_bit::(0).unwrap_unchecked() }; - self.descend_to_byte(first_byte); - self.graft_src_at(src, &[first_byte]); - self.ascend_byte(); + self.graft_masked_branch(src, first_byte); // SAFETY: `first_byte` is one of the two set bits, so it has a successor. let second_byte = unsafe { child_mask.next_bit(first_byte).unwrap_unchecked() }; - self.descend_to_byte(second_byte); - self.graft_src_at(src, &[second_byte]); - self.ascend_byte(); + self.graft_masked_branch(src, second_byte); } _ => { let src_focus = src.get_focus(); @@ -1599,11 +1613,18 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC // to hand out) has no node to merge into even after the split, so the // branches are merged into a fresh node that is grafted in afterwards. // This used to unwrap the missing node. + // + // A focus that does not exist at all is not split either: the split + // would create the path, and grafting nothing must not create a + // location. Its branches also go into a fresh node, which is grafted + // (creating the path) only if the merge produced something. let mut fresh_node: Option> = None; let self_focus_node = match self.try_borrow_focus_mut() { Some(node) => node, None => { - self.split_at_focus(); + if self.path_exists() { + self.split_at_focus(); + } match self.try_borrow_focus_mut() { Some(node) => node, None => { @@ -1653,7 +1674,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC if remove_unset { self.remove_branches(false); } else { - self.remove_unmasked_branches(child_mask.not(), false); + self.empty_masked_branches(child_mask); } }, } @@ -1667,7 +1688,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC if remove_unset { self.remove_branches(false); } else { - self.remove_unmasked_branches(child_mask.not(), false); + self.empty_masked_branches(child_mask); } } } @@ -1675,6 +1696,21 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } } + /// [Self::graft_masked_branches] when the source has no node at all to take branches from: + /// every existing branch of `self` named by `mask` is emptied -- contents and value -- but + /// survives as a dangling path, and no branch is created. This is what grafting nothing + /// onto each branch amounts to; removing the branches outright would prune locations that + /// `graft` of an empty source would have left standing. + fn empty_masked_branches(&mut self, mask: ByteMask) { + let existing = mask & self.child_mask(); + for byte in existing.iter() { + self.descend_to_byte(byte); + self.remove_branches(false); + self.remove_val(false); + self.ascend_byte(); + } + } + /// Optimized implementation of [ZipperWriting::graft_child_maps] for WriteZipperCore /// /// This implementation constructs a new node with the appropriate children directly, @@ -1693,7 +1729,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC for child_byte in child_mask.iter() { let map = maps_iter.next().expect("maps iterator returned fewer items than the number of set bits in child_mask"); let (src_root_node, src_root_val) = map.into_root(); - if let Some(node) = src_root_node { + if let Some(node) = src_root_node.filter(|n| !n.as_tagged().node_is_empty()) { new_node.set_child(child_byte, node); } if let Some(val) = src_root_val { @@ -1701,7 +1737,14 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } } let new_node_odrc = TrieNodeODRc::new_in(new_node, self.alloc.clone()); - self.graft_internal(Some(new_node_odrc)); + if new_node_odrc.as_tagged().node_is_empty() { + // Every map was empty: all that is left of the contract is "remove the unset + // branches", and that must not create the focus if it does not exist. Grafting + // an empty node would leave a dangling path made out of nothing. + self.remove_branches(false); + } else { + self.graft_internal(Some(new_node_odrc)); + } } else { // If we don't have enough children to justify forcing a new ByteNode, just set the nodes if remove_unset { @@ -1712,7 +1755,21 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC let map = maps_iter.next().expect("maps iterator returned fewer items than the number of set bits in child_mask"); let (src_root_node, src_root_val) = map.into_root(); - if let Some(node) = src_root_node { + // Each branch is *replaced* by its map, the way `graft` replaces a subtrie: + // whatever `self` had at `child_byte` -- children and value -- goes first, so an + // empty map empties the branch and a map without a root value removes the value. + // The location itself is neither created nor destroyed (an existing branch that + // receives nothing survives as a dangling path, as after `graft` of an empty + // source). With `remove_unset` everything below the focus is already gone. + if !remove_unset { + self.descend_to_byte(child_byte); + if self.path_exists() { + self.remove_branches(false); + self.remove_val(false); + } + self.ascend_byte(); + } + if let Some(node) = src_root_node.filter(|n| !n.as_tagged().node_is_empty()) { self.set_node_at_child_path(&[child_byte], node) } if let Some(val) = src_root_val { @@ -2439,19 +2496,24 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC RetryF: FnOnce(&mut TaggedNodeRefMut<'_, V, A>, &[u8]) -> R, { let key = self.key.node_key(); - let mut focus_node = self.focus_stack.top_mut().unwrap(); - if let Some((key_bytes, child_node)) = focus_node.node_get_child_mut(key) { - debug_assert_eq!(key_bytes, key.len()); - let (key, node) = node_along_path_mut(child_node, path, true); - let mut node_ref = node.make_mut(); - match node_f(&mut node_ref, key) { - Ok(result) => result, - Err(replacement_node) => { - *node = replacement_node; - retry_f(&mut node.make_mut(), key) - }, + // At the root the focus node *is* the top of the stack and there is no key to look up; + // `node_get_child_mut` must never see an empty key (a `DenseByteNode` indexes `key[0]`). + if key.len() > 0 { + let mut focus_node = self.focus_stack.top_mut().unwrap(); + if let Some((key_bytes, child_node)) = focus_node.node_get_child_mut(key) { + debug_assert_eq!(key_bytes, key.len()); + let (key, node) = node_along_path_mut(child_node, path, true); + let mut node_ref = node.make_mut(); + return match node_f(&mut node_ref, key) { + Ok(result) => result, + Err(replacement_node) => { + *node = replacement_node; + retry_f(&mut node.make_mut(), key) + }, + } } - } else { + } + { self.in_zipper_mut_static_result( |focus_node, partial_key| { let mut key_buf = [0u8; MAX_NODE_KEY_BYTES]; @@ -6088,6 +6150,122 @@ mod tests { assert_eq!(keys(&m), ["cax", "cbx", "cdx", "d"]); } + /// lean/FINDINGS.md #15, `graft_child_maps_dense`: grafting nothing must neither create nor + /// destroy a location, and `graft_child_maps` must not abort on a dense destination. + /// + /// The contract for one masked branch is that of `graft` at that branch: the source's + /// contents and value replace whatever was there. A source with nothing at the branch + /// (absent, or a dangling path) therefore empties an existing branch -- which then survives + /// as a dangling path, exactly as after `graft` of an empty source -- and leaves an absent + /// branch absent. `remove_unset` removes the unset branches outright first. + #[test] + fn graft_child_maps_dense() { + use crate::utils::BitMask; + fn mask(bytes: &[u8]) -> ByteMask { + let mut m = ByteMask::EMPTY; + for b in bytes { m.set_bit(*b); } + m + } + fn dump(m: &PathMap) -> Vec<(Vec, Option)> { + // Every location, dangling ones included, with its value. + let mut z = m.read_zipper(); + let mut out = Vec::new(); + while z.to_next_step() { out.push((z.path().to_vec(), z.val().copied())); } + out + } + fn mk(kvs: &[(&[u8], u64)]) -> PathMap { + let mut m = PathMap::new(); + for (k, v) in kvs { m.set_val_at(k, *v); } + m + } + + // (a) A dense destination (>= 3 branches) used to reach `node_get_child_mut` with an + // empty key and abort. + let mut dst = mk(&[(&[0, 0], 1), (&[1, 0], 1), (&[2, 0], 1), (&[3, 0], 1)]); + let child = mk(&[(&[], 7), (&[3], 8)]); + dst.write_zipper().graft_child_maps(mask(&[0]), vec![child], false); + assert_eq!(dump(&dst), vec![ + (vec![0], Some(7)), (vec![0, 3], Some(8)), + (vec![1], None), (vec![1, 0], Some(1)), + (vec![2], None), (vec![2, 0], Some(1)), + (vec![3], None), (vec![3, 0], Some(1)), + ]); + + // (b) Empty maps at a non-existent focus: nothing is created, both with and without + // `remove_unset`, and the same for `graft_masked_branches` from an empty source. + for remove_unset in [true, false] { + let mut m = mk(&[(&[1], 1)]); + { + let mut wz = m.write_zipper(); + wz.descend_to(&[0u8, 0]); + wz.graft_child_maps(mask(&[0, 1, 2]), vec![PathMap::new(); 3], remove_unset); + assert!(!wz.path_exists()); + } + assert_eq!(dump(&m), vec![(vec![1], Some(1))]); + } + for bits in [&[0u8][..], &[0, 2], &[0, 2, 3]] { + let mut m = mk(&[(&[1], 1)]); + let src = mk(&[(&[1, 0], 5)]); + { + let mut wz = m.write_zipper(); + wz.descend_to(&[2u8, 2]); + wz.graft_masked_branches(&src.read_zipper(), mask(bits), false); + assert!(!wz.path_exists(), "{bits:?}"); + } + assert_eq!(dump(&m), vec![(vec![1], Some(1))], "{bits:?}"); + } + + // (c) A masked branch the source lacks is emptied but its location survives; a branch + // the source has is replaced, value included (and a missing source value removes the + // old one). One-, two- and three-bit masks take different code paths. + let src = mk(&[(&[1, 0], 5)]); + for bits in [&[0u8][..], &[0, 1], &[0, 1, 2]] { + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1), (&[1, 1], 1), (&[3], 3)]); + m.write_zipper().graft_masked_branches(&src.read_zipper(), mask(bits), false); + let mut expected = vec![(vec![0], None)]; + if bits.contains(&1) { + expected.extend([(vec![1], None), (vec![1, 0], Some(5))]); + } else { + expected.extend([(vec![1], Some(1)), (vec![1, 1], Some(1))]); + } + expected.push((vec![3], Some(3))); + assert_eq!(dump(&m), expected, "{bits:?}"); + } + // A source whose focus is a bare value (no node) or a dangling path takes yet another + // path, and must behave as an empty source. + for src in [mk(&[(&[], 4)]), { let mut s = PathMap::::new(); s.write_zipper().create_path(); s }] { + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1), (&[3], 3)]); + m.write_zipper().graft_masked_branches(&src.read_zipper(), mask(&[0, 1, 2]), false); + assert_eq!(dump(&m), vec![(vec![0], None), (vec![1], None), (vec![3], Some(3))]); + } + // With `remove_unset` the branches go outright. + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1), (&[3], 3)]); + m.write_zipper().graft_masked_branches(&src.read_zipper(), mask(&[0, 1, 2]), true); + assert_eq!(dump(&m), vec![(vec![1], None), (vec![1, 0], Some(5))]); + + // (d) `graft_child_maps` fed the source's own child subtries agrees with (c). + for bits in [&[0u8][..], &[0, 1], &[0, 1, 2]] { + let maps: Vec> = bits.iter().map(|b| { + let mut z = src.read_zipper(); + z.descend_to_byte(*b); + z.make_map() + }).collect(); + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1), (&[1, 1], 1), (&[3], 3)]); + let mut reference = m.clone(); + m.write_zipper().graft_child_maps(mask(bits), maps, false); + reference.write_zipper().graft_masked_branches(&src.read_zipper(), mask(bits), false); + assert_eq!(dump(&m), dump(&reference), "{bits:?}"); + } + // A map with contents but no root value removes the old value; an empty map empties + // the branch and keeps the location. + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1)]); + m.write_zipper().graft_child_maps(mask(&[0]), vec![mk(&[(&[3], 3)])], false); + assert_eq!(dump(&m), vec![(vec![0], None), (vec![0, 3], Some(3)), (vec![1], Some(1))]); + let mut m = mk(&[(&[0], 9), (&[0, 7], 9), (&[1], 1)]); + m.write_zipper().graft_child_maps(mask(&[0]), vec![PathMap::new()], false); + assert_eq!(dump(&m), vec![(vec![0], None), (vec![1], Some(1))]); + } + #[test] fn write_zipper_graft_masked_branches_test4() { // Upper bound 0: remove_unset=true with an empty mask. From 3e839e81d444fbc07ec238136b7496f6fbd35ae9 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Tue, 15 Sep 2026 23:34:43 +0000 Subject: [PATCH 05/73] Fix join_into replacing or misreporting a destination that holds the source Cherry-pick of 4c793d0 from archive-bugfix/join-into-empty-dst. The tiny-ref-node arm of `pjoin_dyn` swapped its operands with an un-inverted mask, so the destination was replaced by the source; also the merge_guts identity and the integer `pjoin` setting both identity bits when the values are equal. Verified: the class goes from 36 hits on 20000 crate inputs at seed 7 to 0 and agreement rises 19894 -> 19950. The finding 8 status class falls 29 -> 8 with it, which the identity-bit half of this commit accounts for. Total known hits 106 -> 50, nothing new. The upstream commit's differential.py hunk, which retired its own KNOWN entry, is dropped for the same reason as the previous commit. Co-Authored-By: Claude Opus 5 (1M context) --- differential/src/bin/zipper_bug_repros.rs | 13 +++++- lean/FINDINGS.md | 24 ++++++++++ src/dense_byte_node.rs | 10 +++- src/line_list_node.rs | 41 +++++++++------- src/ring.rs | 19 ++++++-- src/write_zipper.rs | 57 +++++++++++++++++++++++ 6 files changed, 140 insertions(+), 24 deletions(-) diff --git a/differential/src/bin/zipper_bug_repros.rs b/differential/src/bin/zipper_bug_repros.rs index c7a0dbe4..6a517ffb 100644 --- a/differential/src/bin/zipper_bug_repros.rs +++ b/differential/src/bin/zipper_bug_repros.rs @@ -92,7 +92,18 @@ fn run(name: &str) { let mut empty = PathMap::::new(); let st = { let mut w = empty.write_zipper(); w.join_into(&rz) }; - println!(" into EMPTY dst -> {st:?}; dst = {} <-- source lost", vals(&empty)); + println!(" into EMPTY dst -> {st:?}; dst = {}", vals(&empty)); + + // The form that outlived the empty-destination fix: a dense destination that already + // holds everything under the source's mid-key focus was *replaced* by the source. + let mut dense = PathMap::::new(); + for i in 0..4u8 { dense.insert(&[i], i as u64); } + let mut src2 = PathMap::::new(); + src2.insert(&[0u8, 0, 0], 0); + let mut rz2 = src2.read_zipper(); + rz2.descend_to(&[0u8, 0]); + let st = { let mut w = dense.write_zipper(); w.join_into(&rz2) }; + println!(" DENSE dst u mid-key src -> {st:?}; dst = {} (expect Identity, 4 values)", vals(&dense)); let mut nonempty = PathMap::::new(); nonempty.insert(&[9u8], 5); diff --git a/lean/FINDINGS.md b/lean/FINDINGS.md index f68b3d4d..ea43df57 100644 --- a/lean/FINDINGS.md +++ b/lean/FINDINGS.md @@ -42,6 +42,30 @@ same `join_into` into a *non-empty* destination produces the right answer. Only the empty-destination case loses the data, and it reports `Identity` — "self was not modified" — rather than failing. +**Addendum (2026-09-11).** The empty-destination case was fixed in PR #70, but the +same key kept firing (99 of 100k seed-11 inputs). Two causes, both in the node +join, neither specific to an empty destination: + +* A source focus partway into a line node is a `TinyRefNode`, and the + `TINY_REF_NODE_TAG` arm of `pjoin_dyn` (dense and list nodes) evaluated the join + with the operands swapped and returned the identity mask un-inverted. A + destination that already contained the source reported `COUNTER_IDENT`, which + `join_into` takes as "the result is the source": a *dense* destination was + replaced by the source (data loss), a list destination merely reported + `Element`. The swapped operands also made the left-biased value join take the + source's values. The arm now expands the tiny node and keeps `self` on the + left. +* `merge_guts`, pairing a slot that holds an onward child with a slot whose key + is longer, always reported the pair as `Element`, even when the child already + held the other payload. It now reports the child slot as the identity. +* The integer `pjoin` placeholders returned `SELF_IDENT` alone even for equal + values; with equal values the result is also `other`, and without that bit a + join seen from the other side can never be a self-identity. (This also + removed two thirds of the finding-8 `join_map_into`/`restrict` hits.) + +Test: `write_zipper_join_into_mid_key_source_keeps_destination`, +`write_zipper_join_into_contained_under_child_is_identity`. + ## 2. A token-maintaining move leaves the zipper navigating wrongly `case: to_next_val_after_step` -- **silent wrong answer** diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 43d3fa91..09dc10b6 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -1326,8 +1326,16 @@ impl> TrieNode self.pjoin(other_byte_node).map(|new_node| TrieNodeODRc::new_in(new_node, self.alloc.clone())) }, TINY_REF_NODE_TAG => { + //Expand the tiny node and keep `self` on the left. Delegating to the tiny node with + // the operands swapped returned its identity mask un-inverted -- a join that left + // `self` unchanged reported COUNTER_IDENT, and `join_into` then replaced the + // destination with the source -- and made the left-biased value join take the + // source's values let tiny_node = unsafe{ other.as_tiny_unchecked() }; - tiny_node.pjoin_dyn(self.as_tagged()) + match tiny_node.into_full() { + Some(full_node) => self.pjoin_dyn(full_node.as_tagged()), + None => AlgebraicResult::Identity(SELF_IDENT), + } } EMPTY_NODE_TAG => { AlgebraicResult::Identity(SELF_IDENT) diff --git a/src/line_list_node.rs b/src/line_list_node.rs index 6013ef25..e1985048 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -1345,14 +1345,18 @@ fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: u unsafe{ intermediate_node.set_payload_owned::<0>(&a_key[overlap..], a_payload); } debug_assert!(validate_node(&intermediate_node)); let intermediate_node = TrieNodeODRc::new_in(intermediate_node, a.alloc.clone()); - let joined = b_child.pjoin(&intermediate_node).unwrap_or_else(|which_arg| { - match which_arg { - 0 => b_child.clone(), - 1 => intermediate_node, - _ => unreachable!() - } - }, || panic!()); - return AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))) + return match b_child.pjoin(&intermediate_node) { + AlgebraicResult::Element(joined) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))), + //`b`'s child already held `a`'s payload, so `b`'s slot *is* the result -- COUNTER_IDENT + // from the caller's point of view. Reporting it as `Element` made a join whose + // destination was unchanged report `Element`. (The other identity, where the child + // held nothing beyond `a`'s payload, is still built as an `Element`: `a`'s slot has + // the same contents but not the `(prefix, child)` shape callers such as + // `drop_head` rely on.) + AlgebraicResult::Identity(mask) if mask & SELF_IDENT > 0 => AlgebraicResult::Identity(COUNTER_IDENT), + AlgebraicResult::Identity(_) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(intermediate_node))), + AlgebraicResult::None => unreachable!(), //`intermediate_node` is never empty + } } if a_key_len == overlap && a.is_child_ptr::() && b_key_len > overlap { let a_child = unsafe{ a.child_in_slot::() }; @@ -1361,14 +1365,13 @@ fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: u unsafe{ intermediate_node.set_payload_owned::<0>(&b_key[overlap..], b_payload); } debug_assert!(validate_node(&intermediate_node)); let intermediate_node = TrieNodeODRc::new_in(intermediate_node, a.alloc.clone()); - let joined = a_child.pjoin(&intermediate_node).unwrap_or_else(|which_arg| { - match which_arg { - 0 => a_child.clone(), - 1 => intermediate_node, - _ => unreachable!() - } - }, || panic!()); - return AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))) + return match a_child.pjoin(&intermediate_node) { + AlgebraicResult::Element(joined) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))), + //Mirror of the case above: `a`'s slot is the result + AlgebraicResult::Identity(mask) if mask & SELF_IDENT > 0 => AlgebraicResult::Identity(SELF_IDENT), + AlgebraicResult::Identity(_) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(intermediate_node))), + AlgebraicResult::None => unreachable!(), //`intermediate_node` is never empty + } } //If we have overlapping initial bytes that can be joined together, make a new prefix node @@ -2649,8 +2652,12 @@ impl TrieNode for LineListNode } }, TINY_REF_NODE_TAG => { + //Expand the tiny node and keep `self` on the left (see DenseByteNode::pjoin_dyn) let tiny_node = unsafe{ other.as_tiny_unchecked() }; - tiny_node.pjoin_dyn(self.as_tagged()) + match tiny_node.into_full() { + Some(full_node) => self.pjoin_dyn(full_node.as_tagged()), + None => AlgebraicResult::Identity(SELF_IDENT), + } } EMPTY_NODE_TAG => { AlgebraicResult::Identity(SELF_IDENT) diff --git a/src/ring.rs b/src/ring.rs index 4b9b1d45..2c781591 100644 --- a/src/ring.rs +++ b/src/ring.rs @@ -851,15 +851,24 @@ impl Lattice for () { fn pmeet(&self, _other: &Self) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT) } } +/// Left-biased join for the plain integer placeholders: the result is always `self`, but when +/// the two are equal it is also `other`, and the node algebra needs to know that to report an +/// unchanged join as `Identity` (a join evaluated with swapped operands, e.g. against a +/// `TinyRefNode`, otherwise never sees a self-identity) +#[inline] +fn left_biased_pjoin(a: &T, b: &T) -> AlgebraicResult { + if a == b { AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT) } else { AlgebraicResult::Identity(SELF_IDENT) } +} + //GOAT trash impl Lattice for usize { - fn pjoin(&self, _other: &usize) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &usize) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &usize) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } //GOAT trash impl Lattice for u64 { - fn pjoin(&self, _other: &u64) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &u64) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &u64) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } @@ -873,13 +882,13 @@ impl DistributiveLattice for u64 { //GOAT trash impl Lattice for u32 { - fn pjoin(&self, _other: &u32) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &u32) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &u32) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } //GOAT trash impl Lattice for u16 { - fn pjoin(&self, _other: &u16) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &u16) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &u16) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } @@ -893,7 +902,7 @@ impl DistributiveLattice for u16 { //GOAT trash impl Lattice for u8 { - fn pjoin(&self, _other: &u8) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } + fn pjoin(&self, other: &u8) -> AlgebraicResult { left_biased_pjoin(self, other) } fn pmeet(&self, _other: &u8) -> AlgebraicResult { AlgebraicResult::Identity(SELF_IDENT) } } diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 7fc45beb..4f3c262c 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -6886,4 +6886,61 @@ mod tests { } assert_eq!(keys(&m), ["cx", "cy", "d"]); } + + /// `join_into` with the source focus partway into a line node. The focus node is then a + /// `TinyRefNode`, and the join used to be evaluated with the operands swapped and the + /// identity mask *not* swapped back. A destination that already held everything the + /// source has reported `COUNTER_IDENT`, which `join_into` takes as "the result is the + /// source" -- so a dense destination was overwritten by the source (the data loss of + /// FINDINGS.md #1, in the form that survived the empty-destination fix), and a list + /// destination reported `Element` for a join that changed nothing. + #[test] + fn write_zipper_join_into_mid_key_source_keeps_destination() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + let src = mk(&[(&[0, 0, 0], 7)]); + + //Dense destination: was replaced by `[0]=7` + let mut dst = mk(&[(&[0], 7), (&[1], 1), (&[2], 2), (&[3], 3)]); + let before = vals(&dst); + let st = { let mut wz = dst.write_zipper(); let mut rz = src.read_zipper(); rz.descend_to(&[0, 0]); wz.join_into(&rz) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //List destination: was `Element` for an unchanged trie + let mut dst = mk(&[(&[0], 7), (&[0, 0], 0)]); + let before = vals(&dst); + let st = { let mut wz = dst.write_zipper(); let mut rz = src.read_zipper(); rz.descend_to(&[0, 0]); wz.join_into(&rz) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //And a join that does add something still says so, with the destination intact + let mut dst = mk(&[(&[1], 1), (&[2], 2), (&[3], 3)]); + let st = { let mut wz = dst.write_zipper(); let mut rz = src.read_zipper(); rz.descend_to(&[0, 0]); wz.join_into(&rz) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vec![(vec![0], 7), (vec![1], 1), (vec![2], 2), (vec![3], 3)]); + } + + /// `join_into` where a destination slot holds an onward child under a key that is a + /// prefix of the source's longer key. `merge_guts` joined the child with the source's + /// remainder but always reported the pair as `Element`, so a source already contained in + /// that child made the whole join report `Element` although nothing changed. + #[test] + fn write_zipper_join_into_contained_under_child_is_identity() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + let mut dst = mk(&[(&[0, 0], 0), (&[0, 1], 0)]); + let before = vals(&dst); + let src = mk(&[(&[0, 0], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.join_into(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //The mirror image: the source holds the child, the destination the longer key + let mut dst = mk(&[(&[0, 0], 0)]); + let src = mk(&[(&[0, 0], 0), (&[0, 1], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.join_into(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vals(&src)); + } } From 4062aa8e06bbf0628f32e1dcefa2d601db54a01b Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Tue, 15 Sep 2026 23:35:57 +0000 Subject: [PATCH 06/73] Drop dangling paths from meet results Cherry-pick of 5197435 from archive-bugfix/meet-drops-dangling-paths: EmptyNode pmeet, the sentinel ptr-eq shortcut, the pmeet_generic COUNTER_IDENT claim and the dense CoFree pmeet. Three existing tests encoded the old behaviour and are changed to the spec's. Verified: all three corpus reproducers for the class (meet_into-keeps-dangling-child, -only-child, meet_k_path_into-keeps- dangling) go from failing to agreeing, the class falls from 12 hits on 20000 crate inputs at seed 7 to 2, and agreement rises 19950 -> 19960. The two that remain are the residual the branch documented: a dangling path deep inside an Arc shared by both operands still survives the ptr-eq shortcut, which returns early without inspecting the subtrie. One is saved as lean/corpus/meet_into-keeps-dangling-residual-arc-shared.bin; it first diverges under subtract_into rather than meet_into, the same shortcut seen through the other operation. Co-Authored-By: Claude Opus 5 (1M context) --- src/dense_byte_node.rs | 9 ++-- src/empty_node.rs | 10 ++-- src/lib.rs | 37 +++++++++---- src/trie_node.rs | 43 ++++++++++----- src/write_zipper.rs | 116 +++++++++++++++++++++++++++++++++++++++-- 5 files changed, 176 insertions(+), 39 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 09dc10b6..05bb43b5 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -1902,12 +1902,9 @@ impl, Other rec_status.merge(val_status, true, true) } fn pmeet(&self, other: &OtherCf) -> AlgebraicResult { - //If one or the other cofree is dangling, it's an identity result for the dangling cofree - let mut identity_flag = 0; - if !self.has_rec() && !self.has_val() {identity_flag = SELF_IDENT;} - if !other.has_rec() && !other.has_val() {identity_flag |= COUNTER_IDENT;} - if identity_flag > 0 { - return AlgebraicResult::Identity(identity_flag) + //A dangling cofree (a path leading to no value) survives no meet + if (!self.has_rec() && !self.has_val()) || (!other.has_rec() && !other.has_val()) { + return AlgebraicResult::None } //Otherwise actually work with what the cofrees contain diff --git a/src/empty_node.rs b/src/empty_node.rs index ce336dd1..df567331 100644 --- a/src/empty_node.rs +++ b/src/empty_node.rs @@ -135,12 +135,10 @@ impl TrieNode for EmptyNode { fn drop_head_dyn(&mut self, _byte_cnt: usize) -> Option> where V: Lattice { None } - fn pmeet_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { - if other.node_is_empty() { - AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT) - } else { - AlgebraicResult::Identity(SELF_IDENT) - } + fn pmeet_dyn(&self, _other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { + //A dangling path leads to no value, so it survives no meet. (Reporting it as an + // identity kept the empty node -- and its path -- in every meet result.) + AlgebraicResult::None } fn psubtract_dyn(&self, _other: TaggedNodeRef) -> AlgebraicResult> where V: DistributiveLattice { AlgebraicResult::None diff --git a/src/lib.rs b/src/lib.rs index 6d38dc03..f5801701 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -514,6 +514,8 @@ mod tests { assert_eq!(map.val_at(&[0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]), None); } + /// A meet keeps a location only if it leads to a surviving value, so dangling paths on either + /// side never survive it -- whatever the branching factor of the nodes involved. #[test] fn map_meet_dangling_branching_factor_test1() { // Left contains a path without a split @@ -526,16 +528,17 @@ mod tests { right.set_val_at([7u8, 2u8, 0u8], 20); right.create_path([7u8, 3u8]); + // A dangling path meets a value to nothing: the whole intersection is empty let intersection = left.meet(&right); - assert_eq!(intersection.path_exists_at([7u8, 1u8, 0u8]), true); //Should have had its value removed, but the path should remain - assert_eq!(intersection.val_at([7u8, 1u8, 0u8]), None); + assert!(intersection.is_empty()); + assert_eq!(intersection.path_exists_at([7u8, 1u8, 0u8]), false); assert_eq!(intersection.path_exists_at([7u8, 2u8, 0u8]), false); assert_eq!(intersection.path_exists_at([7u8, 3u8]), false); //Make sure the result is the same with the opposite operand order let intersection = right.meet(&left); - assert_eq!(intersection.path_exists_at([7u8, 1u8, 0u8]), true); - assert_eq!(intersection.val_at([7u8, 1u8, 0u8]), None); + assert!(intersection.is_empty()); + assert_eq!(intersection.path_exists_at([7u8, 1u8, 0u8]), false); assert_eq!(intersection.path_exists_at([7u8, 2u8, 0u8]), false); assert_eq!(intersection.path_exists_at([7u8, 3u8]), false); @@ -548,29 +551,43 @@ mod tests { right.create_path([7u8, 3u8]); let intersection = left.meet(&right); - assert_eq!(intersection.path_exists_at([7u8, 1u8]), true); //Should have had its value removed, but the path should remain - assert_eq!(intersection.val_at([7u8, 1u8]), None); + assert!(intersection.is_empty()); + assert_eq!(intersection.path_exists_at([7u8, 1u8]), false); assert_eq!(intersection.path_exists_at([7u8, 2u8]), false); assert_eq!(intersection.path_exists_at([7u8, 3u8]), false); //Make sure the result is the same with the opposite operand order let intersection = right.meet(&left); - assert_eq!(intersection.path_exists_at([7u8, 1u8]), true); - assert_eq!(intersection.val_at([7u8, 1u8]), None); + assert!(intersection.is_empty()); + assert_eq!(intersection.path_exists_at([7u8, 1u8]), false); assert_eq!(intersection.path_exists_at([7u8, 2u8]), false); assert_eq!(intersection.path_exists_at([7u8, 3u8]), false); + + // TEST 3. A dangling path next to a surviving value is dropped; the value survives + let mut left: PathMap = PathMap::new(); + left.set_val_at([7u8, 1u8], 10); + left.create_path([7u8, 2u8]); + let mut right: PathMap = PathMap::new(); + right.set_val_at([7u8, 1u8], 10); + right.set_val_at([7u8, 2u8], 20); + for intersection in [left.meet(&right), right.meet(&left)] { + assert_eq!(intersection.val_count(), 1); + assert_eq!(intersection.path_exists_at([7u8, 1u8]), true); + assert_eq!(intersection.path_exists_at([7u8, 2u8]), false); + } } #[test] fn map_meet_dangling_branching_factor_test2() { - //Test 1: Path subsets + //Test 1: Path subsets. Two dangling paths, one a prefix of the other, meet to nothing let mut left: PathMap<()> = PathMap::new(); left.create_path(b"OneTwo"); let mut right: PathMap<()> = PathMap::new(); right.create_path(b"OneTwoThree"); let intersection = left.meet(&right); - assert_eq!(intersection.path_exists_at(b"OneTwo"), true); + assert!(intersection.is_empty()); + assert_eq!(intersection.path_exists_at(b"OneTwo"), false); assert_eq!(intersection.path_exists_at(b"OneTwoThree"), false); assert_eq!(intersection.path_exists_at(b"OneTwoT"), false); } diff --git a/src/trie_node.rs b/src/trie_node.rs index 27366659..f0ec6908 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -764,6 +764,16 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: } else { pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results); + //`other` holds no payload at this key, so the result has nothing here. That equals + // `other` at this key only if `other` has no path along it at all. A path that shares a + // prefix with the key -- typically a dangling path, which the lookup above reports as + // "nothing found, everything covered" -- is not carried into the result, so claiming + // `COUNTER_IDENT` would hand the caller `other` with the dangling path still in it. + let nothing_here = if other_node.node_key_overlap(keys[idx].0) == 0 { + FatAlgebraicResult::new(COUNTER_IDENT, None) + } else { + FatAlgebraicResult::none() + }; let result = match &self_payloads[idx].1 { PayloadRef::Child(self_link) => { match other_node.get_node_at_key(keys[idx].0).into_option() { @@ -772,20 +782,14 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: FatAlgebraicResult::from_binary_op_result(result, self_link, &other_onward_node) .map(|child| ValOrChild::Child(child)) }, - None => { - //Check to see if we have a dangling path, because a dangling path meet with a value should result in a path, but no value - if self_link.is_empty() && other_node.node_get_val(keys[idx].0).is_some() { - FatAlgebraicResult::new(SELF_IDENT, Some(ValOrChild::Child(TrieNodeODRc::new_empty()))) - } else { - FatAlgebraicResult::new(COUNTER_IDENT, None) - } - } + //Nothing in `other` below this key -- whether or not `self` is dangling here. + // A meet keeps only locations that lead to a surviving value, so a dangling + // `self` path meeting a value in `other` also yields nothing. + None => nothing_here, } }, - PayloadRef::Val(_self_val) => { - //If self_payload is a val and we didn't get a corresponding val, then this result is None - FatAlgebraicResult::new(COUNTER_IDENT, None) - }, + //If self_payload is a val and we didn't get a corresponding val, then this result is None + PayloadRef::Val(_self_val) => nothing_here, _ => unreachable!() }; results[idx] = result; @@ -1525,6 +1529,11 @@ mod tagged_node_ref { } pub fn pmeet_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { + //An empty node (dangling path) meets to nothing, even with itself: the sentinel is + // shared, so this must come before the identity shortcut + if self.node_is_empty() || other.node_is_empty() { + return AlgebraicResult::None; + } if self.shared_node_id() == other.shared_node_id() { return AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT); } @@ -2187,6 +2196,11 @@ mod tagged_node_ref { } pub fn pmeet_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { + //An empty node (dangling path) meets to nothing, even with itself: the sentinel is + // shared, so this must come before the identity shortcut + if self.node_is_empty() || other.node_is_empty() { + return AlgebraicResult::None; + } if self.ptr == other.ptr { return AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT); } @@ -3291,7 +3305,10 @@ impl TrieNodeODRc { } #[inline] pub fn pmeet(&self, other: &Self) -> AlgebraicResult { - if self.ptr_eq(other) { + if self.is_empty() || other.is_empty() { + //A dangling path survives no meet; see `EmptyNode::pmeet_dyn` + AlgebraicResult::None + } else if self.ptr_eq(other) { AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT) } else { self.as_tagged().pmeet_dyn(other.as_tagged()) diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 4f3c262c..7e0f2644 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -3749,7 +3749,10 @@ mod tests { assert_eq!(btm.path_exists_at(&[1, 255, 0]), true); assert_eq!(btm.path_exists_at(&[0, 255, 0]), true); - // Test 3: meet from a higher level with all dangling paths and prune=true + // Test 3: meet from a higher level with all dangling paths and prune=true. A location + // survives a meet only if it leads to a surviving value, so dangling paths never survive + // one -- even where both sides hold the same dangling path. The result is empty, and with + // `prune = true` the focus path itself goes. let mut btm2: PathMap<()> = PathMap::new(); btm2.create_path(&[0, 255, 0]); btm2.create_path(&[0, 255, 1]); @@ -3760,16 +3763,121 @@ mod tests { let mut wz = zh2.write_zipper_at_exclusive_path(&[0]).unwrap(); let rz = zh2.read_zipper_at_path(&[1]).unwrap(); let alg_result = wz.meet_into(&rz, true); - assert_eq!(alg_result, AlgebraicStatus::Element); - drop(wz); + assert_eq!(alg_result, AlgebraicStatus::None); + zh2.cleanup_write_zipper(wz); drop(rz); drop(zh2); // Verify the meet operation did what it should have assert_eq!(btm2.path_exists_at(&[1, 255, 0]), true); - assert_eq!(btm2.path_exists_at(&[0, 255, 0]), true); + assert_eq!(btm2.path_exists_at(&[0, 255, 0]), false); assert_eq!(btm2.path_exists_at(&[0, 200, 5]), false); assert_eq!(btm2.path_exists_at(&[0, 255, 1]), false); + assert_eq!(btm2.path_exists_at(&[0]), false); + } + + /// A meet keeps a location only if it leads to a surviving value, so a dangling path on either + /// side is dropped -- not kept as an identity. Covers the two-slot LineListNode and the + /// DenseByteNode CoFree cases, the shared empty sentinel meeting itself, `meet_k_path_into` + /// and `PathMap::meet`. + #[test] + fn write_zipper_meet_into_drops_dangling_paths() { + // LineListNode: dst = { [2] = 0 } plus a dangling [1]; src = { [1] = 5, [2] = 0 } + let mut dst = PathMap::::new(); + dst.set_val_at(&[2u8], 0); + dst.create_path(&[1u8]); + let mut src = PathMap::::new(); + src.set_val_at(&[1u8], 5); + src.set_val_at(&[2u8], 0); + let mut wz = dst.write_zipper(); + assert_eq!(wz.meet_into(&src.read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(wz.child_count(), 1); + drop(wz); + assert_eq!(dst.path_exists_at(&[1u8]), false); + assert_eq!(dst.get_val_at(&[2u8]), Some(&0)); + + // Only a dangling child left: the whole result is empty + let mut dst = PathMap::::new(); + dst.create_path(&[1u8]); + let mut wz = dst.write_zipper(); + assert_eq!(wz.meet_into(&src.read_zipper(), false), AlgebraicStatus::None); + assert_eq!(wz.child_count(), 0); + drop(wz); + assert_eq!(dst.path_exists_at(&[1u8]), false); + + // The same dangling path on both sides is not an identity either + let mut dst = PathMap::::new(); + dst.create_path(&[1u8]); + let mut both = PathMap::::new(); + both.create_path(&[1u8]); + let mut wz = dst.write_zipper(); + assert_eq!(wz.meet_into(&both.read_zipper(), false), AlgebraicStatus::None); + assert_eq!(wz.child_count(), 0); + drop(wz); + + // DenseByteNode: four children, one of them dangling + let mut dst = PathMap::::new(); + for b in [0u8, 2, 3] { dst.set_val_at(&[b], 0); } + dst.create_path(&[1u8]); + let mut src = PathMap::::new(); + for b in [0u8, 1, 2, 3] { src.set_val_at(&[b], 0); } + let mut wz = dst.write_zipper(); + assert_eq!(wz.meet_into(&src.read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(wz.child_count(), 3); + drop(wz); + assert_eq!(dst.path_exists_at(&[1u8]), false); + assert_eq!(dst.val_count(), 3); + + // DenseByteNode destination holding only a dangling cofree at [2] and a subtree at [3], met + // with a LineListNode source keyed [2, 2] and [3]: the lookup for [2, 2] finds nothing and + // used to report the result as identical to the destination, dangling [2] included. + let mut dst = PathMap::::new(); + for b in [2u8, 4, 5] { dst.set_val_at(&[b], 1); } + dst.set_val_at(&[3u8, 0], 9); + dst.remove_val_at(&[4u8], true); + dst.remove_val_at(&[5u8], true); + let mut src = PathMap::::new(); + src.set_val_at(&[2u8, 2], 204); + src.set_val_at(&[3u8, 0], 9); + let nothing = PathMap::::new(); + let mut nothing_rz = nothing.read_zipper(); + nothing_rz.descend_to(&[1u8]); + let mut wz = dst.write_zipper(); + wz.descend_to(&[2u8]); + wz.graft(¬hing_rz); + wz.ascend(1); + assert_eq!(wz.path_exists(), true); + let mut probe = wz.fork_read_zipper(); + probe.descend_to(&[2u8]); + assert_eq!(probe.path_exists(), true, "the graft of nothing should leave [2] dangling"); + drop(probe); + assert_eq!(wz.meet_into(&src.read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(wz.child_count(), 1); + drop(wz); + assert_eq!(dst.path_exists_at(&[2u8]), false); + assert_eq!(dst.get_val_at(&[3u8, 0]), Some(&9)); + + // meet_k_path_into: the k-paths [0] -> { [2] = 0 } and [1] -> { [2, 0] = 0 } meet to nothing, + // since [2] is dangling on the second. Used to leave a dangling [2] behind and report true. + let mut map = PathMap::::new(); + map.set_val_at(&[0u8, 2], 0); + map.set_val_at(&[1u8, 2, 0], 0); + let mut wz = map.write_zipper(); + assert_eq!(wz.meet_k_path_into(1, false), false); + assert_eq!(wz.child_count(), 0); + drop(wz); + assert_eq!(map.val_count(), 0); + + // PathMap::meet + let mut a = PathMap::::new(); + a.set_val_at(&[1u8], 0); + a.create_path(&[2u8]); + let mut b = PathMap::::new(); + b.set_val_at(&[1u8], 0); + b.set_val_at(&[2u8], 0); + let m = a.meet(&b); + assert_eq!(m.val_count(), 1); + assert_eq!(m.path_exists_at(&[2u8]), false); } /// Tests whether the [WriteZipper::subtract_into] operation will do the right thing with the root value From 3dae731fb74e4b0e4bc6ec447dc32ce4e33a622f Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Tue, 15 Sep 2026 23:38:18 +0000 Subject: [PATCH 07/73] Make meet/join value bias independent of node layout Cherry-pick of 94b4043 from archive-bugfix/value-bias-by-node-layout: five swapped-orientation sites in trie_node/line_list_node/ dense_byte_node, and join_k_path_into's fold made ascending to match `PathMap.dropHead`. Verified: all four corpus reproducers (join_into, join_k_path_into, meet_2, meet_into) go from failing to agreeing, the class falls from 10 hits to 0 on 20000 crate inputs and 2 to 0 on 5000 ACT inputs at seed 7, and agreement rises 19960 -> 19975 and 4957 -> 4959. The finding 8 class falls 20 -> 16 and the dangling class 2 -> 1 with it. Three hunks conflicted with the join_into commit applied earlier. The two tiny-ref dispatch arms keep that commit's form, which fixes the same orientation and also reports the identity mask. The third is a real overlap: it takes this commit's operand order, since the join is left-biased and `a` must be on the left, while keeping the join_into commit's identity reporting, reading COUNTER_IDENT where it read SELF_IDENT because swapping the operands swaps what the bits refer to. Co-Authored-By: Claude Opus 5 (1M context) --- src/dense_byte_node.rs | 119 +++++++++++++++++++++++++++++--- src/line_list_node.rs | 149 +++++++++++++++++++++++++---------------- src/trie_node.rs | 120 +++++++++++++++++++++++++++++---- 3 files changed, 308 insertions(+), 80 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 05bb43b5..ac7b6064 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -260,6 +260,94 @@ impl> ByteNode } } + /// [Self::join_child_into] with `node` as the *left* operand of the join, so on a collision + /// `node`'s values take precedence over `self`'s. The status is still relative to `self`. + pub(crate) fn join_child_into_left(&mut self, k: u8, node: TrieNodeODRc) -> AlgebraicStatus where V: Clone + Lattice { + let ix = self.mask.index_of(k) as usize; + if self.mask.test_bit(k) { + let cf = unsafe { self.values.get_unchecked_mut(ix) }; + match cf.rec_mut() { + Some(existing_node) => { + match node.pjoin(existing_node) { + //`COUNTER_IDENT` means the result is `existing_node`: nothing to do + AlgebraicResult::Identity(mask) if mask & COUNTER_IDENT > 0 => AlgebraicStatus::Identity, + AlgebraicResult::Identity(_) => { + *existing_node = node; + AlgebraicStatus::Element + }, + AlgebraicResult::Element(joined) => { + *existing_node = joined; + AlgebraicStatus::Element + }, + //Only two empty nodes join to nothing, and then there is nothing to change + AlgebraicResult::None => AlgebraicStatus::Identity, + } + }, + None => { + cf.set_rec(node); + AlgebraicStatus::Element + } + } + } else { + self.mask.set_bit(k); + let new_cf = CoFree::new(Some(node), None); + self.values.insert(ix, new_cf); + AlgebraicStatus::Element + } + } + + /// [Self::join_val_into] with `val` as the *left* operand of the join; see [Self::join_child_into_left] + pub(crate) fn join_val_into_left(&mut self, k: u8, val: V) -> AlgebraicStatus where V: Lattice { + let ix = self.mask.index_of(k) as usize; + if self.mask.test_bit(k) { + let cf = unsafe { self.values.get_unchecked_mut(ix) }; + match cf.val_mut() { + Some(existing_val) => { + match val.pjoin(existing_val) { + AlgebraicResult::Identity(mask) if mask & COUNTER_IDENT > 0 => AlgebraicStatus::Identity, + AlgebraicResult::Identity(_) => { + *existing_val = val; + AlgebraicStatus::Element + }, + AlgebraicResult::Element(joined) => { + *existing_val = joined; + AlgebraicStatus::Element + }, + //A join of two present values never has an empty result; see `Lattice::join_into` + AlgebraicResult::None => AlgebraicStatus::Identity, + } + } + None => { + cf.set_val(val); + AlgebraicStatus::Element + } + } + } else { + self.mask.set_bit(k); + let new_cf = CoFree::new(None, Some(val)); + self.values.insert(ix, new_cf); + AlgebraicStatus::Element + } + } + + /// Dispatches to [Self::join_child_into] or [Self::join_child_into_left] + #[inline] + pub(crate) fn join_child_into_oriented(&mut self, k: u8, node: TrieNodeODRc, incoming_is_left: bool) -> AlgebraicStatus where V: Clone + Lattice { + if incoming_is_left { self.join_child_into_left(k, node) } else { self.join_child_into(k, node) } + } + + /// Dispatches to [Self::join_payload_into] or its left-biased counterpart + #[inline] + pub(crate) fn join_payload_into_oriented(&mut self, k: u8, payload: ValOrChild, incoming_is_left: bool) -> AlgebraicStatus where V: Clone + Lattice { + if !incoming_is_left { + return self.join_payload_into(k, payload) + } + match payload { + ValOrChild::Child(child) => self.join_child_into_left(k, child), + ValOrChild::Val(val) => self.join_val_into_left(k, val), + } + } + /// Internal method to remove a CoFree from the node #[inline] fn remove(&mut self, k: u8) -> Option { @@ -548,7 +636,13 @@ impl> ByteNode } /// Merges the entries in the ListNode into the ByteNode - pub fn merge_from_list_node(&mut self, list_node: &LineListNode) -> AlgebraicStatus where V: Clone + Lattice { + /// Joins the contents of `list_node` into `self`. + /// + /// The join is left-biased, so `list_is_left` says which operand `list_node` is: `false` when + /// the caller is computing `self ∪ list_node` (collisions keep `self`'s values), `true` when it + /// is computing `list_node ∪ self` into a clone of the byte node (collisions keep the list + /// node's values). The returned status is always relative to `self`. + pub fn merge_from_list_node(&mut self, list_node: &LineListNode, list_is_left: bool) -> AlgebraicStatus where V: Clone + Lattice { let self_was_empty = self.is_empty(); self.reserve_capacity(2); @@ -558,9 +652,9 @@ impl> ByteNode if key.len() > 1 { let mut child_node = LineListNode::::new_in(self.alloc.clone()); unsafe{ child_node.set_payload_owned::<0>(&key[1..], payload); } - self.join_child_into(key[0], TrieNodeODRc::new_in(child_node, self.alloc.clone())) + self.join_child_into_oriented(key[0], TrieNodeODRc::new_in(child_node, self.alloc.clone()), list_is_left) } else { - self.join_payload_into(key[0], payload) + self.join_payload_into_oriented(key[0], payload, list_is_left) } } else { if self_was_empty { @@ -576,9 +670,9 @@ impl> ByteNode if key.len() > 1 { let mut child_node = LineListNode::::new_in(self.alloc.clone()); unsafe{ child_node.set_payload_owned::<0>(&key[1..], payload); } - self.join_child_into(key[0], TrieNodeODRc::new_in(child_node, self.alloc.clone())) + self.join_child_into_oriented(key[0], TrieNodeODRc::new_in(child_node, self.alloc.clone()), list_is_left) } else { - self.join_payload_into(key[0], payload) + self.join_payload_into_oriented(key[0], payload, list_is_left) } } else { if self_was_empty { @@ -1311,7 +1405,7 @@ impl> TrieNode LINE_LIST_NODE_TAG => { let other_list_node = unsafe{ other.as_list_unchecked() }; let mut new_node = self.clone(); - let status = new_node.merge_from_list_node(other_list_node); + let status = new_node.merge_from_list_node(other_list_node, false); AlgebraicResult::from_status(status, || TrieNodeODRc::new_in(new_node, self.alloc.clone())) }, #[cfg(feature = "bridge_nodes")] @@ -1360,7 +1454,7 @@ impl> TrieNode let other_list_node = unsafe{ other_node.into_list_unchecked() }; //GOAT, optimization opportunity to take the contents from the list, rather than cloning // them, to turn around and drop the ListNode and free them / decrement the refcounts - self.merge_from_list_node(other_list_node) + self.merge_from_list_node(other_list_node, false) }, #[cfg(feature = "bridge_nodes")] TaggedNodeRefMut::BridgeNode(_other_bridge_node) => { @@ -1398,7 +1492,9 @@ impl> TrieNode }, _ => { let mut new_node = Self::new_in(self.alloc.clone()); - while let Some(cf) = self.values.pop() { + //Ascending key order, with the accumulated node as the left operand of every join, + // so a collision keeps the value from the lexicographically first path + for cf in self.values.drain(..) { let child = cf.into_rec().filter(|child| !child.is_empty()); let child = if byte_cnt > 1 { child.and_then(|mut child| child.make_mut().drop_head_dyn(byte_cnt-1)) @@ -1431,7 +1527,9 @@ impl> TrieNode }, LINE_LIST_NODE_TAG => { let other_list_node = unsafe { other.as_list_unchecked() }; - other_list_node.pmeet_dyn(self.as_tagged()).invert_identity() + //`self` is the left operand; the list node enumerates its payloads but must resolve + // every collision as `self op list`, hence `swapped`. + other_list_node.pmeet_dyn_oriented(self.as_tagged(), true).invert_identity() }, #[cfg(feature = "bridge_nodes")] TaggedNodeRef::BridgeNode(other_bridge_node) => { @@ -1443,7 +1541,8 @@ impl> TrieNode }, TINY_REF_NODE_TAG => { let tiny_node = unsafe { other.as_tiny_unchecked() }; - tiny_node.pmeet_dyn(self.as_tagged()).invert_identity() + let full_node = tiny_node.into_full().unwrap(); + self.pmeet_dyn(full_node.as_tagged()) }, EMPTY_NODE_TAG => AlgebraicResult::None, _ => unsafe{ unreachable_unchecked() } diff --git a/src/line_list_node.rs b/src/line_list_node.rs index e1985048..6f44e25f 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -1304,6 +1304,35 @@ fn try_merge<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: us } /// The part of `try_merge` that we probably shouldn't inline +/// The subtrie left of one list-node slot after `byte_cnt` bytes are dropped from every path +/// below it, or `None` if nothing is left. Part of [TrieNode::drop_head_dyn]. +/// +/// A value sitting at depth `<= byte_cnt` is discarded (the joined node has nowhere to put a root +/// value), a dangling child has nothing below the dropped bytes, and a key longer than `byte_cnt` +/// just loses its first `byte_cnt` bytes. +fn drop_head_from_payload(key: &[u8], payload: ValOrChild, byte_cnt: usize, alloc: &A) -> Option> { + let key_len = key.len(); + if byte_cnt < key_len { + let mut new_node = LineListNode::new_in(alloc.clone()); + unsafe { new_node.set_payload_owned::<0>(&key[byte_cnt..], payload); } + debug_assert!(validate_node(&new_node)); + return Some(TrieNodeODRc::new_in(new_node, alloc.clone())) + } + match payload { + ValOrChild::Val(_) => None, + ValOrChild::Child(mut child) => { + if child.is_empty() { + return None + } + if byte_cnt == key_len { + Some(child) + } else { + child.make_mut().drop_head_dyn(byte_cnt - key_len) + } + } + } +} + fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: usize, const BSLOT: usize>(mut overlap: usize, a_key: &'a[u8], a: &LineListNode, b_key: &'a[u8], b: &LineListNode) -> AlgebraicResult<(&'a[u8], ValOrChild)> { debug_assert!(overlap > 0); let a_key_len = a_key.len(); @@ -1345,15 +1374,18 @@ fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: u unsafe{ intermediate_node.set_payload_owned::<0>(&a_key[overlap..], a_payload); } debug_assert!(validate_node(&intermediate_node)); let intermediate_node = TrieNodeODRc::new_in(intermediate_node, a.alloc.clone()); - return match b_child.pjoin(&intermediate_node) { + //`a` is the left operand of this merge, so its payload must be the left operand + // of the join (the join is left-biased on colliding values). + return match intermediate_node.pjoin(b_child) { AlgebraicResult::Element(joined) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(joined))), //`b`'s child already held `a`'s payload, so `b`'s slot *is* the result -- COUNTER_IDENT // from the caller's point of view. Reporting it as `Element` made a join whose // destination was unchanged report `Element`. (The other identity, where the child // held nothing beyond `a`'s payload, is still built as an `Element`: `a`'s slot has // the same contents but not the `(prefix, child)` shape callers such as - // `drop_head` rely on.) - AlgebraicResult::Identity(mask) if mask & SELF_IDENT > 0 => AlgebraicResult::Identity(COUNTER_IDENT), + // `drop_head` rely on.) The bit read here is COUNTER_IDENT rather than SELF_IDENT + // because `b_child` is now the right operand. + AlgebraicResult::Identity(mask) if mask & COUNTER_IDENT > 0 => AlgebraicResult::Identity(COUNTER_IDENT), AlgebraicResult::Identity(_) => AlgebraicResult::Element((&a_key[0..overlap], ValOrChild::Child(intermediate_node))), AlgebraicResult::None => unreachable!(), //`intermediate_node` is never empty } @@ -2624,7 +2656,7 @@ impl TrieNode for LineListNode DENSE_BYTE_NODE_TAG => { let other_dense_node = unsafe{ other.as_dense_unchecked() }; let mut new_node = other_dense_node.clone(); - match new_node.merge_from_list_node(self) { + match new_node.merge_from_list_node(self, true) { //Both nodes were empty so the join is empty too AlgebraicStatus::None => { debug_assert!(self.node_is_empty() && other_dense_node.node_is_empty()); @@ -2641,7 +2673,7 @@ impl TrieNode for LineListNode CELL_BYTE_NODE_TAG => { let other_dense_node = unsafe{ other.as_dense_unchecked() }; let mut new_node = other_dense_node.clone(); - match new_node.merge_from_list_node(self) { + match new_node.merge_from_list_node(self, true) { //See the DENSE_BYTE_NODE_TAG arm: two empty nodes join to an empty result AlgebraicStatus::None => { debug_assert!(self.node_is_empty() && other_dense_node.node_is_empty()); @@ -2677,7 +2709,7 @@ impl TrieNode for LineListNode DENSE_BYTE_NODE_TAG => { let other_dense_node = unsafe{ other_node.as_dense_unchecked() }; let mut new_node = other_dense_node.clone(); - let status = new_node.merge_from_list_node(self); + let status = new_node.merge_from_list_node(self, true); debug_assert!(!status.is_none()); (AlgebraicStatus::Element, Err(TrieNodeODRc::new_in(new_node, self.alloc.clone()))) }, @@ -2688,7 +2720,7 @@ impl TrieNode for LineListNode CELL_BYTE_NODE_TAG => { let other_dense_node = unsafe{ other_node.as_cell_unchecked() }; let mut new_node = other_dense_node.clone(); - let status = new_node.merge_from_list_node(self); + let status = new_node.merge_from_list_node(self, true); debug_assert!(!status.is_none()); (AlgebraicStatus::Element, Err(TrieNodeODRc::new_in(new_node, self.alloc.clone()))) }, @@ -2848,45 +2880,63 @@ impl TrieNode for LineListNode return Some(TrieNodeODRc::new_in(temp_node, self.alloc.clone())) } - //The final case is to construct a brand new node from the remaining parts of the key after we have - // discarded what we can discard and then merged together what's left. And then call this function - // recursively on the newly merged nodes - let chop_bytes = key0_len.min(key1_len); - debug_assert!(chop_bytes <= byte_cnt); - debug_assert!(chop_bytes > 0); - let new_key0 = &key0[chop_bytes-1..]; - let new_key1 = &key1[chop_bytes-1..]; - - let overlap = find_prefix_overlap(&key0[chop_bytes..], &key1[chop_bytes..]); - let merged_payload = match merge_guts::(overlap+1, new_key0, &temp_node, new_key1, &temp_node) { - AlgebraicResult::Element((_shared_key, merged_payload)) => merged_payload, - AlgebraicResult::Identity(mask) => { - if mask & SELF_IDENT > 0 { - temp_node.clone_payload::<0>().unwrap() - } else { - debug_assert_eq!(mask, COUNTER_IDENT); - temp_node.clone_payload::<1>().unwrap() - } - }, - AlgebraicResult::None => unreachable!() //`merge_guts` shouldn't return AlgebraicResult::None because that should have been caught by an earlier case + //The final case: at least one key is no longer than `byte_cnt`. Drop the bytes from each + // slot on its own and join the two results, slot 0 (the lexicographically smaller key) on + // the left. That is the order `PathMap::drop_head` is specified in -- a fold over the + // k-paths in sorted order, keeping the first value on a collision -- and it is what the + // byte node does one level up. Merging the two slots at an intermediate depth and then + // dropping the remaining bytes, as this used to, joins the subtries in a different order + // and keeps different values. + let mut key0_buf: [MaybeUninit; KEY_BYTES_CNT] = [MaybeUninit::new(0); KEY_BYTES_CNT]; + let mut key1_buf: [MaybeUninit; KEY_BYTES_CNT] = [MaybeUninit::new(0); KEY_BYTES_CNT]; + let (key0, key1) = unsafe { + core::ptr::copy_nonoverlapping(key0.as_ptr(), key0_buf.as_mut_ptr().cast::(), key0_len); + core::ptr::copy_nonoverlapping(key1.as_ptr(), key1_buf.as_mut_ptr().cast::(), key1_len); + (core::slice::from_raw_parts(key0_buf.as_ptr().cast::(), key0_len), + core::slice::from_raw_parts(key1_buf.as_ptr().cast::(), key1_len)) }; - - if let ValOrChild::Child(mut child_node) = merged_payload { - //A dangling child (the empty sentinel) has nothing below the dropped bytes and can't be made mutable - if child_node.is_empty() { - return None - } - if chop_bytes == byte_cnt { - return Some(child_node) - } else { - return child_node.make_mut().drop_head_dyn(byte_cnt-chop_bytes) + //Take slot 1 first: taking slot 0 would shift slot 1 into its place. + let payload1 = temp_node.take_payload::<1>().unwrap(); + let payload0 = temp_node.take_payload::<0>().unwrap(); + let dropped0 = drop_head_from_payload(key0, payload0, byte_cnt, &self.alloc); + let dropped1 = drop_head_from_payload(key1, payload1, byte_cnt, &self.alloc); + match (dropped0, dropped1) { + (None, None) => None, + (Some(node), None) | (None, Some(node)) => Some(node), + (Some(node0), Some(node1)) => match node0.pjoin(&node1) { + AlgebraicResult::Element(joined) => Some(joined), + AlgebraicResult::Identity(mask) => Some(if mask & SELF_IDENT > 0 { node0 } else { node1 }), + AlgebraicResult::None => None, } } - - unreachable!() } fn pmeet_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { + self.pmeet_dyn_oriented(other, false) + } + fn psubtract_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: DistributiveLattice { + debug_assert!(validate_node(self)); + let slot0_result = self.subtract_from_slot_contents::<0>(other); + let slot1_result = self.subtract_from_slot_contents::<1>(other); + self.combine_slot_results_into_node_result(slot0_result, slot1_result) + } + fn prestrict_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> { + debug_assert!(validate_node(self)); + let slot0_result = self.restrict_slot_contents::<0>(other); + let slot1_result = self.restrict_slot_contents::<1>(other); + self.combine_slot_results_into_node_result(slot0_result, slot1_result) + } + fn clone_self(&self) -> TrieNodeODRc { + TrieNodeODRc::new_in(self.clone(), self.alloc.clone()) + } +} + +impl LineListNode { + /// The body of [TrieNode::pmeet_dyn]. `swapped` means `self` is really the *right* operand of + /// the meet and `other` the left one; see [pmeet_generic]. A node type that cannot enumerate + /// its own payloads cheaply (a `ByteNode`) meets a list node by calling this with `swapped = + /// true` and inverting the identity mask of the result. + pub(crate) fn pmeet_dyn_oriented(&self, other: TaggedNodeRef, swapped: bool) -> AlgebraicResult> where V: Lattice { debug_assert!(validate_node(self)); let mut self_payloads_buf: [(&[u8], PayloadRef); 2] = [(&[], PayloadRef::None); 2]; @@ -2911,7 +2961,7 @@ impl TrieNode for LineListNode _ => unsafe{ unreachable_unchecked() } }; - pmeet_generic::<2, V, A, _>(self_payloads, other, |payloads| { + pmeet_generic::<2, V, A, _>(self_payloads, other, swapped, |payloads| { debug_assert_eq!(payloads.len(), self_payloads.len()); let slot0_payload = payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into()); let slot1_payload = payloads.get_mut(1).and_then(|p| core::mem::take(p)).map(|p| p.into()); @@ -2919,24 +2969,7 @@ impl TrieNode for LineListNode TrieNodeODRc::new_in(new_node, self.alloc.clone()) }) } - fn psubtract_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: DistributiveLattice { - debug_assert!(validate_node(self)); - let slot0_result = self.subtract_from_slot_contents::<0>(other); - let slot1_result = self.subtract_from_slot_contents::<1>(other); - self.combine_slot_results_into_node_result(slot0_result, slot1_result) - } - fn prestrict_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> { - debug_assert!(validate_node(self)); - let slot0_result = self.restrict_slot_contents::<0>(other); - let slot1_result = self.restrict_slot_contents::<1>(other); - self.combine_slot_results_into_node_result(slot0_result, slot1_result) - } - fn clone_self(&self) -> TrieNodeODRc { - TrieNodeODRc::new_in(self.clone(), self.alloc.clone()) - } -} -impl LineListNode { /// Part of the implementation of methods the remove subtries from a node fn remove_subtries(&mut self, remove_0: bool, remove_1: bool, key0_starts_with: bool, prune: bool, key_len: usize) { //NOTE: the order here is important because removing slot_0 first might shift the diff --git a/src/trie_node.rs b/src/trie_node.rs index f0ec6908..984bc8ee 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -642,7 +642,16 @@ impl ValOrChildUnion { // was observed. Therefore the the ~20% slowdown is simply the higher overheads of this generic function. // //The next port of call for optimization is probably to remove the recursion -pub(crate) fn pmeet_generic(self_payloads: &[(&[u8], PayloadRef)], other: TaggedNodeRef, merge_f: MergeF) -> AlgebraicResult> +// +/// `swapped` says which operand `self_payloads` came from. The meet is left-biased (a `Lattice` +/// impl resolves a collision as `left.pmeet(right)`), so when a caller enumerates the *right* +/// operand's payloads because that node type is the easier one to iterate, it passes `swapped = +/// true`: every value and every recursive node meet is then computed as `other op self` and the +/// identity masks are re-expressed relative to `self_payloads`. The caller still applies +/// `invert_identity()` to the final result to get back to its own orientation. Without this, a +/// dense-node-versus-list-node meet returned the list node's values regardless of which side it +/// was on. +pub(crate) fn pmeet_generic(self_payloads: &[(&[u8], PayloadRef)], other: TaggedNodeRef, swapped: bool, merge_f: MergeF) -> AlgebraicResult> where MergeF: FnOnce(&mut [Option>]) -> TrieNodeODRc, V: Clone + Send + Sync + Lattice @@ -657,7 +666,7 @@ pub(crate) fn pmeet_generic(self_payloads, &mut request_keys[..], &mut request_results[..], &mut element_results[..], other); + let is_exhaustive = pmeet_generic_internal::(self_payloads, &mut request_keys[..], &mut request_results[..], &mut element_results[..], other, swapped); let mut is_none = true; let mut combined_mask = SELF_IDENT | COUNTER_IDENT; let mut result_payloads = ArrayVec::>, MAX_PAYLOAD_CNT>::new(); @@ -697,7 +706,7 @@ pub(crate) fn node_count_branches_recursive(self_payloads: &[(&[u8], PayloadRef)], keys: &mut [(&[u8], bool)], request_results: &mut [(usize, PayloadRef<'trie, V, A>)], results: &mut [FatAlgebraicResult>], other_node: TaggedNodeRef<'trie, V, A>) -> bool +pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: Allocator>(self_payloads: &[(&[u8], PayloadRef)], keys: &mut [(&[u8], bool)], request_results: &mut [(usize, PayloadRef<'trie, V, A>)], results: &mut [FatAlgebraicResult>], other_node: TaggedNodeRef<'trie, V, A>, swapped: bool) -> bool where V: Clone + Send + Sync + Lattice { //If is_exhaustive gets set to `false`, then the pmeet method cannot return a `COUNTER_IDENTITY` result @@ -729,14 +738,14 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: // we have the same node as the previous time through the loop if cur_group.is_some() { if (cur_group.as_ref().unwrap().1 as *const TrieNodeODRc) != (child as *const TrieNodeODRc) { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results); + pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results, swapped); cur_group = Some((idx, child)); } } else { cur_group = Some((idx, child)); } } else { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results); + pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results, swapped); //We've arrived at a contained value or onward link that has a correspondence // to one of the values or links in `self` @@ -745,13 +754,13 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: let result = match &self_payloads[idx].1 { PayloadRef::Child(self_link) => { let other_link = payload.child(); - let result = self_link.pmeet(other_link); + let result = if swapped { other_link.pmeet(self_link).invert_identity() } else { self_link.pmeet(other_link) }; FatAlgebraicResult::from_binary_op_result(result, self_link, other_link) .map(|child| ValOrChild::Child(child)) }, PayloadRef::Val(self_val) => { let other_val = payload.val(); - let result = (*self_val).pmeet(other_val); + let result = if swapped { other_val.pmeet(*self_val).invert_identity() } else { (*self_val).pmeet(other_val) }; FatAlgebraicResult::from_binary_op_result(result, *self_val, other_val) .map(|val| ValOrChild::Val(val)) }, @@ -762,7 +771,7 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: results[idx] = result; } } else { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results); + pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results, swapped); //`other` holds no payload at this key, so the result has nothing here. That equals // `other` at this key only if `other` has no path along it at all. A path that shares a @@ -778,7 +787,11 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: PayloadRef::Child(self_link) => { match other_node.get_node_at_key(keys[idx].0).into_option() { Some(other_onward_node) => { - let result = self_link.as_tagged().pmeet_dyn(other_onward_node.as_tagged()); + let result = if swapped { + other_onward_node.as_tagged().pmeet_dyn(self_link.as_tagged()).invert_identity() + } else { + self_link.as_tagged().pmeet_dyn(other_onward_node.as_tagged()) + }; FatAlgebraicResult::from_binary_op_result(result, self_link, &other_onward_node) .map(|child| ValOrChild::Child(child)) }, @@ -795,7 +808,7 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: results[idx] = result; } } - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, keys.len(), self_payloads, keys, request_results, results); + pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, keys.len(), self_payloads, keys, request_results, results, swapped); is_exhaustive } @@ -803,7 +816,7 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: /// Effectively part of `pmeet_generic_internal`, but factored out separately because it's called in /// several different places. Resets the `cur_group` state and does a recursive call of `pmeet_generic_internal` #[inline] -fn pmeet_generic_recursive_reset<'trie, const MAX_PAYLOAD_CNT: usize, V, A: Allocator>(cur_group: &mut Option<(usize, &'trie TrieNodeODRc)>, is_exhaustive: &mut bool, idx: usize, self_payloads: &[(&[u8], PayloadRef)], keys: &mut [(&[u8], bool)], request_results: &mut [(usize, PayloadRef<'trie, V, A>)], results: &mut [FatAlgebraicResult>]) +fn pmeet_generic_recursive_reset<'trie, const MAX_PAYLOAD_CNT: usize, V, A: Allocator>(cur_group: &mut Option<(usize, &'trie TrieNodeODRc)>, is_exhaustive: &mut bool, idx: usize, self_payloads: &[(&[u8], PayloadRef)], keys: &mut [(&[u8], bool)], request_results: &mut [(usize, PayloadRef<'trie, V, A>)], results: &mut [FatAlgebraicResult>], swapped: bool) where V: Clone + Send + Sync + Lattice { match core::mem::take(cur_group) { @@ -811,7 +824,7 @@ fn pmeet_generic_recursive_reset<'trie, const MAX_PAYLOAD_CNT: usize, V, A: Allo let group_keys = &mut keys[group_start..idx]; let group_results = &mut results[group_start..idx]; let group_self_payloads = &self_payloads[group_start..idx]; - if !pmeet_generic_internal::(group_self_payloads, group_keys, request_results, group_results, next_node.as_tagged()) { + if !pmeet_generic_internal::(group_self_payloads, group_keys, request_results, group_results, next_node.as_tagged(), swapped) { *is_exhaustive = false; } }, @@ -3463,6 +3476,89 @@ mod tests { use crate::PathMap; use crate::zipper::*; + fn mk(ps: &[(&[u8], u64)]) -> PathMap { + let mut m = PathMap::::new(); + for (p, v) in ps { m.set_val_at(p, *v); } + m + } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { + m.iter().map(|(p, v)| (p.to_vec(), *v)).collect() + } + + /// `Lattice for u64` keeps `self` on a collision, so a meet carries the *left* operand's value. + /// That must not depend on which node type each operand happens to be stored in: a byte node + /// meeting a list node used to run the meet with the operands swapped and returned the list + /// node's values whichever side it was on. Found by lean/differential.py. + #[test] + fn meet_value_bias_is_left_regardless_of_node_layout() { + let two_payload_list = mk(&[(&[0], 0), (&[0, 0], 0), (&[3, 0], 1)]); + let single_line = mk(&[(&[0], 1)]); + let dense = mk(&[(&[0], 0), (&[1], 0), (&[2], 0), (&[3], 0), (&[4], 0)]); + for (a, b, expect) in [ + (&two_payload_list, &single_line, 0u64), + (&single_line, &two_payload_list, 1), + (&dense, &single_line, 0), + (&single_line, &dense, 1), + (&dense, &two_payload_list, 0), + (&two_payload_list, &dense, 0), + ] { + let mut out = PathMap::::new(); + { let mut wz = out.write_zipper(); wz.meet_2(&a.read_zipper(), &b.read_zipper()); } + assert_eq!(out.get_val_at(&[0]), Some(&expect), "meet_2 of {:?} and {:?}", vals(a), vals(b)); + + let mut into = a.clone(); + { let mut wz = into.write_zipper(); wz.meet_into(&b.read_zipper(), false); } + assert_eq!(into.get_val_at(&[0]), Some(&expect), "meet_into of {:?} and {:?}", vals(a), vals(b)); + } + } + + /// The same for joins: `PathMap::join` and `join_into` keep the left operand's value, whether + /// the left operand is a list node joining into a byte node or the other way round. + #[test] + fn join_value_bias_is_left_regardless_of_node_layout() { + let line = mk(&[(&[1], 0)]); + let dense = mk(&[(&[0], 0), (&[1], 1), (&[2], 0)]); + assert_eq!(line.join(&dense).get_val_at(&[1]), Some(&0)); + assert_eq!(dense.join(&line).get_val_at(&[1]), Some(&1)); + + let mut into = line.clone(); + { let mut wz = into.write_zipper(); wz.join_into(&dense.read_zipper()); } + assert_eq!(vals(&into), vec![(vec![0], 0), (vec![1], 0), (vec![2], 0)]); + let mut into = dense.clone(); + { let mut wz = into.write_zipper(); wz.join_into(&line.read_zipper()); } + assert_eq!(vals(&into), vec![(vec![0], 0), (vec![1], 1), (vec![2], 0)]); + + //A deeper collision, so the child-node join is exercised as well as the value join + let line = mk(&[(&[1, 5], 0), (&[1, 6], 0)]); + let dense = mk(&[(&[0], 0), (&[1, 5], 1), (&[2], 0)]); + assert_eq!(line.join(&dense).get_val_at(&[1, 5]), Some(&0)); + assert_eq!(dense.join(&line).get_val_at(&[1, 5]), Some(&1)); + } + + /// `join_k_path_into` joins the surviving subtries in path order (`PathMap.dropHead` in the + /// Lean model folds over the k-paths in sorted order), so on a collision the value from the + /// lexicographically first k-path survives. The byte node used to fold from the highest byte + /// down, and the list node's two-slot merge used to join with the second slot on the left. + #[test] + fn join_k_path_into_keeps_lexicographically_first_value() { + let mut m = mk(&[(&[0, 0, 0, 2], 0), (&[0, 1, 0, 2], 1), (&[0, 1, 0, 2, 0], 0)]); + { let mut wz = m.write_zipper(); wz.join_k_path_into(3, false); } + assert_eq!(vals(&m), vec![(vec![2], 0), (vec![2, 0], 0)]); + + let mut m = mk(&[(&[1, 0, 3], 0), (&[0], 0), (&[0, 0, 0], 0), (&[0, 0, 3], 1)]); + { let mut wz = m.write_zipper(); wz.join_k_path_into(2, false); } + assert_eq!(vals(&m), vec![(vec![0], 0), (vec![3], 1)]); + + let mut m = mk(&[(&[0, 0, 0], 0), (&[0], 0), (&[1, 0, 0], 1)]); + { let mut wz = m.write_zipper(); wz.join_k_path_into(2, false); } + assert_eq!(vals(&m), vec![(vec![0], 0)]); + + //Three-plus branches at the root make it a byte node + let mut m = mk(&[(&[0, 0, 7], 0), (&[1, 0, 7], 1), (&[2, 0, 7], 2), (&[3, 0, 7], 3)]); + { let mut wz = m.write_zipper(); wz.join_k_path_into(2, false); } + assert_eq!(vals(&m), vec![(vec![7], 0)]); + } + #[test] fn slim_ptrs_test1() { let map = PathMap::<()>::new(); From 23d5d12b03f6730591b1759036d172cf218da494 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Tue, 15 Sep 2026 23:49:11 +0000 Subject: [PATCH 08/73] Fuzz: separate the ACT ascend_until defect from finding 9 The whole remaining ACT residual -- 293 hits over 36000 inputs, and nothing else -- sat in the `ascend_until_wz` bucket, whose note says finding 9, a *write* zipper corrupted at a node boundary. It is not that. In all 30 sampled the write zipper's fields are identical on both sides and only the read zipper moved: the ACT zipper reports a shorter ascent than the model, one byte short in 18 of 22 and two in the rest. The entry keys on the bare substring `ascend_until`, so it claimed them. The giveaway was the asymmetry: 293 hits in ACT mode and 0 in crate mode, for a defect in a write zipper that both modes share. `read_zipper_only` tags a divergence whose write-zipper fields agree, in `--act` mode only, where the read zipper is the ArenaCompactTree one; the new entry is tested before the write-zipper one. The ACT residual moves to it in full, and `ascend_until_wz` now fires nowhere. Crate agreement is unchanged. That also settles archive-bugfix/ascend-until-write-zipper, which retired this entry and which I had held back for still appearing to fire: its claim was right, and what remained was this misattribution. Co-Authored-By: Claude Opus 5 (1M context) --- lean/differential.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lean/differential.py b/lean/differential.py index 47b593dd..ac20e208 100755 --- a/lean/differential.py +++ b/lean/differential.py @@ -215,6 +215,12 @@ def run(self, blob): (["join_into"], "join_into() drops the source subtrie when the destination map is empty " "[join_into_empty_dst]"), + # Tested before the write-zipper entry below, which keys on the bare op name + # and was claiming these: here the write zipper is identical on both sides + # and only the ACT read zipper moved, so finding 9 cannot be the cause. + (["ascend_until", "ACT-READ-ZIPPER-ONLY"], + "ACTZipper::ascend_until()/ascend_until_branch() report a different ascent " + "than the model, usually one byte short [act: ascend_until_short]"), (["ascend_until"], "ascend_until()/ascend_until_branch() corrupt a write zipper rooted at a " "node boundary [ascend_until_wz]"), @@ -391,6 +397,20 @@ def divergence_shape(a, b): return None +def read_zipper_only(a, b): + """Do these differ only in the read zipper, leaving the write zipper equal? + + In `--act` mode the read zipper is the ArenaCompactTree one and the write + zipper is still a `PathMap` one, so this separates an ACT read-side defect + from a write-side defect that the same operation would also report. + """ + if any(" W=" not in t or " R=" not in t for t in (a, b)): + return False + w = lambda t: t.split(" W=", 1)[1].split(" R=", 1)[0] + r = lambda t: t.split(" R=", 1)[1] + return w(a) == w(b) and r(a) != r(b) + + def act_valcount_only(a, b): """Do these two trace lines differ *only* in the read zipper's val_count? @@ -430,6 +450,8 @@ def compare(blob, oracle, other, other_label, act=False): for i, (a, b) in enumerate(zip(lean, real)): if a != b: tags = ["ACT-VALCOUNT-ONLY"] if act and act_valcount_only(a, b) else [] + if act and read_zipper_only(a, b): + tags.append("ACT-READ-ZIPPER-ONLY") shape = divergence_shape(a, b) if shape: tags.append(shape) From 315b41c81ec7288ab28eab869188b73120f00f8a Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 00:13:13 +0000 Subject: [PATCH 09/73] Fix ACTZipper ascend_until landing on the wrong stop above an off-trie focus `ascend_to_branch`, behind `ascend_until` and `ascend_until_branch`, first ascends the non-existent tail of the path (`ascend_invalid`) and then has to decide whether the real ancestor it lands on is already a stop. It read that off the *node* instead of off the focus, and only looked for a value: match &self.cur_node { Node::Line(line) => if need_value && line.value.is_some() { return } Node::Branch(node) => if need_value && node.value.is_some() { return } } A line node stores a run of bytes and carries its value at the *end* of that run, so `line.value` says nothing about a focus that sits partway into the line -- and a focus left mid-line is exactly what `ascend_invalid` produces when the path walked off the trie in the middle of a line. The ascent then stopped there and reported a value stop at a position that has no value, short of the real stop further up: paths: [1,2,3,4], [5] focus [1,2,9], one byte off the trie ascend_until() -> ACT 1 ([1,2], no value, one child), model 3 (the root) The other half is the case the match never considered at all: a branch is a stop too. When the deepest real ancestor is a branch node with more than one child, the model stops there, while the loop below fell straight through and popped the frame, running one byte past it: paths: [5,2], [5,6] focus [5,9] ascend_until_branch() -> ACT 2 (the root), model 1 ([5]) Both conditions now come off the focus, through `is_val` and `child_count`, which handle a mid-line focus correctly -- a line's interior has exactly one child and no value -- and match what the loop below applies to every other position it ascends past. This was the only remaining ACT-mode divergence class. ACT differential runs, maxlen 300, the harness's `act: ascend_until_short` entry untouched: seed 7, 5000 inputs: 4959/5000 agree, 41 known -> 5000/5000, 0 known seed 0, 4000 inputs: 3964/4000 agree, 36 known -> 4000/4000, 0 known seed 10, 4000 inputs: 3964/4000 agree, 36 known -> 4000/4000, 0 known seed 12, 4000 inputs: 3971/4000 agree, 29 known -> 4000/4000, 0 known seed 21, 20000 inputs (after only): 20000/20000 agree, 0 known, 0 new No new divergence class is unmasked, and crate mode is unchanged: 20000 inputs, seed 7, 19975/20000 agree, 25 known, 0 new, before and after. `cargo test --release` and `cargo test --release --features arena_compact,random` pass. Regression test: `act_zipper_ascend_until_from_an_off_trie_focus`, both operations at four roots against the `PathMap` read zipper, over off-trie foci below a value-terminated line, below a branch, and below a branching value. Fails before this change (1 vs 3 on the first case, 2 vs 1 on the branch one). Co-Authored-By: Claude Opus 5 (1M context) --- src/arena_compact.rs | 70 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/src/arena_compact.rs b/src/arena_compact.rs index 6212bd34..fe216598 100644 --- a/src/arena_compact.rs +++ b/src/arena_compact.rs @@ -2668,17 +2668,14 @@ where Storage: AsRef<[u8]> return start_len - self.path.len(); } - match &self.cur_node { - Node::Line(line) => { - if need_value && line.value.is_some() { - return start_len - self.path.len(); - } - } - Node::Branch(node) => { - if need_value && node.value.is_some() { - return start_len - self.path.len(); - } - } + //The focus is now the deepest real ancestor of where we started, which is a strict + //ancestor, so it is a candidate stop in its own right. It stops the ascent under + //exactly the conditions the rest of this method uses: a value here when `need_value`, + //or a branch. Both have to be read off the *focus*, not off the node: a line node + //carries its value at its end, so `line.value` says nothing about a focus that sits + //partway into the line, and a line's interior always has exactly one child. + if (need_value && self.is_val()) || self.child_count() > 1 { + return start_len - self.path.len(); } } while let Some(top_frame) = self.stack.last_mut() { @@ -4313,4 +4310,55 @@ mod tests { while az.to_next_step() { seen.push(az.path().to_vec()); } assert_eq!(seen, vec![vec![1u8], vec![1, 0], vec![1, 0, 2], vec![3]]); } + + /// `ascend_to_branch`, behind `ascend_until` and `ascend_until_branch`, ascends the + /// non-existent tail of the path first and then has to decide whether the real ancestor it + /// lands on already stops the ascent. It read that off the *node* rather than off the + /// focus: it took a line node's `value`, which sits at the line's end, for a value at a + /// focus partway into the line, and it never considered branching at all. So from an + /// off-trie focus the ascent stopped short of the first real value or branch whenever the + /// deepest real ancestor sat mid-line in a line that ends in a value, and ran a byte past + /// that ancestor whenever it was a branch. + #[test] + fn act_zipper_ascend_until_from_an_off_trie_focus() { + use crate::zipper::*; + let mut m = PathMap::::new(); + m.insert(&[1u8, 2, 3, 4], 11); //a line under 01, its value at the line's end + m.insert(&[5u8, 2], 22); //a branch at 05, two children, no value + m.insert(&[5u8, 6], 33); + m.insert(&[7u8], 44); //a value at 07, which branches below it as well + m.insert(&[7u8, 8], 55); + m.insert(&[7u8, 9], 66); + let t = ArenaCompactTree::from_zipper(m.read_zipper(), |&v| v); + + let off_trie: [&[u8]; 9] = [&[1, 2, 9], &[1, 2, 3, 9], &[1, 2, 3, 4, 9], &[1, 9, 9], + &[5, 9], &[5, 2, 9], &[7, 9, 9], &[7, 8, 9, 9], &[9]]; + for root in [&[][..], &[1u8], &[1, 2], &[7]] { + for focus in off_trie { + if !focus.starts_with(root) { continue } + let focus = &focus[root.len()..]; + for need_value in [false, true] { + let mut az = t.read_zipper_at_path_u64(root); + let mut pz = m.read_zipper_at_path(root); + assert_eq!(az.descend_to(focus), pz.descend_to(focus)); + assert!(!az.path_exists() && !pz.path_exists(), "{root:?} {focus:?}"); + let (a, p) = if need_value { + (az.ascend_until(), pz.ascend_until()) + } else { + (az.ascend_until_branch(), pz.ascend_until_branch()) + }; + assert_eq!(a, p, "root {root:?} focus {focus:?} need_value {need_value}"); + assert_eq!(az.path(), pz.path(), "root {root:?} focus {focus:?}"); + //Whether the frame still agrees with the path the ascent left the zipper + //at only shows on the next move, so look at the focus and then move + assert_eq!(az.path_exists(), pz.path_exists(), "{root:?} {focus:?}"); + assert_eq!(az.val(), pz.val(), "{root:?} {focus:?}"); + assert_eq!(az.child_count(), pz.child_count(), "{root:?} {focus:?}"); + assert_eq!(az.descend_first_byte(), pz.descend_first_byte(), "{root:?} {focus:?}"); + assert_eq!(az.path(), pz.path(), "{root:?} {focus:?} after descend"); + assert_eq!(az.val(), pz.val(), "{root:?} {focus:?} after descend"); + } + } + } + } } From 2d3b6f2dd88cfdbf2dca0a8f6be3034e88a1146d Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 00:07:22 +0000 Subject: [PATCH 10/73] Report Identity from the dense restrict when nothing was dropped Finding 8 ("AlgebraicStatus::Identity is not returned when nothing changed") still fired on `restrict` for 8 of 20k seed-7 inputs. Two defects in the dense node's restrict, both in `src/dense_byte_node.rs`: - `ByteNode::prestrict` seeded its identity flag with `self.mask == mm && other.mask == mm` (line 2488, `mm = self.mask & other.mask`). Restrict is non-commutative: only `self`'s branches can be dropped, so the result is an identity of `self` exactly when `self.mask == mm`. Requiring `other.mask == mm` as well meant that any source with a branch the destination did not have forced `Element`, even though the destination was returned untouched. The flag is now seeded with `self.mask == mm` alone. - `prestrict_abstract` (the dense-against-list/tiny path, line 580) walks `self`'s entries. When `other` has no value at a byte, `self`'s value at that byte is dropped -- the empty path does not validate -- but the identity flag was only cleared by the onward link's own result. A co-free holding both a value and an onward link that restricted to an identity therefore reported `Identity`, and the caller kept `self` unchanged, so the value that should have been dropped stayed in the map. This one is a wrong answer, not just an imprecise status. The flag is now also cleared when the dropped co-free carries a value. Differential, before -> after, 0 new divergences throughout and no other class moved (`status_imprecise` and `meet_keeps_dangling` counts are unchanged): seed 7, 20k: 19975 -> 19982 agree; finding-8 restrict key 8 -> 1 seed 0, 10k: 9987 -> 9992 agree; finding-8 restrict key 6 -> 1 seed 10, 10k: 9989 -> 9992 agree; finding-8 restrict key 4 -> 1 seed 12, 10k: 9988 -> 9993 agree; finding-8 restrict key 5 -> 0 ACT, seed 7, 5k: 4959 -> 4959 agree, identical bucket table The residual restrict hits are a different defect: `restrict` against a destination that holds an empty child node materialised by an earlier `meet_into` keeps the dangling branch the spec drops -- the `meet_keeps_dangling` family, not an identity-mask problem. cargo test --release: 909 passed. cargo test --release --features arena_compact,random: 1044 passed. Tests: write_zipper_restrict_wider_source_is_identity, write_zipper_restrict_drops_value_beside_kept_child. Co-Authored-By: Claude Opus 5 (1M context) --- src/dense_byte_node.rs | 14 +++++++++++- src/write_zipper.rs | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index ac7b6064..858e0658 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -592,6 +592,14 @@ impl> ByteNode new_node.values.push(cf.clone()); } else { + //Without a value in `other`, the empty path doesn't validate, so a value of + // `self`'s at this byte is dropped. That is a modification, even when the + // onward link below it restricts to an identity, and it is also the whole of + // the change when there is no onward link at all. + if cf.val().is_some() { + is_identity = false; + } + //If there is an onward link in the CF and other node, continue the restriction recursively if let Some(self_child) = cf.rec() { let other_child = other.get_node_at_key(&[key_byte]); @@ -2478,7 +2486,11 @@ impl> ByteNode // Iterate the overlap mask directly. Slot indexes are recovered with // prefix popcounts in each dense-mask word. let mut mm: ByteMask = self.mask & other.mask; - let mut is_identity = self.mask == mm && other.mask == mm; + //NOTE: restrict is non-commutative. The result is an identity of `self` when every one of + // `self`'s branches survives, i.e. `self.mask == mm`. Branches that exist only in `other` + // never contribute to the result, so requiring `other.mask == mm` as well only threw away + // identities that were there. + let mut is_identity = self.mask == mm; let mmc = [mm.0[0].count_ones(), mm.0[1].count_ones(), mm.0[2].count_ones(), mm.0[3].count_ones()]; diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 7e0f2644..c1492d0b 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -7051,4 +7051,55 @@ mod tests { assert_eq!(st, AlgebraicStatus::Element); assert_eq!(vals(&dst), vals(&src)); } + + /// `restrict` between two dense nodes, where the source has branches the destination lacks. + /// `restrict` is non-commutative: only the destination's branches can be dropped, so branches + /// that exist only in the source say nothing about whether the destination changed. The dense + /// restrict nevertheless required the two masks to be equal before it would report `Identity`. + #[test] + fn write_zipper_restrict_wider_source_is_identity() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + //Both roots are dense. Every path in `dst` is prefixed by a path to a value in `src`, so + // the restriction keeps all of `dst`; `src`'s extra branches are irrelevant. + let mut dst = mk(&[(&[0], 1), (&[1], 2), (&[2], 3)]); + let before = vals(&dst); + let src = mk(&[(&[0], 0), (&[1], 0), (&[2], 0), (&[3], 0), (&[4], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //A restriction that really does drop a branch still reports it + let mut dst = mk(&[(&[0], 1), (&[1], 2), (&[2], 3)]); + let src = mk(&[(&[0], 0), (&[1], 0), (&[3], 0), (&[4], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vec![(vec![0], 1), (vec![1], 2)]); + } + + /// `restrict` of a dense node against a node that can't be iterated as a dense one (a list or + /// a tiny node) walks the destination's entries. An entry holding *both* a value and an + /// onward link, where the source has no value at that byte, loses its value -- the empty path + /// never validates -- but the identity flag was only cleared by the onward link's own result, + /// so an onward link that restricted to an identity made the whole node report `Identity` and + /// the dropped value stayed in the map. + #[test] + fn write_zipper_restrict_drops_value_beside_kept_child() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + //Build a dense root, then meet it down to the single byte-0 branch, which keeps the dense + // node type. That branch holds a value (at `[0]`) beside an onward link (to `[0, 0]`). + let mut dst = mk(&[(&[0], 7), (&[0, 0], 1), (&[1], 2), (&[2], 3)]); + let filter = mk(&[(&[0], 0), (&[0, 0], 0), (&[5], 0), (&[6], 0)]); + { let mut wz = dst.write_zipper(); wz.meet_into(&filter.read_zipper(), false); } + assert_eq!(vals(&dst), vec![(vec![0], 7), (vec![0, 0], 1)]); + + //`src` has no value at `[0]`, so `[0]` is not kept, while `[0, 0]` is + let src = mk(&[(&[0, 0], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(vals(&dst), vec![(vec![0, 0], 1)]); + assert_eq!(st, AlgebraicStatus::Element); + } } From f8a4599716a3de0c3b0742360001f01b67dc47c1 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 00:12:43 +0000 Subject: [PATCH 11/73] Return Identity from the integer psubtract when nothing was subtracted Cherry-pick of a subagent's 24ccaaf, plus the one model line it has to land with. `impl DistributiveLattice for u64` and `for u16` in src/ring.rs answered a subtraction of unequal values with `Element(*self)` -- the right value under the wrong constructor. The node algebra propagates identity *masks*, not values, so a single `Element` below a node forced the whole node, and with it `subtract_into`, to report `Element` for a byte- identical trie. `impl DistributiveLattice for bool` in the same file already returned `Identity(SELF_IDENT)`; the integer instances were the outliers. `u64Ops` in lean/PathMapModel/Basic.lean is a transcription of that instance, not an independent claim -- its own docstring says it "reproduces it exactly rather than assuming a real lattice", and SPEC_WARTS.md records it as "copied from `impl Lattice for u64`". So it follows the crate here: `psub` returns `.identity true false`. The wart SPEC_WARTS.md actually flags is that `pjoin`/`pmeet` are left-biased projections, which is untouched. Changing the crate without this line leaves the model asserting the behaviour of a version that no longer exists, and the two must move together. Verified, with the KNOWN table unmodified throughout: crate seed 0, 10000 9992 -> 10000/10000, 0 known crate seed 7, 10000 9992 -> 9999/10000, 1 known crate seed 10, 10000 9992 -> 9999/10000, 1 known crate seed 12, 10000 9993 -> 10000/10000, 0 known ACT seed 0 and 7, 5000 5000/5000, 0 known 0 new divergences in every run. The one hit left at seeds 7 and 10 is a different defect: `restrict` against a destination holding an empty node materialised by an earlier `meet_into` keeps a dangling branch the spec drops. That belongs to the empty-node-materialisation family, not to this identity-mask class. Tests: 911 and 1047 pass, 0 failed. Co-Authored-By: Claude Opus 5 (1M context) --- lean/PathMapModel/Basic.lean | 12 +++++++----- src/ring.rs | 20 ++++++++++++++++++-- src/write_zipper.rs | 25 +++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/lean/PathMapModel/Basic.lean b/lean/PathMapModel/Basic.lean index ff22667f..d3264a83 100644 --- a/lean/PathMapModel/Basic.lean +++ b/lean/PathMapModel/Basic.lean @@ -40,8 +40,8 @@ inductive ValRes (V : Type) where These return `ValRes` rather than plain values because `pathmap` reports `AlgebraicStatus` to the caller, and the status depends on *which constructor* the value operation returned, not on whether the value changed. `u64`'s -`psubtract`, for instance, returns `Element(*self)` — an `Element` status even -though the stored value is unchanged. -/ +`pjoin`, for instance, returns `Identity(SELF_IDENT)` rather than `Element` of +the value it selected, so a join reports that nothing changed. -/ structure ValOps (V : Type) where /-- `Lattice::pjoin` -/ pjoin : V → V → ValRes V @@ -63,12 +63,14 @@ def ValRes.resolve {V : Type} : ValRes V → V → V → Option V Both `pjoin` and `pmeet` return `Identity(SELF_IDENT)`: they are *left-biased projections* that ignore the counterpart value entirely. `psubtract` annihilates only when the two values are equal, and otherwise returns -`Element(*self)`. This is the instance the differential fuzz target uses, so -the model reproduces it exactly rather than assuming a "real" lattice. -/ +`Identity(SELF_IDENT)`: subtracting a value that is not there leaves the +destination alone, and says so. This is the instance the differential fuzz +target uses, so the model reproduces it exactly rather than assuming a "real" +lattice. -/ def u64Ops : ValOps UInt64 where pjoin _ _ := .identity true false pmeet _ _ := .identity true false - psub a b := if a == b then .none else .elem a + psub a b := if a == b then .none else .identity true false beq a b := a == b /-! ## Prefix order -/ diff --git a/src/ring.rs b/src/ring.rs index 2c781591..cf94e210 100644 --- a/src/ring.rs +++ b/src/ring.rs @@ -754,6 +754,22 @@ fn option_subtract_test() { assert_eq!(Some(Some(Some(()))).psubtract(&Some(Some(Some(())))), AlgebraicResult::None); } +/// Subtracting a value that isn't there leaves the destination alone, and the integer placeholders +/// have to say so with `Identity(SELF_IDENT)`. Returning `Element(*self)` is the same value, but +/// the node algebra propagates identity *masks*, not values, so an `Element` anywhere below a node +/// forces the whole node -- and with it `subtract_into` -- to report `Element` for a trie that did +/// not change. +#[test] +fn integer_subtract_is_self_identity() { + assert_eq!(3u64.psubtract(&5), AlgebraicResult::Identity(SELF_IDENT)); + assert_eq!(3u64.psubtract(&3), AlgebraicResult::None); + assert_eq!(3u16.psubtract(&5), AlgebraicResult::Identity(SELF_IDENT)); + assert_eq!(3u16.psubtract(&3), AlgebraicResult::None); + //The same, seen through `Option`, which is what the co-free node payloads use + assert_eq!(Some(3u64).psubtract(&Some(5)), AlgebraicResult::Identity(SELF_IDENT)); + assert_eq!(Some(3u64).psubtract(&Some(3)), AlgebraicResult::None); +} + // =-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-==-**-= // =-* `Option<&V>` *-= @@ -876,7 +892,7 @@ impl Lattice for u64 { impl DistributiveLattice for u64 { fn psubtract(&self, other: &Self) -> AlgebraicResult where Self: Sized { if self == other { AlgebraicResult::None } - else { AlgebraicResult::Element(*self) } + else { AlgebraicResult::Identity(SELF_IDENT) } } } @@ -896,7 +912,7 @@ impl Lattice for u16 { impl DistributiveLattice for u16 { fn psubtract(&self, other: &Self) -> AlgebraicResult { if self == other { AlgebraicResult::None } - else { AlgebraicResult::Element(*self) } + else { AlgebraicResult::Identity(SELF_IDENT) } } } diff --git a/src/write_zipper.rs b/src/write_zipper.rs index c1492d0b..d876eecc 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -3960,6 +3960,31 @@ mod tests { assert_eq!(remaining, vec![(vec![0], 0), (vec![0, 0, 0], 0)]); } + /// `subtract_into` where every value of the source collides with a *different* value in the + /// destination. Nothing annihilates, so the destination comes back untouched and the status + /// has to be `Identity`. The integer `psubtract` used to answer `Element(*self)`, and the node + /// algebra -- which propagates identity masks, not values -- turned that into `Element` for the + /// whole trie. + #[test] + fn write_zipper_subtract_into_unequal_values_is_identity() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + let mut dst = mk(&[(&[0], 1), (&[0, 0], 2), (&[1], 3), (&[2], 4)]); + let before = vals(&dst); + let src = mk(&[(&[0], 9), (&[0, 0], 9), (&[1], 9), (&[2], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), before); + + //An equal value still annihilates, and that is an `Element`, not an identity + let mut dst = mk(&[(&[0], 1), (&[0, 0], 2), (&[1], 3), (&[2], 4)]); + let src = mk(&[(&[0], 9), (&[0, 0], 2), (&[1], 9), (&[2], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vec![(vec![0], 1), (vec![1], 3), (vec![2], 4)]); + } + /// Tests how `subtract_into` handles dangling paths, including situations with extraneous empty nodes hanging around #[test] fn write_zipper_subtract_into_test2() { From 3a2b499bff347ce6ef2cb4f99e034ec7dc4182a4 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 00:25:08 +0000 Subject: [PATCH 12/73] Answer a sibling step from a focus that is not in the trie A zipper's focus is allowed to sit on a path that does not exist: `descend_to` an absent path puts it there and `path_exists()` reports false, and the API is specified to keep answering from there. The siblings of such a focus are still well defined, because they come from the parent's `child_mask`, not from the focus itself. The `ZipperMoving` default `to_prev_sibling_byte` computed that correctly and then asserted something that is not true of it. On the arm where `prev_bit` finds no previous sibling, the method puts the focus back exactly where it started and returns `None` -- and then asserted `self.path_exists()` on the restored focus (src/zipper.rs:457). Restoring a focus cannot make it exist, so for every off-trie focus that assert fired. Every write zipper takes this default impl (`WriteZipperCore` defines no sibling methods of its own), so in a debug build `WriteZipper::to_prev_sibling_byte` panicked instead of returning `None`. The minimal case is an empty map: let mut wz = map.write_zipper(); wz.descend_to(&[0u8]); // off-trie; the root's child mask is empty wz.to_prev_sibling_byte(); // panicked; must be None The answer itself was always right. `toPrevSiblingByte` in the Lean model returns `(none, z)` in exactly this situation, which is what the code produces, and the release build -- where the assert is compiled out -- already agreed with the model on all of these inputs. So the defect is the assertion's invariant, not the computation: the only thing that holds on the restore arm is that the focus is put back unchanged, including whether it exists. That is what is asserted now, against the existence recorded before the ascent. `to_next_sibling_byte` never panicked, because its matching arm carried no assert at all; its answer was already correct. It gets the same check for symmetry, and it holds. The `Some` arms of both keep the original `path_exists()` assert, which is sound: that byte came out of the parent's `child_mask`, so the sibling landed on does exist. Measured with lean/differential.py, 4000 random inputs per seed, DEBUG build. The class is `[prev_sibling_missing]`; the KNOWN table is untouched. seed before after 7 3246/4000, 238 prev_sibling 3460/4000, 0 10 3202/4000, 248 prev_sibling 3432/4000, 0 12 3191/4000, 219 prev_sibling 3394/4000, 0 `new divergences` stays 0 on all three. Comparing per-input outcomes, every input whose classification changed was previously `prev_sibling_missing`; nothing that agreed before stopped agreeing. Most of them now agree outright (214/230/203), and the remainder (24/17/15) run past the panic and reach `[remove_unmasked_dangling]`, plus one input each on seeds 10 and 12 that reaches `[status_imprecise]`. Those are pre-existing defects the panic was hiding, not new ones. RELEASE is unchanged, as expected for an assert-only defect: 20000 inputs, seed 7, 19975/20000 agree, 25 known, 0 new, before and after. cargo test --release, cargo test --release --features arena_compact,random and cargo test (debug) all pass. Co-Authored-By: Claude Opus 5 (1M context) --- src/zipper.rs | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/src/zipper.rs b/src/zipper.rs index 57a70141..adee0fad 100644 --- a/src/zipper.rs +++ b/src/zipper.rs @@ -416,18 +416,29 @@ pub trait ZipperMoving: Zipper { /// where the index passed is 1 more than the index of the current focus position. fn to_next_sibling_byte(&mut self) -> Option { let cur_byte = self.focus_byte()?; + //The focus is allowed to sit on a path that isn't in the trie, and such a focus still has + // well-defined siblings because the siblings come from the parent's `child_mask`. Remember + // whether the focus exists, so the no-sibling arm can check it restores exactly what it found. + #[cfg(debug_assertions)] + let focus_existed = self.path_exists(); if !self.ascend_byte() { return None } let mask = self.child_mask(); match mask.next_bit(cur_byte) { Some(byte) => { + //`byte` came out of the parent's `child_mask`, so the sibling we land on exists self.descend_to_byte(byte); debug_assert!(self.path_exists()); Some(byte) }, None => { + //There is no next sibling, so the focus goes back exactly where it started. That + // location need not be an existing path, so all we can check is that putting it + // back didn't change whether it exists. self.descend_to_byte(cur_byte); + #[cfg(debug_assertions)] + debug_assert_eq!(self.path_exists(), focus_existed); None } } @@ -442,19 +453,27 @@ pub trait ZipperMoving: Zipper { /// where the index passed is 1 less than the index of the current focus position. fn to_prev_sibling_byte(&mut self) -> Option { let cur_byte = self.focus_byte()?; + //See the note in `to_next_sibling_byte`; the focus may legitimately be off the trie + #[cfg(debug_assertions)] + let focus_existed = self.path_exists(); if !self.ascend_byte() { return None } let mask = self.child_mask(); match mask.prev_bit(cur_byte) { Some(byte) => { + //`byte` came out of the parent's `child_mask`, so the sibling we land on exists self.descend_to_byte(byte); debug_assert!(self.path_exists()); Some(byte) }, None => { + //There is no previous sibling, so the focus goes back exactly where it started. + // That location need not be an existing path, so all we can check is that putting + // it back didn't change whether it exists. self.descend_to_byte(cur_byte); - debug_assert!(self.path_exists()); + #[cfg(debug_assertions)] + debug_assert_eq!(self.path_exists(), focus_existed); None } } @@ -6401,6 +6420,76 @@ mod tests { assert_eq!(z.to_prev_sibling_byte(), None); } + /// A sibling step from a focus that is not in the trie. `descend_to` an absent path is + /// legal and leaves `path_exists()` false, and the siblings of that focus are still well + /// defined because they come from the parent's `child_mask`. The `ZipperMoving` default + /// `to_prev_sibling_byte` asserted `path_exists()` on the arm that finds no previous + /// sibling and puts the focus back where it started, so every write zipper (which takes the + /// default impl) panicked in a debug build instead of returning `None`. `to_next_sibling_byte` + /// had no assert on its matching arm and so answered correctly; both are checked here. + #[test] + fn sibling_step_from_a_focus_that_does_not_exist() { + //The shrunk fuzz case: an empty map, so the root's child mask is empty + let mut empty = PathMap::::new(); + let mut wz = empty.write_zipper(); + wz.descend_to(&[0u8]); + assert!(!wz.path_exists()); + assert_eq!(wz.to_prev_sibling_byte(), None); + assert_eq!(wz.path(), &[0u8]); + assert!(!wz.path_exists()); + assert_eq!(wz.to_next_sibling_byte(), None); + assert_eq!(wz.path(), &[0u8]); + drop(wz); + + let mut map = PathMap::::new(); + map.insert(&[1u8, 3], 13); + map.insert(&[1u8, 5], 15); + map.insert(&[7u8], 7); + + //A focus that is missing from an existing parent answers from the parent's children: + // `None` when nothing lies to that side, and the real neighbour when something does + for (byte, prev, next) in [(2u8, None, Some(3u8)), (4, Some(3), Some(5)), (6, Some(5), None)] { + let mut wz = map.write_zipper_at_path(&[1u8]); + wz.descend_to(&[byte]); + assert!(!wz.path_exists(), "byte {byte}"); + assert_eq!(wz.to_prev_sibling_byte(), prev, "byte {byte}"); + //A step that moved landed on a real sibling; one that didn't left the focus off-trie + assert_eq!(wz.path_exists(), prev.is_some(), "byte {byte}"); + assert_eq!(wz.path(), &[prev.unwrap_or(byte)], "byte {byte}"); + drop(wz); + + let mut wz = map.write_zipper_at_path(&[1u8]); + wz.descend_to(&[byte]); + assert_eq!(wz.to_next_sibling_byte(), next, "byte {byte}"); + drop(wz); + + //The read zipper's native impls must agree with the write zipper's default impls + let mut rz = map.read_zipper_at_path(&[1u8]); + rz.descend_to(&[byte]); + assert!(!rz.path_exists(), "byte {byte}"); + assert_eq!(rz.to_prev_sibling_byte(), prev, "byte {byte}"); + let mut rz = map.read_zipper_at_path(&[1u8]); + rz.descend_to(&[byte]); + assert_eq!(rz.to_next_sibling_byte(), next, "byte {byte}"); + } + + //A focus whose parent is itself missing has no siblings at all, and the failed step + // must leave the focus exactly where it was + let mut wz = map.write_zipper(); + wz.descend_to(&[9u8, 9]); + assert!(!wz.path_exists()); + assert_eq!(wz.to_prev_sibling_byte(), None); + assert_eq!(wz.path(), &[9u8, 9]); + assert_eq!(wz.to_next_sibling_byte(), None); + assert_eq!(wz.path(), &[9u8, 9]); + assert!(!wz.path_exists()); + //...and the zipper is still usable afterwards + wz.ascend(2); + assert_eq!(wz.to_next_sibling_byte(), None); + wz.descend_to(&[7u8]); + assert_eq!(wz.val(), Some(&7)); + } + /// Tests iteration behavior of to_next_val implementations, comparing the default impl /// against the native imple, and a third run that interleaves calls to each #[test] From ecd7e21a5ee5cca667764922c7cacfd55b263704 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 00:26:38 +0000 Subject: [PATCH 13/73] Don't filter branches through a dangling path in remove_unmasked_branches `WriteZipperCore::remove_unmasked_branches` (src/write_zipper.rs:2315) descends through an onward link by calling `node_get_child_mut` on the focus node, and falls back to filtering the focus node's own branches when that returns `None`. But a dangling path is stored as an onward link to the empty-node sentinel, and `get_child_mut` deliberately declines to hand out an empty child (src/line_list_node.rs:495 and :506; the same convention is documented at src/write_zipper.rs:1603 for `graft_masked_branches`). So a focus sitting at -- or below -- a dangling stub never descended, and the stub's own key was handed to the focus node as if it were one of that node's branches. That breaks the invariant `LineListNode::node_remove_unmasked_branches` relies on, "the calling code should have descended through this node if that key specifies an onward link", and its `debug_assert!(!self.is_child_ptr::())` at src/line_list_node.rs:1964 and :1971 fired. In release the assertions compile away and `remove_subtries(false, false, ..)` happens to be the right answer, so this only showed up in debug builds. Fix: recognise the dangling stub at the zipper before the descent and do nothing. Nothing lives below a stub, so there are no branches to filter, and the dangling path itself must survive -- which is what `removeUnmaskedBranches` in lean/PathMapModel/Write.lean specifies, since `z.childMask` at a dangling focus is empty. The node-level assertions are left exactly as they were. Instrumenting the call site over 1500 seed-7 inputs shows the new guard taking 333 hits and `node_get_child_mut` always consuming the whole `node_key`, so the single-level check covers every case the fuzzer reaches. Measurements, `lean/differential.py --random 4000 --maxlen 300`, DEBUG: seed agree before -> after [remove_unmasked_dangling] new divergences 7 3246 -> 3734 513 -> 0 0 -> 0 10 3202 -> 3720 548 -> 0 0 -> 0 12 3191 -> 3752 588 -> 0 0 -> 0 The class is gone. It was masking two pre-existing classes, which grow because those inputs now run past the abort instead of stopping at it: [prev_sibling_missing] 239/249/220 -> 263/277/244 and [status_imprecise] 3-4 -> 3-4 (+1 on seed 12). No new class appeared and no input newly FAILed. RELEASE, --random 20000 --seed 7: 19975/20000 agree, 25 known, 0 new -- unchanged from baseline, as expected. `cargo test`, `cargo test --release` and `cargo test --release --features arena_compact,random` all pass. Co-Authored-By: Claude Opus 5 (1M context) --- src/write_zipper.rs | 60 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/write_zipper.rs b/src/write_zipper.rs index d876eecc..f345c621 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -2326,9 +2326,26 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } /// See [WriteZipper::remove_unmasked_branches] pub fn remove_unmasked_branches(&mut self, mask: ByteMask, prune: bool) { - let mut focus_node = self.focus_stack.top_mut().unwrap(); let node_key = self.key.node_key(); - if node_key.len() > 0 { + + //`get_child_mut` declines to hand out an empty child node, so a focus that sits at or + // below such a dangling stub never descends and lands in the `None` arm below, where the + // node would mistake the stub's own key for one of its branches. (`LineListNode` asserts + // on exactly that.) A stub has nothing below it, so there are no branches to filter, and + // the dangling path itself must survive. + let below_dangling_stub = node_key.len() > 0 + && match self.focus_stack.top() { + Some(focus_node) => match focus_node.node_get_child(node_key) { + Some((_consumed_bytes, child_node)) => child_node.is_empty(), + None => false + }, + None => false + }; + + let mut focus_node = self.focus_stack.top_mut().unwrap(); + if below_dangling_stub { + //Nothing to do + } else if node_key.len() > 0 { match focus_node.node_get_child_mut(node_key) { Some((consumed_bytes, child_node)) => { if node_key.len() >= consumed_bytes { @@ -5006,6 +5023,45 @@ mod tests { wz.remove_unmasked_branches(ByteMask::EMPTY, false); } + /// `remove_unmasked_branches` with the focus on a dangling path. A dangling path has no + /// branches below it, so the call must do nothing at all: the dangling path itself survives + /// (`prune` is `false`) and the rest of the trie is untouched. + /// + /// A dangling path is stored as an onward link to the empty-node sentinel, which + /// `get_child_mut` declines to hand out, so the zipper used to fail to descend through it and + /// hand the stub's own key to the focus node as if it were one of the node's branches. In a + /// `LineListNode` that tripped a `debug_assert!(!self.is_child_ptr::())`. + #[test] + fn write_zipper_test_remove_unmasked_branches_dangling_focus() { + //A LineListNode root: slot 0 is an onward link at [0], slot 1 the dangling stub at [1, 0] + let mut map = PathMap::::new(); + map.set_val_at([0u8], 0); + map.set_val_at([0u8, 0], 1); + assert!(map.create_path([1u8, 0])); + + //Focus exactly on the dangling path + let mut wz = map.write_zipper_at_path(&[1u8, 0]); + assert!(wz.path_exists()); + wz.remove_unmasked_branches(ByteMask::EMPTY, false); + assert!(wz.path_exists()); + drop(wz); + + //Focus below the dangling path + let mut wz = map.write_zipper_at_path(&[1u8, 0, 7]); + wz.remove_unmasked_branches(ByteMask::EMPTY, false); + drop(wz); + + //Nothing may have changed + assert_eq!(map.val_at([0u8]), Some(&0)); + assert_eq!(map.val_at([0u8, 0]), Some(&1)); + assert_eq!(map.val_at([1u8, 0]), None); + let mut rz = map.read_zipper(); + rz.descend_to([1u8, 0]); + assert!(rz.path_exists()); + drop(rz); + assert_eq!(map.val_count(), 2); + } + #[test] fn write_zipper_test_zipper_conversion() { let keys = [ From e6ced6a37ae0a0e0b01838351c33901c231311d3 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 01:14:36 +0000 Subject: [PATCH 14/73] Drop an unvalidated dangling branch from the dense restrict `ZipperWriting::restrict` keeps a path of the destination only when the source carries a value at some non-empty prefix of it (`validatedBy` / `restrictBelowRoot` in lean/PathMapModel/PathMap.lean). That applies to a dangling path -- one that exists but leads to no value -- exactly as it does to a valued one: an unvalidated dangling path is dropped. `ByteNode::prestrict_abstract` (src/dense_byte_node.rs:580, the dense-against- list/tiny/bridge path) walks `self`'s entries and rebuilds the node. For a byte where `other` has a path but no value, it kept whatever the onward link restricted to and cleared `is_identity` for the value it had to drop. A co-free with *neither* a value nor an onward link -- the dense representation of a dangling path, which `meet_into` materialises when it empties a branch -- fell through both of those: it was correctly left out of the rebuilt node, but nothing cleared `is_identity`. When every other byte restricted to an identity the node therefore returned `Identity(SELF_IDENT)`, and `WriteZipper::restrict` (src/write_zipper.rs:2230) responded by keeping `self` untouched -- with the dangling branch still in it. So this is a wrong answer, not only an imprecise status: both the kept path and the reported `Identity` were wrong, and the status follows from the content. The fix is the missing `else` arm: no onward link and no value means nothing of that byte reaches the result, which is a modification. The sibling paths were already right -- `Cf::prestrict` sends a dangling co-free to `None` via its `_` arm, `EmptyNode::prestrict_dyn` returns `None`, and `restrict_slot_contents` reaches that through `prestrict_dyn` -- so the dense/abstract walk was the only one on the wrong side of the asymmetry. A dangling path that *is* validated is still kept, as the model keeps it. Differential (release, --maxlen 300 --max-fails 0), before -> after, with 0 new divergences throughout: seed 7, 10000: 9999 -> 10000 agree (1 known -> 0) seed 10, 10000: 9999 -> 10000 agree (1 known -> 0) seed 0, 10000: 10000 -> 10000 seed 3, 10000: 10000 -> 10000 seed 11, 10000: 10000 -> 10000 seed 12, 10000: 10000 -> 10000 seed 99, 10000: 10000 -> 10000 ACT seed 0/7/12, 5000 each: 5000/5000 -> 5000/5000 Debug build (assertions on, 4000 inputs each): seed 7: 4000 -> 4000 seed 12: 4000 -> 4000 seed 10: 3999 -> 4000 (the same bug, hit once) The `[status_imprecise]` residual on seeds 1 and 2 did not move: seed 1 still has 1 hit and seed 2 still has 2, all `subtract_into`/`meet_into` status-only with every state field equal. lean/corpus/*.bin still agree 2/2. Tests: `write_zipper_restrict_drops_dangling_branch` covers the dense node with an unvalidated dangling co-free, the validated dangling co-free that must stay, and the shrunk differential reproducer spelled out as a program. It fails on the parent commit with `left: Identity, right: Element`. cargo test --release: 914 + 5 + 10 passed. cargo test --release --features arena_compact,random: 1050 + 5 + 19 passed. cargo test: 914 + 5 + 10 passed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 8 ++++++ src/write_zipper.rs | 63 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 858e0658..b9d24628 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -626,6 +626,14 @@ impl> ByteNode is_identity = false; } } + } else { + //No onward link either, so nothing of this byte reaches the result. A + // co-free with neither a value nor a child is the dense representation of a + // dangling path, and an unvalidated dangling path is dropped just like an + // unvalidated value -- so this branch is a modification too. Leaving the + // flag set reported `Identity`, and the caller then kept `self` with the + // dangling path still in it. + is_identity = false; } } } else { diff --git a/src/write_zipper.rs b/src/write_zipper.rs index f345c621..545c926b 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -7183,4 +7183,67 @@ mod tests { assert_eq!(vals(&dst), vec![(vec![0, 0], 1)]); assert_eq!(st, AlgebraicStatus::Element); } + + /// A dangling path -- one that exists but leads to no value -- is kept by `restrict` only when + /// it is validated, i.e. when the source carries a value at some non-empty prefix of it. In a + /// dense node a dangling path is a co-free with neither a value nor an onward link; the + /// destination-walking `prestrict_abstract` dropped such a co-free from the result but never + /// cleared its identity flag, so the node reported `Identity` and the caller kept `self` -- + /// dangling path included. That is a wrong answer, not just an imprecise status. + #[test] + fn write_zipper_restrict_drops_dangling_branch() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + //Build a dense root and meet it down to bytes 0 and 1, which keeps the dense node type, + // then strip the value at [0] without pruning so that [0] is left dangling. + let mut dst = mk(&[(&[0], 1), (&[1], 2), (&[2], 3), (&[3], 4)]); + let filter = mk(&[(&[0], 0), (&[1], 0)]); + { let mut wz = dst.write_zipper(); wz.meet_into(&filter.read_zipper(), false); } + dst.remove_val_at(&[0u8], false); + assert_eq!(dst.path_exists_at(&[0u8]), true, "[0] should be left dangling"); + assert_eq!(vals(&dst), vec![(vec![1], 2)]); + + //`src` is a list node: it has a path through byte 0 but no value at [0], so the dangling + // [0] is not validated and must go, while [1] carries a value and is kept. Every byte of + // `dst` is present in `src`, so nothing else can clear the identity flag. + let src = mk(&[(&[0, 9], 0), (&[1], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(dst.path_exists_at(&[0u8]), false); + assert_eq!(vals(&dst), vec![(vec![1], 2)]); + + //A dangling path that *is* validated stays: `src` has a value at [0]. + let mut dst = mk(&[(&[0], 1), (&[1], 2), (&[2], 3), (&[3], 4)]); + { let mut wz = dst.write_zipper(); wz.meet_into(&filter.read_zipper(), false); } + dst.remove_val_at(&[0u8], false); + let src = mk(&[(&[0], 0), (&[1], 0)]); + let st = { let mut wz = dst.write_zipper(); wz.restrict(&src.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(dst.path_exists_at(&[0u8]), true); + assert_eq!(vals(&dst), vec![(vec![1], 2)]); + + //The shrunk differential reproducer, spelled out: `meet_into` against a source whose focus + // is a leaf leaves an empty child node at [0], and the following `restrict` kept it. + let mut map0 = PathMap::::new(); + map0.set_val_at(&[0u8], 0); + let mut map1 = PathMap::::new(); + map1.set_val_at(&[0u8, 0, 0, 0], 0); + map1.set_val_at(&[1u8], 0); + { + let mut wz = map0.write_zipper_at_path(&[]); + let mut rz = map1.read_zipper_at_path(&[]); + wz.join_into(&rz); + wz.descend_first_byte(); + rz.to_next_val(); + wz.subtract_into(&rz, false); + rz.to_next_val(); + wz.meet_into(&rz, false); + } + assert_eq!(map0.path_exists_at(&[0u8]), true, "meet_into leaves [0] dangling"); + let st = { let mut wz = map0.write_zipper(); wz.restrict(&map1.read_zipper()) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(map0.path_exists_at(&[0u8]), false); + assert_eq!(vals(&map0), vec![(vec![1], 0)]); + } } From c882a38f2e14d66161db01e914b90907e6f8eb26 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 01:34:16 +0000 Subject: [PATCH 15/73] Don't report a change for dropping a shadowed dangling slot Cherry-pick of a subagent's 194887a. A LineListNode may hold a value and an onward child under the *same* key -- that is how a path which both ends in a value and continues into a subtrie is stored. Emptying the child without pruning leaves the link in place, and the slot then carries nothing at all: there is nothing below an empty link, and the path it stands at is already there because of the value beside it. `remove_subtries` creates these routinely. Both algebras correctly *drop* such a slot, and both wrongly reported the drop as a change. `psubtract_dyn` saw `(None, Identity)` from `subtract_from_slot_contents`, fell through to the generic arm and returned `Element`; `pmeet_dyn_oriented` put the slot in `self_payloads`, where `EmptyNode::pmeet_dyn` answered `None` with a zero mask and `pmeet_generic` ANDed the combined mask to zero. No mask constant was wrong -- the identity bit was dropped by a slot that should not have been in the computation at all. A new `slot_is_shadowed_dangling` predicate (empty onward link AND a key equal to the other slot's) keeps subtract's identity and leaves the slot out of meet's payloads. The key comparison is exact: an empty link under a key of its own really does take a path away, and `Element` is right there. Verified, KNOWN table unmodified: seed 1, 10000 9999 -> 10000/10000 seed 2, 10000 9998 -> 10000/10000 seeds 0,3,7,10,11,12,99 10000/10000, unchanged Seed 2 needed both halves: its two hits were a subtract_into status case and a meet_into one, the same root cause in sibling paths. Conflicted with the restrict fix, which appended tests at the same point in write_zipper.rs; both test blocks are kept. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/line_list_node.rs | 82 +++++++++++++++++++++++++++++++++++++++---- src/write_zipper.rs | 78 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 6 deletions(-) diff --git a/src/line_list_node.rs b/src/line_list_node.rs index 6f44e25f..2ab29c91 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -1192,6 +1192,31 @@ impl LineListNode { AlgebraicResult::Identity(SELF_IDENT) } } + /// `true` when the slot holds an *empty* onward link under the same key as the node's other + /// slot. + /// + /// Two slots may share a key: that is how a value and the onward child at the same path are + /// stored (see [validate_node]). When the onward child of such a pair is empty, the slot + /// contributes nothing at all -- there is nothing below an empty link, and the existence of + /// the path itself is already carried by the sibling standing at the same key. So the node + /// holds exactly the same trie with the slot as without it. + /// + /// Pairs like this are ordinary: `remove_subtries` leaves an empty link behind whenever it + /// clears a subtrie without pruning, and `graft_child_maps` with `remove_unmasked` does the + /// same. + fn slot_is_shadowed_dangling(&self, slot: usize) -> bool { + let is_used_child = if slot == 0 { self.is_used_child_0() } else { self.is_used_child_1() }; + if !is_used_child || !self.is_used::<1>() { + return false + } + let (key0, key1) = self.get_both_keys(); + if key0 != key1 { + return false + } + let child = unsafe{ if slot == 0 { self.child_in_slot::<0>() } else { self.child_in_slot::<1>() } }; + child.as_tagged().node_is_empty() + } + /// Internal method to restrict the contents of `SLOT` with the contents of the `other` node fn restrict_slot_contents(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Clone { if self.is_used::() { @@ -2918,6 +2943,24 @@ impl TrieNode for LineListNode debug_assert!(validate_node(self)); let slot0_result = self.subtract_from_slot_contents::<0>(other); let slot1_result = self.subtract_from_slot_contents::<1>(other); + + //A dangling path the source reaches does not survive a subtraction, so + // `subtract_from_slot_contents` answers `None` for a slot holding an empty onward link, + // and dropping it is right. But when that slot is *shadowed* -- an empty link sharing its + // key with the sibling slot -- it was carrying nothing to begin with, so dropping it is + // not a change, and the node that is left holds exactly the trie it held before. Letting + // the `None` through rebuilt the node around the surviving slot and reported `Element` for + // a destination that `subtract_into` had not touched. + match (&slot0_result, &slot1_result) { + (AlgebraicResult::None, AlgebraicResult::Identity(_)) if self.slot_is_shadowed_dangling(0) => { + return AlgebraicResult::Identity(SELF_IDENT) + }, + (AlgebraicResult::Identity(_), AlgebraicResult::None) if self.slot_is_shadowed_dangling(1) => { + return AlgebraicResult::Identity(SELF_IDENT) + }, + _ => {} + } + self.combine_slot_results_into_node_result(slot0_result, slot1_result) } fn prestrict_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> { @@ -2941,16 +2984,35 @@ impl LineListNode { let mut self_payloads_buf: [(&[u8], PayloadRef); 2] = [(&[], PayloadRef::None); 2]; + //A shadowed dangling slot carries nothing (see `slot_is_shadowed_dangling`), and a dangling + // path survives no meet, so it can only ever answer `None` and drag the identity mask to + // zero -- making the node report `Element` for a meet result that holds exactly what + // `self` holds. Leaving it out of the meet altogether gives the same result trie with an + // identity mask that tells the truth. + let skipped = if self.slot_is_shadowed_dangling(0) { + Some(0) + } else if self.slot_is_shadowed_dangling(1) { + Some(1) + } else { + None + }; + let self_slot_count = self.used_slot_count(); - let self_payloads = match self_slot_count { - 0 => return AlgebraicResult::None, - 1 => { + let self_payloads = match (self_slot_count, skipped) { + (0, _) => return AlgebraicResult::None, + (_, Some(0)) => { + let key = unsafe{ self.key_unchecked::<1>() }; + let payload = unsafe{ self.payload_in_slot::<1>() }; + self_payloads_buf[0] = (key, payload); + &self_payloads_buf[..1] + }, + (1, _) | (_, Some(1)) => { let key = unsafe{ self.key_unchecked::<0>() }; let payload = unsafe{ self.payload_in_slot::<0>() }; self_payloads_buf[0] = (key, payload); &self_payloads_buf[..1] }, - 2 => { + (2, None) => { let (key0, key1) = self.get_both_keys(); let payload0 = unsafe{ self.payload_in_slot::<0>() }; let payload1 = unsafe{ self.payload_in_slot::<1>() }; @@ -2963,8 +3025,16 @@ impl LineListNode { pmeet_generic::<2, V, A, _>(self_payloads, other, swapped, |payloads| { debug_assert_eq!(payloads.len(), self_payloads.len()); - let slot0_payload = payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into()); - let slot1_payload = payloads.get_mut(1).and_then(|p| core::mem::take(p)).map(|p| p.into()); + //With a slot skipped, the single result belongs to the slot that stayed in, and the + // skipped one is dropped -- which is what the meet would have done with it anyway. + let (slot0_payload, slot1_payload) = match skipped { + Some(0) => (None, payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into())), + Some(_) => (payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into()), None), + None => ( + payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into()), + payloads.get_mut(1).and_then(|p| core::mem::take(p)).map(|p| p.into()), + ), + }; let new_node = self.clone_with_updated_payloads(slot0_payload, slot1_payload).unwrap(); TrieNodeODRc::new_in(new_node, self.alloc.clone()) }) diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 545c926b..410d6e1a 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -7246,4 +7246,82 @@ mod tests { assert_eq!(map0.path_exists_at(&[0u8]), false); assert_eq!(vals(&map0), vec![(vec![1], 0)]); } + + /// A list node may hold a value and an onward child under the *same* key -- that is how a path + /// that both ends and continues is stored. Emptying the child without pruning leaves the link + /// in place, and the slot is then carrying nothing at all: there is nothing below an empty + /// link, and the path it stands at is already there because of the value beside it. + /// + /// `subtract_into` and `meet_into` both drop such a slot, correctly -- a dangling path the + /// source reaches survives neither operation. But dropping something that was carrying + /// nothing is not a change, and both used to rebuild the node around the surviving slot and + /// report `Element` for a destination holding exactly what it held before. + /// + /// The shadowing matters: an empty link under a key of its *own* really does take a path away + /// with it, and that is an `Element`. Only a link sharing its key with the slot beside it is + /// invisible. + #[test] + fn write_zipper_shadowed_dangling_slot_is_identity() { + fn mk(ps: &[(&[u8], u64)]) -> PathMap { let mut m = PathMap::new(); for (p, v) in ps { m.set_val_at(p, *v); } m } + fn vals(m: &PathMap) -> Vec<(Vec, u64)> { m.iter().map(|(k, v)| (k.to_vec(), *v)).collect() } + + //`insert_prefix` puts a real onward node under `[0]`, a value is set beside it, and + // `remove_branches` then empties the node but leaves the link. Both slots stand at `[0]`. + fn dst_with_shadowed_dangling() -> PathMap { + let seed = mk(&[(&[0, 0], 0)]); + let mut dst = PathMap::::new(); + { + let mut wz = dst.write_zipper(); + wz.graft(&seed.read_zipper()); + wz.descend_to_byte(0); + wz.insert_prefix(&[0]); + wz.get_val_or_set_mut(1); + wz.remove_branches(false); + } + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + dst + } + + //Nothing of `src` collides with the value at `[0]`, so the subtraction takes nothing away + let mut dst = dst_with_shadowed_dangling(); + let src = mk(&[(&[0, 0], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + + //The meet keeps the value at `[0]` and drops the dangling link, which changes nothing + let mut dst = dst_with_shadowed_dangling(); + let src = mk(&[(&[0], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.meet_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + + //A subtraction that really does annihilate the value beside the dangling link still says so + let mut dst = dst_with_shadowed_dangling(); + let src = mk(&[(&[0], 1)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::None); + assert_eq!(vals(&dst), vec![]); + + //...and so does a meet that drops it + let mut dst = dst_with_shadowed_dangling(); + let src = mk(&[(&[1], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.meet_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::None); + assert_eq!(vals(&dst), vec![]); + + //An empty link that is *not* shadowed stands at a path of its own, `[0, 0]` here, and + // dropping it takes that path away -- a change, and still reported as one + let mut dst = mk(&[(&[0], 1), (&[0, 0], 2)]); + { + let empty = PathMap::::new(); + let mut wz = dst.write_zipper_at_path(&[0, 0]); + wz.graft(&empty.read_zipper()); + } + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + let src = mk(&[(&[0, 0], 9)]); + let st = { let mut wz = dst.write_zipper(); wz.subtract_into(&src.read_zipper(), false) }; + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(vals(&dst), vec![(vec![0], 1)]); + } } From ff41532929c29a76947ce80b243d3ae021db2512 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 04:30:31 +0000 Subject: [PATCH 16/73] Revive the Rust reference model, and bring it up to the current Lean one The port of lean/PathMapModel/ to Rust was written against a tree where the harness was examples/common/harness.rs and the model was its own cargo example target. Both are gone: the harness is the `differential` crate now, so the model lands in differential/src/reference/ with differential/src/bin/reference.rs as its trace front end, next to pathmap_trace and act_trace. The move costs one invariant and buys it back. "The model shares no code with the implementation" used to be a fact about the build -- examples/reference/ did not depend on `pathmap` at all -- and `differential` does depend on it, so a stray `use pathmap::utils::ByteMask` would now compile and the differential would start agreeing for the wrong reason. So it is asserted instead: `reference::tests::model_does_not_touch_the_crate` reads the model's own source and fails if any of it names the crate. `reference` is also not glob re-exported from lib.rs, because it defines its own `PathMap`, `run` and `hex_path`. The Lean model moved slightly while this was shelved, and the model follows it rather than the other way round -- it is the reference being validated against: * `u64Ops.psub` returns `Identity(SELF_IDENT)`, not `Element(*self)`, when the values differ: subtracting a value that is not there leaves the destination alone and says so. * every `skip` renders as `skip:`. The seven reasons are transcribed from Fuzz.lean rather than imported from harness.rs -- those are the crate side's strings, and a second transcription that shares a constant with the thing it is checking is not a second transcription. * `restricting` tests the empty-focus guard before the ACT guard, and `meet_k_path_into` splits `meetKPathUnspecified` into its two disjuncts, so the skip names which rule fired. Both were invisible while every skip rendered as a bare `skip`. * op 54 `graft_child_maps` is no longer quarantined, so the model runs it: the source's own child submaps, one single-byte mask each. This is the one op the archived model had never executed, its `graft_child_maps` having been written and then skipped in the same commit. * `Zip::to_next_sibling_byte` carries the current model's account of the read zipper root escape -- a live bug, skipped at the root -- not the archive's "FIXED, no longer skipped". `differential.py --model` comes back with it (it was removed in ceaaa30 when the port was staged separately), including the rule that the KNOWN table does not apply in that mode: it is a list of crate defects and the crate is not involved, so every divergence is new. PATHMAP_REFERENCE overrides the binary search the same way PATHMAP_TRACE does. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/bin/reference.rs | 47 ++ differential/src/lib.rs | 9 + differential/src/reference/basic.rs | 245 ++++++++ differential/src/reference/check.rs | 528 +++++++++++++++++ differential/src/reference/fuzz.rs | 814 ++++++++++++++++++++++++++ differential/src/reference/laws.rs | 387 ++++++++++++ differential/src/reference/map.rs | 153 +++++ differential/src/reference/mod.rs | 159 +++++ differential/src/reference/pathmap.rs | 575 ++++++++++++++++++ differential/src/reference/write.rs | 705 ++++++++++++++++++++++ differential/src/reference/zipper.rs | 578 ++++++++++++++++++ lean/differential.py | 40 +- 12 files changed, 4234 insertions(+), 6 deletions(-) create mode 100644 differential/src/bin/reference.rs create mode 100644 differential/src/reference/basic.rs create mode 100644 differential/src/reference/check.rs create mode 100644 differential/src/reference/fuzz.rs create mode 100644 differential/src/reference/laws.rs create mode 100644 differential/src/reference/map.rs create mode 100644 differential/src/reference/mod.rs create mode 100644 differential/src/reference/pathmap.rs create mode 100644 differential/src/reference/write.rs create mode 100644 differential/src/reference/zipper.rs diff --git a/differential/src/bin/reference.rs b/differential/src/bin/reference.rs new file mode 100644 index 00000000..05a456d7 --- /dev/null +++ b/differential/src/bin/reference.rs @@ -0,0 +1,47 @@ +//! Prints the differential trace for a fuzzer input, from the **Rust reference +//! model** — the port of `lean/PathMapModel/` in `differential/src/reference/`. +//! +//! reference # or read the bytes from stdin +//! reference --act # ACT-mode skips +//! +//! There are three front ends over one wire format: +//! +//! | binary | drives | +//! |---|---| +//! | `lean/.lake/build/bin/pathmap-oracle` | the Lean model (`lean/PathMapModel/Fuzz.lean`) | +//! | `pathmap_trace` | the real crate (`differential/src/harness.rs`) | +//! | this | the Rust model (`differential/src/reference/`) | +//! +//! `lean/differential.py` diffs any two of them. Model-against-model — +//! `differential.py --model` — is the acceptance test for the port: the two are +//! independent transcriptions of the same specification in different languages, +//! so a diff means one of them is wrong and nothing about the crate is in +//! question. The `KNOWN` table of tolerated crate defects therefore does not +//! apply in that mode, and `differential.py` does not consult it. +//! +//! Once the two models are known to agree, `in_process` drops the pipes +//! entirely and compares the Rust model against the crate in memory. + +use differential::reference::fuzz::run; +use differential::server::serve; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let act = args.iter().any(|a| a == "--act"); + // Resident mode: one process, many inputs over stdin. See `serve`. + if args.iter().any(|a| a == "--server") { + serve(act, run); + return; + } + let file = args.into_iter().find(|a| a != "--act" && a != "--server"); + let bytes: Vec = match file { + Some(p) => std::fs::read(p).expect("cannot read input"), + None => { + use std::io::Read; + let mut v = Vec::new(); + std::io::stdin().read_to_end(&mut v).unwrap(); + v + } + }; + print!("{}", run(&bytes, act)); +} diff --git a/differential/src/lib.rs b/differential/src/lib.rs index 661cf803..436335e7 100644 --- a/differential/src/lib.rs +++ b/differential/src/lib.rs @@ -8,9 +8,14 @@ //! * [`server`] is the resident-process protocol the driver speaks. //! * [`repro`] turns an input back into standalone `pathmap` calls. //! * [`act`] is the `ArenaCompactTree` read source behind `act_trace`. +//! * [`reference`] is a second executable model: a Rust transcription of the +//! same Lean specification, sharing no code with `pathmap`. It is what +//! `bin/reference.rs` and `bin/in_process.rs` drive, and what +//! `differential.py --model` validates against the Lean oracle. pub mod act; pub mod harness; +pub mod reference; pub mod repro; pub mod server; @@ -18,3 +23,7 @@ pub use act::*; pub use harness::*; pub use repro::*; pub use server::*; + +// `reference` is deliberately *not* glob re-exported: it defines its own +// `PathMap`, `run`, `hex_path` and `show_val`, which are the model's and must +// never be confused with the crate's. Name it by path. diff --git a/differential/src/reference/basic.rs b/differential/src/reference/basic.rs new file mode 100644 index 00000000..2bb3ea87 --- /dev/null +++ b/differential/src/reference/basic.rs @@ -0,0 +1,245 @@ +//! `Basic.lean` — the vocabulary the rest of the model is written in. +//! +//! * [`path`] — keys, and the prefix order on them. +//! * [`ValOps`] / [`ValRes`] — the fragment of `Lattice` / `DistributiveLattice` +//! that the trie operations actually consume. +//! * [`ByteMask`] — the 256-bit child mask. +//! * [`AlgStatus`] — `pathmap::ring::AlgebraicStatus`. + +// =========================================================================== +// Basic.lean — paths +// =========================================================================== + +/// Path helpers. +/// +/// `Path.isPrefixOf`, `Path.stripPrefix` and `Path.lt` are all provided by +/// `[u8]` itself — `starts_with`, `strip_prefix` and `Ord` respectively — and +/// the slice order is exactly the model's `Path.lt`, so they are not restated. +pub mod path { + /// `Path.prefixes`: every prefix of `p`, shortest first: `[]`, `p[..1]`, ..., `p`. + pub fn prefixes(p: &[u8]) -> impl Iterator { + (0..=p.len()).map(move |n| &p[..n]) + } + + /// `Path.properPrefixes`: every *proper* prefix of `p`, shortest first. + /// + /// Used to phrase "the nearest ancestor such that ..." as a filter over + /// ancestors rather than as a loop that walks upward. + pub fn proper_prefixes(p: &[u8]) -> impl Iterator { + (0..p.len()).map(move |n| &p[..n]) + } + + /// The deepest proper prefix of `p` satisfying `pred`, if any. + /// + /// `proper_prefixes` is shortest-first and the Lean model takes the *last* + /// qualifying element; scanning from the deep end and stopping at the first + /// hit is the same answer without building the list. + pub fn deepest_proper_prefix(p: &[u8], mut pred: impl FnMut(&[u8]) -> bool) -> Option<&[u8]> { + (0..p.len()).rev().map(|n| &p[..n]).find(|a| pred(a)) + } + + /// `p ++ q`, as an owned path. + pub fn cat(p: &[u8], q: &[u8]) -> Vec { + let mut r = Vec::with_capacity(p.len() + q.len()); + r.extend_from_slice(p); + r.extend_from_slice(q); + r + } + + /// `p ++ [b]`, as an owned path. + pub fn push(p: &[u8], b: u8) -> Vec { + let mut r = Vec::with_capacity(p.len() + 1); + r.extend_from_slice(p); + r.push(b); + r + } +} + +// =========================================================================== +// Basic.lean — value operations +// =========================================================================== + +/// `pathmap::ring::AlgebraicResult` at the *value* level: either a freshly +/// computed element, an assertion that the result is already one of the operands +/// (`Identity`, with the `SELF_IDENT` / `COUNTER_IDENT` mask spelled out as two +/// flags), or annihilation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ValRes { + /// `AlgebraicResult::Element` + Elem(V), + /// `AlgebraicResult::Identity(mask)`; the flags are `SELF_IDENT`, `COUNTER_IDENT`. + Identity(bool, bool), + /// `AlgebraicResult::None` + None, +} + +impl ValRes { + /// Turn a `ValRes` into the value it denotes, given both operands. + pub fn resolve(self, a: &V, b: &V) -> Option + where + V: Clone, + { + match self { + ValRes::Elem(v) => Some(v), + ValRes::Identity(s, _) => Some(if s { a.clone() } else { b.clone() }), + ValRes::None => None, + } + } +} + +/// A value type together with the algebraic operations `pathmap` requires of it. +/// +/// These return [`ValRes`] rather than plain values because `pathmap` reports +/// `AlgebraicStatus` to the caller, and the status depends on *which variant* +/// the value operation returned, not on whether the stored value changed. +/// `u64`'s `pjoin`, for instance, returns `Identity(SELF_IDENT)` rather than +/// `Element` of the value it selected, so a join reports that nothing changed. +pub trait ValOps { + /// `Lattice::pjoin` + fn pjoin(&self, a: &V, b: &V) -> ValRes; + /// `Lattice::pmeet` + fn pmeet(&self, a: &V, b: &V) -> ValRes; + /// `DistributiveLattice::psubtract` + fn psub(&self, a: &V, b: &V) -> ValRes; + /// Decidable equality on values. + fn beq(&self, a: &V, b: &V) -> bool; +} + +/// The instance `pathmap` provides for `u64` (see `impl Lattice for u64` in +/// `src/ring.rs`). +/// +/// Both `pjoin` and `pmeet` return `Identity(SELF_IDENT)`: they are *left-biased +/// projections* that ignore the counterpart value entirely. `psubtract` +/// annihilates only when the two values are equal, and otherwise returns +/// `Identity(SELF_IDENT)`: subtracting a value that is not there leaves the +/// destination alone, and says so. This is the instance the differential fuzz +/// target uses, so the model reproduces it exactly rather than assuming a "real" +/// lattice. +#[derive(Clone, Copy, Debug, Default)] +pub struct U64Ops; + +impl ValOps for U64Ops { + fn pjoin(&self, _: &u64, _: &u64) -> ValRes { + ValRes::Identity(true, false) + } + fn pmeet(&self, _: &u64, _: &u64) -> ValRes { + ValRes::Identity(true, false) + } + fn psub(&self, a: &u64, b: &u64) -> ValRes { + if a == b { ValRes::None } else { ValRes::Identity(true, false) } + } + fn beq(&self, a: &u64, b: &u64) -> bool { + a == b + } +} + +// =========================================================================== +// Basic.lean — byte masks +// =========================================================================== + +/// `pathmap`'s 256-bit child mask, modelled as the ascending, duplicate-free +/// list of its set bytes — the only thing the API observes about it. +/// +/// Deliberately *not* `pathmap::utils::ByteMask`: a model that borrowed the +/// crate's mask could not detect a bug in it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ByteMask(Vec); + +impl ByteMask { + /// Sort/dedup a byte list into canonical form — `ByteMask.ofList`. + pub fn of_list(bs: impl IntoIterator) -> Self { + let mut v: Vec = bs.into_iter().collect(); + v.sort_unstable(); + v.dedup(); + ByteMask(v) + } + + /// Every byte set — the mask a `remove_unmasked_branches` no-op needs. + pub fn full() -> Self { + ByteMask((0..=255u8).collect()) + } + + /// The set bytes, ascending. + pub fn bytes(&self) -> &[u8] { + &self.0 + } + + /// `ByteMask::count_bits`. + pub fn count_bits(&self) -> usize { + self.0.len() + } + + /// Whether `b` is set. + pub fn contains(&self, b: u8) -> bool { + self.0.binary_search(&b).is_ok() + } + + /// `ByteMask::indexed_bit::` — the `idx`-th set byte in ascending order. + pub fn indexed_bit(&self, idx: usize) -> Option { + self.0.get(idx).copied() + } + + /// `ByteMask::next_bit` — the least set byte strictly greater than `b`. + pub fn next_bit(&self, b: u8) -> Option { + self.0.iter().copied().find(|&x| x > b) + } + + /// `ByteMask::prev_bit` — the greatest set byte strictly less than `b`. + pub fn prev_bit(&self, b: u8) -> Option { + self.0.iter().copied().rev().find(|&x| x < b) + } +} + +// =========================================================================== +// Basic.lean — algebraic status +// =========================================================================== + +/// `pathmap::ring::AlgebraicStatus`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AlgStatus { + /// `self` holds the operation's output. + Element, + /// `self` was not modified by the operation. + Identity, + /// `self` was annihilated and is now empty. + None, +} + +impl AlgStatus { + /// `AlgebraicStatus::merge`, transcribed from `src/ring.rs`. + pub fn merge(a: AlgStatus, b: AlgStatus, self_none: bool, b_none: bool) -> AlgStatus { + match (a, b) { + (AlgStatus::None, AlgStatus::None) => AlgStatus::None, + (AlgStatus::None, AlgStatus::Element) => AlgStatus::Element, + (AlgStatus::None, AlgStatus::Identity) => { + if self_none { AlgStatus::Identity } else { AlgStatus::Element } + } + (AlgStatus::Identity, AlgStatus::Element) => AlgStatus::Element, + (AlgStatus::Identity, AlgStatus::Identity) => AlgStatus::Identity, + (AlgStatus::Identity, AlgStatus::None) => { + if b_none { AlgStatus::Identity } else { AlgStatus::Element } + } + (AlgStatus::Element, _) => AlgStatus::Element, + } + } + + /// The `AlgebraicStatus` a [`ValRes`] induces. + pub fn of_val_res(r: &ValRes) -> AlgStatus { + match r { + ValRes::Elem(_) => AlgStatus::Element, + ValRes::Identity(_, _) => AlgStatus::Identity, + ValRes::None => AlgStatus::None, + } + } +} + +impl std::fmt::Display for AlgStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + AlgStatus::Element => "Element", + AlgStatus::Identity => "Identity", + AlgStatus::None => "None", + }) + } +} + diff --git a/differential/src/reference/check.rs b/differential/src/reference/check.rs new file mode 100644 index 00000000..161bb4fd --- /dev/null +++ b/differential/src/reference/check.rs @@ -0,0 +1,528 @@ +//! `Check.lean` — the checks, plus a randomised sweep with no Lean counterpart. +//! +//! `Check.lean`'s `#guard`s are evaluated by the Lean compiler on every build; +//! here they are `#[test]`s, which is why the example carries `test = true`. + +// =========================================================================== +// Check.lean — build-time checks +// =========================================================================== + +#[cfg(test)] +mod tests { + //! Two kinds of check, transcribed from `lean/PathMapModel/Check.lean`, where + //! they are `#guard`s evaluated by the Lean compiler on every build: + //! + //! * **Regression fixtures** transcribed from `pathmap`'s own unit tests + //! (`src/write_zipper.rs`). These pin the model to observed crate behaviour + //! for the operations whose semantics are hardest to read off the source — + //! pruning and `drop_head`. + //! * **Law checks**: the metamorphic properties from [`crate::reference::laws`], + //! evaluated over a battery of maps chosen to cover branch points, + //! single-child runs, values at interior nodes, dangling paths, and the + //! empty map. + + use crate::reference::basic::U64Ops; + use crate::reference::laws::*; + use crate::reference::map; + use crate::reference::pathmap::PathMap; + use crate::reference::zipper::Zip; + + type T = PathMap; + const OPS: U64Ops = U64Ops; + + /// Build a map from a list of path/value pairs. + fn mk(entries: &[(&[u8], u64)]) -> T { + let mut t = PathMap::empty(); + for (p, v) in entries { + t.set_val(p, *v); + } + t + } + + fn zip_at(t: &T, root: &[u8], path: &[u8]) -> Zip { + Zip::at_path(t.clone(), root, path) + } + + // -- fixtures ----------------------------------------------------------- + + /// Branching at the root and at depth 1, with a value at an interior node. + fn f_branch() -> T { + mk(&[(&[], 0), (&[0], 1), (&[0, 0], 2), (&[0, 1], 3), (&[1], 4)]) + } + /// A single-child run: the shape `descend_until` / `ascend_until` care about. + fn f_run() -> T { + mk(&[(&[0, 0, 0, 0], 7)]) + } + /// Overlapping prefixes without a root value. + fn f_side() -> T { + mk(&[(&[0, 0], 9), (&[0, 2], 8), (&[2], 7)]) + } + /// The empty map. + fn f_empty() -> T { + PathMap::empty() + } + /// A dangling path: `[0,1,2]` exists but carries no value. + fn f_dangle() -> T { + let mut t = mk(&[(&[0], 5)]); + t.add_path(&[0, 1, 2]); + t + } + /// Two values under a shared 2-byte prefix, plus a deeper third. + fn f_deep() -> T { + mk(&[(&[1, 1, 0], 1), (&[1, 1, 1], 2), (&[1, 1, 1, 3], 3)]) + } + + fn fixtures() -> Vec { + vec![f_branch(), f_run(), f_side(), f_empty(), f_dangle(), f_deep()] + } + + /// Focus positions worth probing in each fixture. + fn probes() -> Vec> { + vec![ + vec![], + vec![0], + vec![1], + vec![0, 0], + vec![0, 1], + vec![1, 1], + vec![0, 1, 2], + vec![3], + ] + } + + fn all_zips() -> Vec> { + let mut zs = Vec::new(); + for t in fixtures() { + for p in probes() { + zs.push(Zip::at_path(t.clone(), &[], &p)); + } + } + zs + } + + // -- regression fixtures from `src/write_zipper.rs` ---------------------- + + /// `write_zipper_prune_path_test2`, first phase: removing the value at + /// `[0,0,1,0,0]` and pruning removes exactly 3 bytes, back to the branch at + /// `[0,0]`. + #[test] + fn prune_path_test2() { + let t = mk(&[(&[0], 0), (&[0, 0, 0], 0), (&[0, 0, 1, 0, 0], 0)]); + let mut z = zip_at(&t, &[], &[0, 0, 1, 0, 0]); + z.remove_val(false); + assert!(z.path_exists(), "remove_val(false) leaves the location dangling"); + assert_eq!(z.prune_path(), 3); + assert!(!z.path_exists()); + assert_eq!(zip_at(&t, &[], &[0, 0]).child_count(), 2); + } + + /// Same test, later phase: a chain with every value removed prunes all the + /// way to the map root (7 bytes), and the root itself survives. + #[test] + fn prune_path_test2b() { + let mut t = mk(&[(&[0, 0, 0, 1, 2, 3, 4], 0)]); + t.remove_val(&[0, 0, 0, 1, 2, 3, 4]); + + let mut z = zip_at(&t, &[], &[0, 0, 0, 1, 2, 3, 4]); + assert_eq!(z.prune_path(), 7); + z.reset(); + assert!(z.path_exists(), "the map root always survives"); + + // Pruning is a no-op above the dangling tip (the location still has a + // child) and below it (the location does not exist). + assert_eq!(zip_at(&t, &[], &[0, 0, 0, 1, 2, 3]).prune_path(), 0); + assert_eq!(zip_at(&t, &[], &[0, 0, 0, 1, 2, 3, 4, 5]).prune_path(), 0); + } + + /// Pruning *does* rise above the zipper's root, contradicting the doc comment + /// on `ZipperWriting::prune_path`. A zipper rooted at `[0,0]` looking at the + /// dangling tip of the same chain prunes all 7 bytes, back to the map root — + /// not the 5 that lie below its own root. Verified against pathmap 0.3.1. + #[test] + fn prune_path_rises_above_zipper_root() { + let mut t = mk(&[(&[0, 0, 0, 1, 2, 3, 4], 0)]); + t.remove_val(&[0, 0, 0, 1, 2, 3, 4]); + let mut z = zip_at(&t, &[0, 0], &[0, 1, 2, 3, 4]); + assert_eq!(z.prune_path(), 7); + assert!(z.trie.is_empty_map()); + } + + /// `write_zipper_drop_head_test3`: `[[0,0],[0,1],[1,0],[1,1]]` with + /// `join_k_path_into(1)` collapses to 2 values. + #[test] + fn drop_head_test3() { + let t = mk(&[(&[0, 0], 0), (&[0, 1], 1), (&[1, 0], 2), (&[1, 1], 3)]); + let mut z = zip_at(&t, &[], &[]); + z.join_k_path_into(&OPS, 1, true); + assert_eq!(z.val_count(), 2); + } + + /// `write_zipper_drop_head_test6`: dropping 4 bytes from paths that are at + /// most 4 long annihilates everything, because values at depth exactly `k` + /// are lost. + #[test] + fn drop_head_test6() { + let t = mk(&[ + (&[193, 191, 193, 193, 191], 0), + (&[193, 191, 193, 194, 12, 28], 1), + (&[193, 191, 193, 194, 18, 9], 2), + (&[193, 191, 194, 193, 191], 3), + (&[193, 191, 194, 194, 12, 28], 4), + (&[193, 191, 194, 194, 15, 47], 5), + (&[193, 191, 194, 194, 18, 9], 6), + ]); + let mut z = zip_at(&t, &[], &[193, 191]); + assert!(!z.join_k_path_into(&OPS, 4, true)); + assert_eq!(z.val_count(), 0); + } + + /// `write_zipper_drop_head_test1`: under the root `123:`, dropping 4 bytes + /// rewrites `abc:Bob` to `Bob` and `dog:Bob:Fido` to `Bob:Fido`. + #[test] + fn drop_head_test1() { + let t = mk(&[(b"123:abc:Bob", 0), (b"123:dog:Bob:Fido", 1)]); + let mut z = zip_at(&t, b"123:", &[]); + z.join_k_path_into(&OPS, 4, true); + let r = z.trie; + assert_eq!(r.val_at(b"123:Bob"), Some(&0)); + assert_eq!(r.val_at(b"123:Bob:Fido"), Some(&1)); + assert_eq!(r.val_count(&[]), 2); + } + + // -- structural invariants over every fixture --------------------------- + + #[test] + fn structural_invariants() { + for t in fixtures() { + assert!(vals_exist(&t)); + assert!(prefix_closed(&t)); + assert!(val_count_agrees(&t)); + for p in probes() { + assert!(child_mask_agrees(&t, &p)); + } + } + } + + // -- zipper law checks -------------------------------------------------- + + #[test] + fn zipper_laws() { + for z in all_zips() { + assert!(remove_val_leaves_path(&z)); + assert!(create_path_idempotent(&z)); + assert!(create_then_prune(&OPS, &z)); + assert!(descend_indexed_lands(&z)); + assert!(sibling_round_trip(&z)); + assert!(descend_until_observed_exact(&z)); + assert!(ascend_until_accounts(&z)); + assert!(to_next_val_monotone(&z)); + assert!(set_val_then_val(&OPS, &z, 42)); + assert!(take_then_graft(&OPS, &z)); + assert!(join_empty_identity(&OPS, &z)); + assert!(join_self_identity(&OPS, &z)); + assert!(remove_branches_keeps_val(&OPS, &z)); + assert!(remove_unmasked_full_mask(&OPS, &z)); + assert!(remove_unmasked_empty_mask(&OPS, &z)); + for p in probes() { + assert!(descend_to_existing_lands(&z, &p)); + } + for pre in [&[0u8][..], &[0, 1][..], &[2, 2][..]] { + assert!(drop_head_undoes_insert_prefix(&OPS, &z, pre)); + } + } + } + + #[test] + fn graft_laws() { + let zs = all_zips(); + for z in &zs { + for w in &zs { + assert!(graft_then_make_map(&OPS, z, w)); + } + } + for s in fixtures() { + for (a, b) in [ + (&[0u8][..], &[9u8][..]), + (&[1][..], &[2][..]), + (&[0, 0][..], &[1, 1][..]), + ] { + assert!(grafted_copies_independent(&OPS, &s, a, b, &[3, 7], 999)); + } + } + } + + // -- algebraic law checks ----------------------------------------------- + + #[test] + fn algebraic_laws() { + let fs = fixtures(); + for a in &fs { + assert!(join_idem(&OPS, a)); + assert!(meet_idem_on_vals(&OPS, a)); + assert!(sub_self_empty_vals(&OPS, a)); + assert!(restrict_self(&OPS, a)); + for b in &fs { + assert!(join_comm_on_paths(&OPS, a, b)); + for c in &fs { + assert!(join_assoc(&OPS, a, b, c)); + } + } + } + } + + // -- naive oracles ------------------------------------------------------ + // + // `PathMap::join` / `meet` / `sub` are defined in terms of `ValOps`, which was + // transcribed from `src/ring.rs` — so a defect in the crate's `Option` + // lattice impls could have been copied into the model, after which the + // differential would agree and report nothing. + // + // These oracles are written from set theory instead: they say which *keys* + // survive without consulting `ValOps`, `PathMap`, or anything else the model + // shares with the crate. They are deliberately naive and quadratic. Where + // they and the real definitions agree, the shared-derivation risk is excluded + // for that operation. + // + // `U64Ops::psub` annihilates exactly when the two values are equal, which is + // the one fact about the value type these need. + + fn keys(t: &T) -> Vec> { + t.vals().map(|(k, _)| k.clone()).collect() + } + + fn sorted_dedup(mut v: Vec>) -> Vec> { + v.sort(); + v.dedup(); + v + } + + /// Keys of `a` and `b` together: what `join` must produce. + fn join_keys_oracle(a: &T, b: &T) -> Vec> { + let mut v = keys(a); + v.extend(keys(b)); + sorted_dedup(v) + } + + /// Keys in both: what `meet` must produce. + fn meet_keys_oracle(a: &T, b: &T) -> Vec> { + sorted_dedup(keys(a).into_iter().filter(|k| b.val_at(k).is_some()).collect()) + } + + /// Keys of `a` whose value `b` does not annihilate: what `sub` must produce. + /// For `u64`, `psubtract` annihilates exactly on equal values. + fn sub_keys_oracle(a: &T, b: &T) -> Vec> { + sorted_dedup( + a.vals() + .filter(|(k, av)| b.val_at(k) != Some(av)) + .map(|(k, _)| k.clone()) + .collect(), + ) + } + + /// `restrict` against the `BTreeSet` oracle from + /// `tests/pathmap_algebra_differential.rs`: a path of `a` survives when some + /// prefix of it — the empty prefix and the path itself both count — carries a + /// value in `b`. + fn restrict_oracle(a: &T, b: &T) -> Vec> { + a.vals() + .filter(|(p, _)| (0..=p.len()).any(|i| b.val_at(&p[..i]).is_some())) + .map(|(p, _)| p.clone()) + .collect() + } + + #[test] + fn naive_oracles_agree() { + let fs = fixtures(); + for a in &fs { + for b in &fs { + assert_eq!(keys(&PathMap::join(&OPS, a, b)), join_keys_oracle(a, b)); + assert_eq!(keys(&PathMap::meet(&OPS, a, b)), meet_keys_oracle(a, b)); + assert_eq!(keys(&PathMap::sub(&OPS, a, b)), sub_keys_oracle(a, b)); + assert_eq!(keys(&map::restrict(a, b)), restrict_oracle(a, b)); + } + } + } + + /// The minimal shapes from `restrict_matches_btreeset_oracle`, which caught a + /// real `prestrict` bug: a location that both carries a value and branches. + #[test] + fn restrict_minimal_shapes() { + let shapes: Vec> = vec![ + vec![(&[0, 1], 0), (&[0, 1, 2], 1)], + vec![(&[0, 1], 0), (&[0, 1, 2], 1), (&[0, 1, 3], 2)], + vec![(&[0], 0), (&[0, 1], 1), (&[0, 1, 2], 2)], + vec![(&[0], 0), (&[0, 1, 2], 1), (&[0, 1, 3], 2)], + vec![(&[0], 0), (&[0, 1], 1), (&[0, 1, 2], 2), (&[0, 1, 3], 3), (&[9, 9], 4)], + ]; + for es in shapes { + assert!(restrict_self(&OPS, &mk(&es))); + } + } +} + +#[cfg(test)] +mod fuzz_tests { + //! Randomised self-consistency, over far more states than the fixture + //! battery reaches. + //! + //! The fixtures in [`crate::reference::check::tests`] are the ones the Lean `#guard`s use: 6 + //! maps × 8 probes. They were chosen to cover the interesting *shapes*, but + //! they are still 48 states, and a range query with an off-by-one in it can + //! easily survive all 48. So this walks random operation sequences, and + //! after every step asserts + //! + //! * the map is still canonical — the one invariant `BTreeMap` does not + //! enforce for us, and the one every constructor here is responsible for; + //! * every law from [`crate::reference::laws`] that applies to a single state. + //! + //! What this cannot catch is a *faithful-looking but wrong* transcription + //! that is internally consistent — a mis-ordered `AlgebraicStatus::merge`, + //! say. Only a differential against the Lean model or the crate finds those. + + use crate::reference::basic::U64Ops; + use crate::reference::laws::*; + use crate::reference::pathmap::PathMap; + use crate::reference::zipper::Zip; + + const OPS: U64Ops = U64Ops; + + /// splitmix64 — a deterministic PRNG, so a failure is reproducible from the + /// seed alone and the test pulls in no dependency. + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + /// A short path over a 3-letter alphabet, so the generated maps share + /// prefixes heavily — that is where branch points, dangling chains and + /// single-child runs live. + fn path(&mut self, max: u64) -> Vec { + let n = self.below(max + 1); + (0..n).map(|_| self.below(3) as u8).collect() + } + } + + /// Every law that is a predicate on one zipper state. + fn check_state(z: &Zip) { + assert!(z.trie.is_canonical(), "not canonical: {:?}", z.trie.entries.keys()); + assert!(vals_exist(&z.trie)); + assert!(prefix_closed(&z.trie)); + assert!(val_count_agrees(&z.trie)); + assert!(child_mask_agrees(&z.trie, &z.focus())); + assert!(remove_val_leaves_path(z)); + assert!(create_path_idempotent(z)); + // Not universal: `prune_path` reaches above the zipper root, so a + // dangling ancestor is swept up along with the created chain. + if create_then_prune_applies(z) { + assert!(create_then_prune(&OPS, z)); + } + assert!(descend_indexed_lands(z)); + assert!(sibling_round_trip(z)); + assert!(descend_until_observed_exact(z)); + assert!(ascend_until_accounts(z)); + assert!(to_next_val_monotone(z)); + assert!(set_val_then_val(&OPS, z, 42)); + assert!(take_then_graft(&OPS, z)); + assert!(join_empty_identity(&OPS, z)); + assert!(join_self_identity(&OPS, z)); + assert!(remove_branches_keeps_val(&OPS, z)); + assert!(remove_unmasked_full_mask(&OPS, z)); + assert!(remove_unmasked_empty_mask(&OPS, z)); + assert!(drop_head_undoes_insert_prefix(&OPS, z, &[0, 1])); + assert!(join_idem(&OPS, &z.trie)); + assert!(meet_idem_on_vals(&OPS, &z.trie)); + assert!(sub_self_empty_vals(&OPS, &z.trie)); + // Not universal: a dangling path with no valued ancestor is dropped by + // `restrict`. See `restrict_self`. + if restrict_self_applies(&z.trie) { + assert!(restrict_self(&OPS, &z.trie)); + } + } + + #[test] + fn random_programs_keep_every_law() { + for seed in 0..256u64 { + let mut r = Rng(seed.wrapping_mul(0x2545F4914F6CDD1D) | 1); + let root = r.path(2); + let mut z = Zip::at(PathMap::empty(), &root); + // A source zipper over an independently seeded map, for the binary + // operations. + let mut src_map = PathMap::empty(); + for _ in 0..r.below(6) { + src_map.set_val(&r.path(4), r.below(4)); + } + let src = Zip::at_path(src_map, &[], &r.path(2)); + + for _ in 0..40 { + let k = r.path(3); + match r.below(18) { + 0 => { + z.set_val(r.below(4)); + } + 1 => { + z.remove_val(r.below(2) == 1); + } + 2 => { + z.create_path(); + } + 3 => { + z.prune_path(); + } + 4 => { + z.remove_branches(r.below(2) == 1); + } + 5 => { + z.descend_to(&k); + } + 6 => { + z.ascend(r.below(3) as usize); + } + 7 => { + z.ascend_until(); + } + 8 => { + z.descend_until(); + } + 9 => { + z.to_next_val(); + } + 10 => { + z.graft(&src); + } + 11 => { + z.join_into(&OPS, &src); + } + 12 => { + z.meet_into(&OPS, &src, r.below(2) == 1); + } + 13 => { + z.subtract_into(&OPS, &src, r.below(2) == 1); + } + 14 => { + z.restrict(&OPS, &src); + } + 15 => { + z.insert_prefix(&k); + } + 16 => { + z.join_k_path_into(&OPS, 1 + r.below(2) as usize, r.below(2) == 1); + } + _ => { + z.take_map(r.below(2) == 1); + } + } + check_state(&z); + // The zipper must never leave the subtree it was created in. + assert!(z.focus().starts_with(&root), "escaped its root"); + } + } + } +} diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs new file mode 100644 index 00000000..2db64c34 --- /dev/null +++ b/differential/src/reference/fuzz.rs @@ -0,0 +1,814 @@ +//! `Fuzz.lean` — the differential front end: decode a fuzzer input into a +//! program over two maps and two zippers, run it against the model, and render +//! the trace. +//! +//! A transcription of `lean/PathMapModel/Fuzz.lean`, which it must track byte +//! for byte: the same operand decoding in the same order (including bytes +//! consumed before an operation decides to `skip`), the same operation table, +//! and the same rendering. `NOPS` and `MAX_STEPS` are the same contract +//! `differential/src/harness.rs` names. +//! +//! Kept as its own module rather than inline in a binary so both front ends — +//! `bin/reference.rs` and the in-process comparator `bin/in_process.rs` — drive +//! the model through the same op table, instead of there being a fourth copy of +//! it. The three copies that must stay in lockstep are `Fuzz.lean`, +//! `differential/src/harness.rs` and this file; there is no fourth. + +// One growable buffer rather than a `Vec`; see the note in +// `differential/src/harness.rs`. +use core::fmt::Write as _; + +use crate::reference::basic::{AlgStatus, ByteMask, U64Ops, path}; +use crate::reference::pathmap::PathMap; +use crate::reference::zipper::Zip; + +/// Number of distinct operations. Must match `PathMapModel.Fuzz.nops` and +/// `NOPS` in `differential/src/harness.rs`. +const NOPS: usize = 56; +/// Maximum operations executed. Must match the `maxSteps` default in `Fuzz.run`. +const MAX_STEPS: usize = 256; +/// Maximum entries in a `dump`. Must match `Fuzz.dumpAt`. +const DUMP_CAP: usize = 64; + +const OPS: U64Ops = U64Ops; + +// --------------------------------------------------------------------------- +// Skip reasons — `Fuzz` §Skip reasons +// --------------------------------------------------------------------------- +// +// Why an operation was skipped. Every `skip` in the trace carries one of these, +// so a skipped op says which rule declined it rather than just that something +// declined it. `lean/PathMapModel/Fuzz.lean` and `differential/src/harness.rs` +// emit the same tokens; all three must agree exactly or every input with a skip +// diverges. +// +// Spelled out here rather than imported from `crate::harness`: those are the +// *crate side's* strings, and the point of a second transcription is that it +// agrees with the Lean one by having been written from it, not by sharing a +// constant with the thing it is checking. +// +// Each is recorded in lean/FINDINGS.md and commented at its site. + +/// The ACT read source cannot be a merge source (`ZipperInfallibleSubtries` is +/// not implemented for it) or does not implement the trait the op needs. +const SKIP_ACT: &str = "skip:act"; +/// `to_next`/`to_prev_sibling_byte` at the zipper root, where the native read +/// zipper escapes its own root. +const SKIP_AT_ROOT: &str = "skip:at-root"; +/// A degenerate `k = 0`. +const SKIP_K0: &str = "skip:k0"; +/// The focus has nothing below it, where the op's behaviour is a function of +/// node materialisation rather than of trie state. +const SKIP_EMPTY_FOCUS: &str = "skip:empty-focus"; +/// `insert_prefix("")`, which destroys the subtrie. +const SKIP_EMPTY_PATH: &str = "skip:empty-path"; +/// A prune on a write zipper not rooted at the map root, where the depth pruned +/// is a function of internal node layout. +const SKIP_OFF_ROOT_PRUNE: &str = "skip:off-root-prune"; +/// The op is disabled outright. Nothing uses this today: op 54 +/// (`graft_child_maps`) was quarantined when the archive was taken and has since +/// been let back in. Kept because `Fuzz.lean` and `harness.rs` both still +/// define it, and the vocabulary is the contract. +#[allow(dead_code)] +const SKIP_QUARANTINED: &str = "skip:quarantined"; + +// --------------------------------------------------------------------------- +// Decoder — `Fuzz.Dec` +// --------------------------------------------------------------------------- + +struct Dec<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl Dec<'_> { + /// Read one byte; `None` once the input is exhausted, which ends the program. + fn u8(&mut self) -> Option { + let b = *self.bytes.get(self.pos)?; + self.pos += 1; + Some(b) + } + /// Read one byte reduced modulo `m`. + fn modn(&mut self, m: usize) -> Option { + let b = self.u8()?; + Some(if m == 0 { 0 } else { (b as usize) % m }) + } + /// Path bytes live in a 4-letter alphabet so generated tries share prefixes. + fn path_byte(&mut self) -> Option { + Some(self.u8()? % 4) + } + fn path_n(&mut self, n: usize) -> Option> { + let mut v = Vec::with_capacity(n); + for _ in 0..n { + v.push(self.path_byte()?); + } + Some(v) + } + /// Read a length-prefixed path (`len := u8 % lim`). + fn path(&mut self, lim: usize) -> Option> { + let n = self.modn(lim)?; + self.path_n(n) + } + fn boolean(&mut self) -> Option { + Some(self.u8()? % 2 == 1) + } +} + +// --------------------------------------------------------------------------- +// Rendering — `Fuzz` §Rendering +// --------------------------------------------------------------------------- + +fn hex_path(p: &[u8]) -> String { + if p.is_empty() { + "_".to_string() + } else { + p.iter().map(|b| format!("{b:02x}")).collect() + } +} + +fn show_val(v: Option<&u64>) -> String { + match v { + None => "-".to_string(), + Some(v) => format!("{v}"), + } +} + +/// Render the `Option` the movement operations return: the byte moved to, or +/// `-` for "did not move". +fn show_byte_opt(b: Option) -> String { + match b { + None => "-".to_string(), + Some(b) => format!("{b:02x}"), + } +} + +fn show_bool(b: bool) -> &'static str { + if b { "1" } else { "0" } +} + +fn show_status(s: AlgStatus) -> String { + s.to_string() +} + +/// All locations at and below `root`, depth-first, capped so a runaway trie +/// cannot make the trace unbounded. +fn dump_at(t: &PathMap, root: &[u8]) -> String { + t.subtrie(root) + .paths() + .take(DUMP_CAP) + .map(|q| format!("{}:{}", hex_path(q), show_val(t.val_at(&path::cat(root, q))))) + .collect::>() + .join(",") +} + +// --------------------------------------------------------------------------- +// Interpreter state — `Fuzz.St` +// --------------------------------------------------------------------------- + +/// Two maps, two zippers: `wz` writes into map0, `rz` reads map1. Keeping the +/// read source in a *separate* map is what lets the real crate hold both zippers +/// at once. +struct St { + wz: Zip, + rz: Zip, + out: String, + step: usize, + /// Read source is an `ArenaCompactTree` rather than a `PathMap`, which cannot + /// be the source of a graft or an algebraic merge. Those operations report + /// `skip` on both sides of the comparison. + act: bool, +} + +/// The per-step fingerprint of one zipper. +fn fingerprint(z: &Zip) -> String { + format!( + "{} o{} e{} v{} c{} n{} f{}", + hex_path(&z.path), + hex_path(&z.focus()), + show_bool(z.path_exists()), + show_val(z.val()), + z.child_count(), + z.val_count(), + // `focus_byte` is unspecified at the root, so it is only compared below it. + if z.at_root() { "?".to_string() } else { show_byte_opt(z.focus_byte()) } + ) +} + +impl St { + fn emit(&mut self, name: &str, ret: &str) { + let _ = writeln!(self.out, + "{} {} ret={} W={} R={}", + self.step, + name, + ret, + fingerprint(&self.wz), + fingerprint(&self.rz) + ); + self.step += 1; + } + + /// The zipper selected by a target byte (`0` = write zipper, `1` = read). + fn target(&mut self, t: usize) -> &mut Zip { + if t == 0 { &mut self.wz } else { &mut self.rz } + } + + fn target_ref(&self, t: usize) -> &Zip { + if t == 0 { &self.wz } else { &self.rz } + } + + /// Is the *explicit* `prune_path` / `prune_ascend` well-defined for this + /// state? Only for a write zipper rooted at the map root: with a non-empty + /// root the depth pruned becomes a function of node layout rather than of the + /// logical trie. See `Fuzz.pruneable` and lean/FINDINGS.md finding 7. + fn pruneable(&self) -> bool { + self.wz.root.is_empty() + } +} + +/// The `prune` flag the harness passes to operations that take one. +/// +/// Always `false`: the flag's effect is a function of internal node layout rather +/// than of the logical trie, so there is nothing for a model to agree with. +const NO_PRUNE: bool = false; + +/// A full `k`-path iteration: `descend_first_k_path` followed by `to_next_k_path` +/// until it runs out (capped at 32 stops). This is the only well-defined way to +/// use the k-path primitives. +fn k_walk(z: &mut Zip, k: usize) -> Vec> { + if !z.descend_first_k_path(k) { + return Vec::new(); + } + let mut acc = vec![z.path.clone()]; + for _ in 0..31 { + if z.to_next_k_path(k) { + acc.push(z.path.clone()); + } else { + break; + } + } + acc +} + +// --------------------------------------------------------------------------- +// The operation table — `Fuzz.step` +// --------------------------------------------------------------------------- +// +// `op % NOPS` selects the operation. Ops `0`–`26` act on a target zipper chosen +// by a following `u8 % 2` byte (`0` = write zipper, `1` = read zipper); the rest +// are write-zipper operations. +// +// Operand bytes are consumed *before* an operation decides to `skip`, exactly as +// in `Fuzz.lean`, because the byte stream must stay in step across all three +// front ends. `None` means the input ran out mid-operation, which ends the run. + +fn step(s: &mut St, d: &mut Dec) -> Option<()> { + let op = d.u8()? as usize % NOPS; + match op { + 0 => { + let t = d.modn(2)?; + let p = d.path(6)?; + s.target(t).descend_to(&p); + s.emit("descend_to", &hex_path(&p)); + } + 1 => { + let t = d.modn(2)?; + let b = d.path_byte()?; + s.target(t).descend_to_byte(b); + s.emit("descend_to_byte", &format!("{b:02x}")); + } + 2 => { + let t = d.modn(2)?; + let n = d.modn(8)?; + let r = s.target(t).ascend(n); + s.emit("ascend", &r.to_string()); + } + 3 => { + let t = d.modn(2)?; + let r = s.target(t).ascend_byte(); + s.emit("ascend_byte", show_bool(r)); + } + 4 => { + let t = d.modn(2)?; + s.target(t).reset(); + s.emit("reset", "-"); + } + 5 => { + let t = d.modn(2)?; + let r = s.target(t).descend_first_byte(); + s.emit("descend_first_byte", &show_byte_opt(r)); + } + 6 => { + let t = d.modn(2)?; + let r = s.target(t).descend_last_byte(); + s.emit("descend_last_byte", &show_byte_opt(r)); + } + 7 => { + let t = d.modn(2)?; + let i = d.modn(6)?; + let r = s.target(t).descend_indexed_byte(i); + s.emit("descend_indexed_byte", &show_byte_opt(r)); + } + 8 => { + let t = d.modn(2)?; + let r = s.target(t).descend_until(); + s.emit("descend_until", show_bool(r)); + } + 9 => { + let t = d.modn(2)?; + let r = s.target(t).ascend_until(); + s.emit("ascend_until", &r.to_string()); + } + 10 => { + let t = d.modn(2)?; + let r = s.target(t).ascend_until_branch(); + s.emit("ascend_until_branch", &r.to_string()); + } + 11 => { + // Skipped at the zipper root: `ReadZipper::to_next_sibling_byte` + // escapes its own root there (see the notes in + // `Zip::to_next_sibling_byte`). + let t = d.modn(2)?; + if s.target_ref(t).at_root() { + s.emit("to_next_sibling_byte", SKIP_AT_ROOT); + } else { + let r = s.target(t).to_next_sibling_byte(); + s.emit("to_next_sibling_byte", &show_byte_opt(r)); + } + } + 12 => { + let t = d.modn(2)?; + if s.target_ref(t).at_root() { + s.emit("to_prev_sibling_byte", SKIP_AT_ROOT); + } else { + let r = s.target(t).to_prev_sibling_byte(); + s.emit("to_prev_sibling_byte", &show_byte_opt(r)); + } + } + 13 => { + let t = d.modn(2)?; + let r = s.target(t).to_next_step(); + s.emit("to_next_step", show_bool(r)); + } + 14 => { + // `ZipperIteration` is read-only: the target byte is still consumed, + // but the operation always applies to the read zipper. + let _t = d.modn(2)?; + let r = s.rz.to_next_val(); + s.emit("to_next_val", show_bool(r)); + } + 15 => { + let _t = d.modn(2)?; + let k = d.modn(4)?; + // `k = 0` is degenerate: `k_path_internal` treats "already at depth + // base+0" as a hit and reports success without moving, then + // `to_next_k_path(0)` reports success forever. Skipped. + if k == 0 { + s.emit("descend_first_k_path", SKIP_K0); + } else { + let r = s.rz.descend_first_k_path(k); + s.emit("descend_first_k_path", show_bool(r)); + } + } + 16 => { + let _t = d.modn(2)?; + let k = d.modn(4)?; + // `to_next_k_path` is only meaningful as the continuation of a + // `descend_first_k_path` iteration, so the op is the whole walk. + if k == 0 { + s.emit("k_path_walk", SKIP_K0); + } else { + let ps = k_walk(&mut s.rz, k); + let ret = ps.iter().map(|p| hex_path(p)).collect::>().join(","); + s.emit("k_path_walk", &ret); + } + } + 17 => { + let _t = d.modn(2)?; + let r = s.rz.descend_last_path(); + s.emit("descend_last_path", show_bool(r)); + } + 18 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let n = s.target(t).move_to_path(&p); + s.emit("move_to_path", &n.to_string()); + } + 19 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let n = s.target(t).descend_to_existing(&p); + s.emit("descend_to_existing", &n.to_string()); + } + 20 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let n = s.target(t).descend_to_val(&p); + s.emit("descend_to_val", &n.to_string()); + } + 21 => { + let t = d.modn(2)?; + let b = d.path_byte()?; + let r = s.target(t).descend_to_existing_byte(b); + s.emit("descend_to_existing_byte", show_bool(r)); + } + 22 => { + let t = d.modn(2)?; + let n = d.modn(8)?; + let r = s.target(t).descend_until_max_bytes(n); + s.emit("descend_until_max_bytes", show_bool(r)); + } + 23 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let r = s.target(t).descend_to_check(&p); + s.emit("descend_to_check", show_bool(r)); + } + 24 => { + let t = d.modn(2)?; + let p = d.path(6)?; + let ret = show_val(s.target_ref(t).val_at(&p)); + s.emit("val_at", &ret); + } + 25 => { + let t = d.modn(2)?; + if s.act && t == 1 { + s.emit("make_map_val_count", SKIP_ACT); + } else { + let ret = s.target_ref(t).make_map().val_count(&[]).to_string(); + s.emit("make_map_val_count", &ret); + } + } + 26 => { + let t = d.modn(2)?; + let z = s.target_ref(t); + let ret = dump_at(&z.trie, &z.focus()); + s.emit("dump", &ret); + } + 27 => { + let v = d.u8()?; + let old = s.wz.set_val(v as u64); + s.emit("set_val", &show_val(old.as_ref())); + } + 28 => { + let _pr = d.boolean()?; + let old = s.wz.remove_val(NO_PRUNE); + s.emit("remove_val", &show_val(old.as_ref())); + } + 29 => { + let r = s.wz.create_path(); + s.emit("create_path", show_bool(r)); + } + 30 => { + if s.pruneable() { + let n = s.wz.prune_path(); + s.emit("prune_path", &n.to_string()); + } else { + s.emit("prune_path", SKIP_OFF_ROOT_PRUNE); + } + } + 31 => { + if s.pruneable() { + let n = s.wz.prune_ascend(); + s.emit("prune_ascend", &n.to_string()); + } else { + s.emit("prune_ascend", SKIP_OFF_ROOT_PRUNE); + } + } + 32 => { + let _pr = d.boolean()?; + let leaky = s.wz.focus_node_is_empty(); + let r = s.wz.remove_branches(NO_PRUNE); + let ret = if leaky { "?".to_string() } else { show_bool(r).to_string() }; + s.emit("remove_branches", &ret); + } + 33 => { + let n = d.modn(4)?; + let m = d.path_n(n)?; + let _pr = d.boolean()?; + let mask = ByteMask::of_list(m); + s.wz.remove_unmasked_branches(&mask, NO_PRUNE); + let ret = hex_path(mask.bytes()); + s.emit("remove_unmasked_branches", &ret); + } + 34 => { + if s.act { + s.emit("graft", SKIP_ACT); + } else { + s.wz.graft(&s.rz); + s.emit("graft", "-"); + } + } + 35 => { + let p = d.path(6)?; + if s.act { + s.emit("graft_src_at", SKIP_ACT); + } else { + s.wz.graft_src_at(&s.rz, &p); + s.emit("graft_src_at", &hex_path(&p)); + } + } + 36 => { + if s.act { + s.emit("join_into", SKIP_ACT); + } else { + let st = s.wz.join_into(&OPS, &s.rz); + s.emit("join_into", &show_status(st)); + } + } + 37 => { + if s.act { + s.emit("join_map_into", SKIP_ACT); + } else { + let leaky = s.wz.focus_node_is_empty(); + let m = s.rz.make_map(); + let st = s.wz.join_map_into(&OPS, &m); + let ret = if leaky { "?".to_string() } else { show_status(st) }; + s.emit("join_map_into", &ret); + } + } + 38 => { + let _pr = d.boolean()?; + if s.act { + s.emit("meet_into", SKIP_ACT); + } else { + let st = s.wz.meet_into(&OPS, &s.rz, NO_PRUNE); + s.emit("meet_into", &show_status(st)); + } + } + 39 => { + let _pr = d.boolean()?; + if s.act { + s.emit("subtract_into", SKIP_ACT); + } else { + let st = s.wz.subtract_into(&OPS, &s.rz, NO_PRUNE); + s.emit("subtract_into", &show_status(st)); + } + } + 40 => { + if s.act { + s.emit("restrict", SKIP_ACT); + } else { + let leaky = s.wz.focus_node_is_empty(); + let st = s.wz.restrict(&OPS, &s.rz); + let ret = if leaky { "?".to_string() } else { show_status(st) }; + s.emit("restrict", &ret); + } + } + 41 => { + // Skipped, not merely masked, when either side has nothing below its + // focus: there `restricting` branches on whether an empty node + // happens to be materialised, and the two branches differ in + // *effect*, not just in the reported bool. See FINDINGS.md #8. + // + // This guard is checked *before* the ACT one because the harness + // reaches the ACT skip only by calling `do_restricting`, which it + // does not do once this guard has fired; the two orders were + // indistinguishable while both reasons rendered as a bare `skip`. + if s.wz.focus_node_is_empty() || s.rz.focus_node_is_empty() { + s.emit("restricting", SKIP_EMPTY_FOCUS); + } else if s.act { + s.emit("restricting", SKIP_ACT); + } else { + let r = s.wz.restricting(&s.rz); + s.emit("restricting", show_bool(r)); + } + } + 42 => { + let k = d.modn(4)?; + let _pr = d.boolean()?; + // `join_k_path_into(0)` should be the identity but destroys the + // subtrie in pathmap 0.3.1. + if k == 0 { + s.emit("join_k_path_into", SKIP_K0); + } else { + // The bool is another `AbstractNodeRef` leak: an empty node still + // comes back as `Some(...)` from `into_option()` for some + // representations. Compared only when something survived. + let r = s.wz.join_k_path_into(&OPS, k, NO_PRUNE); + let ret = + if s.wz.focus_node_is_empty() { "?".to_string() } else { show_bool(r).to_string() }; + s.emit("join_k_path_into", &ret); + } + } + 43 => { + let p = d.path(6)?; + // `insert_prefix("")` destroys the subtrie in pathmap 0.3.1. + if p.is_empty() { + s.emit("insert_prefix", SKIP_EMPTY_PATH); + } else { + let r = s.wz.insert_prefix(&p); + s.emit("insert_prefix", show_bool(r)); + } + } + 44 => { + let n = d.modn(6)?; + let r = s.wz.remove_prefix(n); + s.emit("remove_prefix", show_bool(r)); + } + 45 => { + let _pr = d.boolean()?; + let leaky = s.wz.focus_node_is_empty() && s.wz.val().is_none(); + let m = s.wz.take_map(NO_PRUNE); + if leaky { + let mm = m.unwrap_or_else(PathMap::empty); + s.wz.graft_map(&mm); + s.emit("take_map_restore", "?"); + } else { + match m { + Some(mm) => { + s.wz.graft_map(&mm); + s.emit("take_map_restore", "1"); + } + None => s.emit("take_map_restore", "0"), + } + } + } + 46 => { + let k = d.modn(4)?; + let _pr = d.boolean()?; + // `meet_k_path_into` is not implementable for these arguments; see + // `Zip::meet_k_path_unspecified`, whose two disjuncts are split out + // here so the skip names which one fired. The crate side matches. + if k == 0 { + s.emit("meet_k_path_into", SKIP_K0); + } else if s.wz.focus_node_is_empty() { + s.emit("meet_k_path_into", SKIP_EMPTY_FOCUS); + } else { + let r = s.wz.meet_k_path_into(&OPS, k, NO_PRUNE); + s.emit("meet_k_path_into", show_bool(r)); + } + } + 47 => { + let t = d.modn(2)?; + // The blind-zipper addition: `descend_until` reporting the bytes it + // descended. The observer's output is a blind zipper's only account + // of where it went, so it is compared byte for byte. + let (r, obs) = s.target(t).descend_until_observed(); + let ret = format!("{}:{}", show_bool(r), hex_path(&obs)); + s.emit("descend_until_observed", &ret); + } + 48 => { + let v = d.u8()?; + // Writing through the reference `get_val_mut` hands back. It must + // behave like `set_val` where a value exists and do nothing — + // crucially, *not* create the path — where one does not. + let old = s.wz.get_val_mut_write(v as u64); + s.emit("get_val_mut_write", &show_val(old.as_ref())); + } + 49 => { + let v = d.u8()?; + let r = s.wz.get_val_or_set_mut(v as u64); + s.emit("get_val_or_set_mut", &show_val(Some(&r))); + } + 50 => { + let v = d.u8()?; + // `ran` records whether the closure was invoked. The contract says it + // supplies the value "if no value exists", so invoking it when a value + // is already present is observable to any caller whose closure has a + // side effect. + let (r, ran) = s.wz.get_val_or_set_mut_with(v as u64); + let ret = format!("{}:{}", show_val(Some(&r)), show_bool(ran)); + s.emit("get_val_or_set_mut_with", &ret); + } + 51 => { + let t = d.modn(2)?; + let p = d.path(6)?; + // `get_val`/`get_val_at` differ from `val`/`val_at` only in the + // lifetime of the reference they return, so they must give the same + // answer. `agree` is `1` in the model by construction; a `0` from the + // crate is the whole point of the op. + let z = s.target_ref(t); + let ret = format!("{}:{}:1", show_val(z.val()), show_val(z.val_at(&p))); + s.emit("get_val_agrees", &ret); + } + 52 => { + // `to_next_get_val` must advance exactly as `to_next_val` does and + // hand back the value at the new focus. + if s.act { + s.emit("to_next_get_val", SKIP_ACT); + } else { + let moved = s.rz.to_next_val(); + let v = if moved { s.rz.val().copied() } else { None }; + let ret = format!("{}:{}:1", show_bool(moved), show_val(v.as_ref())); + s.emit("to_next_get_val", &ret); + } + } + 53 => { + let n = d.modn(4)?; + let m = d.path_n(n)?; + let ru = d.boolean()?; + if s.act { + s.emit("graft_masked_branches", SKIP_ACT); + } else { + let mask = ByteMask::of_list(m); + s.wz.graft_masked_branches(&s.rz, &mask, ru); + let ret = format!("{}:{}", hex_path(mask.bytes()), show_bool(ru)); + s.emit("graft_masked_branches", &ret); + } + } + 54 => { + let n = d.modn(4)?; + let m = d.path_n(n)?; + let ru = d.boolean()?; + // Fed the source's own child subtries, this must agree with + // `graft_masked_branches` on the same mask. It used to be skipped + // outright (`skip:quarantined`) because `graft_child_maps` was broken + // three ways (FINDINGS.md #15); the Lean model has since let it back + // in, and this follows. + if s.act { + s.emit("graft_child_maps", SKIP_ACT); + } else { + let mask = ByteMask::of_list(m); + let sf = s.rz.focus(); + let maps: Vec<(ByteMask, PathMap)> = mask + .bytes() + .iter() + .map(|&b| { + (ByteMask::of_list([b]), s.rz.trie.subtrie(&path::push(&sf, b))) + }) + .collect(); + s.wz.graft_child_maps(&maps, ru); + let ret = format!("{}:{}", hex_path(mask.bytes()), show_bool(ru)); + s.emit("graft_child_maps", &ret); + } + } + 55 => { + let p = d.path(6)?; + // `meet_2` takes two sources; the second is the first moved to `p`. + if s.act { + s.emit("meet_2", SKIP_ACT); + } else { + let mut b = s.rz.clone(); + b.path.extend_from_slice(&p); + let st = s.wz.meet_2(&OPS, &s.rz, &b); + s.emit("meet_2", &show_status(st)); + } + } + _ => s.emit("nop", "-"), + } + Some(()) +} + +// --------------------------------------------------------------------------- +// Header and entry point — `Fuzz.header` / `Fuzz.run` +// --------------------------------------------------------------------------- + +/// Decode `n` seed entries and insert them into `t`. +fn seed(t: &mut PathMap, d: &mut Dec, n: usize) -> Option<()> { + for _ in 0..n { + let p = d.path(6)?; + let v = d.u8()?; + t.set_val(&p, v as u64); + } + Some(()) +} + +/// Decode the header: two seeded maps and the two zipper roots. +fn header(d: &mut Dec, act: bool) -> Option { + let n0 = d.modn(8)?; + let mut m0 = PathMap::empty(); + seed(&mut m0, d, n0)?; + let n1 = d.modn(8)?; + let mut m1 = PathMap::empty(); + seed(&mut m1, d, n1)?; + let r0 = d.path(4)?; + let r1 = d.path(4)?; + // Both zipper roots are created if absent. A zipper whose *root* does not + // exist can escape it — `to_next_sibling_byte` and `to_next_step` fall back + // on the parent's child mask and walk out of the granted subtrie. Making the + // roots exist keeps that one bug from contaminating every other comparison. + if !r0.is_empty() { + m0.add_path(&r0); + } + if !r1.is_empty() { + m1.add_path(&r1); + } + Some(St { + wz: Zip::at(m0, &r0), + rz: Zip::at(m1, &r1), + out: String::new(), + step: 0, + act, + }) +} + +/// Decode and run a fuzzer input, returning the trace lines. +pub fn run(bytes: &[u8], act: bool) -> String { + let mut d = Dec { bytes, pos: 0 }; + let Some(mut s) = header(&mut d, act) else { + return "EMPTY\n".to_string(); + }; + for _ in 0..MAX_STEPS { + if step(&mut s, &mut d).is_none() { + break; + } + } + let mut out = s.out; + let _ = writeln!(out, "MAP0 {}", dump_at(&s.wz.trie, &[])); + let _ = writeln!(out, "MAP1 {}", dump_at(&s.rz.trie, &[])); + let _ = writeln!(out, "ROOT0 {}", hex_path(&s.wz.root)); + let _ = writeln!(out, "ROOT1 {}", hex_path(&s.rz.root)); + out +} + +// The resident-server protocol. Plumbing only — it knows nothing about tries, diff --git a/differential/src/reference/laws.rs b/differential/src/reference/laws.rs new file mode 100644 index 00000000..197b2f61 --- /dev/null +++ b/differential/src/reference/laws.rs @@ -0,0 +1,387 @@ +//! `Spec.lean` §2 — checkable laws. +//! +//! Metamorphic properties relating *different* API functions: "`join` is +//! commutative", "`take_map` then `graft_map` is the identity", "`drop_head` +//! undoes `insert_prefix`". +//! +//! Each returns `true` when it holds, so the same function can serve as a unit +//! test over the fixture battery *and* as a runtime assertion on every state a +//! fuzzer reaches. A property that holds in the model but fails in `pathmap` is +//! a crate bug; one that fails in both is a spec bug. Both are worth finding, +//! which is why the laws are kept separate from the definitions. +//! +//! The Lean model's §1 — the *proved* theorems about the cursor algebra — has no +//! counterpart here. Those are proofs, not tests; see +//! `lean/PathMapModel/Spec.lean`. + +use crate::reference::basic::{AlgStatus, ByteMask, ValOps, path}; +use crate::reference::map; +use crate::reference::pathmap::{Entry, PathMap}; +use crate::reference::zipper::Zip; + +/// Every location carrying a value exists. +/// +/// Structurally true of any [`PathMap`] — a value cannot be recorded at a +/// location that is not in the map — so a failure here means a map was built +/// by hand rather than through the constructors. +pub fn vals_exist(t: &PathMap) -> bool { + t.vals().all(|(k, _)| t.path_exists(k)) +} + +/// The set of existing locations is closed under taking prefixes. +pub fn prefix_closed(t: &PathMap) -> bool { + t.path_exists(&[]) && t.paths().all(|q| path::prefixes(q).all(|r| t.path_exists(r))) +} + +/// `child_mask` lists exactly the bytes whose child location exists. +pub fn child_mask_agrees(t: &PathMap, p: &[u8]) -> bool { + let m = t.child_mask(p); + m.bytes().iter().all(|&b| t.path_exists(&path::push(p, b))) + && t.paths().all(|q| { + if q.len() == p.len() + 1 && q.starts_with(p) { m.contains(q[p.len()]) } else { true } + }) +} + +/// `val_count` at the root counts exactly the entries of `iter`. +pub fn val_count_agrees(t: &PathMap) -> bool { + t.val_count(&[]) == t.vals().count() +} + +/// After `set_val` the focus is a location carrying exactly that value. +/// +/// One `Entry` case, rather than "the value is `v`" and "the path exists" +/// checked separately — the second was only there because the first could not +/// say it. +pub fn set_val_then_val(ops: &impl ValOps, z: &Zip, v: V) -> bool { + let mut z = z.clone(); + z.set_val(v.clone()); + match z.entry() { + Entry::Valued(w) => ops.beq(w, &v), + Entry::Bare | Entry::Absent => false, + } +} + +/// `remove_val` clears the value but leaves the location dangling — which is +/// to say the focus ends up exactly [`Entry::Bare`]. +pub fn remove_val_leaves_path(z: &Zip) -> bool { + if !z.path_exists() { + return true; + } + let mut z = z.clone(); + z.remove_val(false); + z.entry().is_bare() +} + +/// `create_path` makes the focus exist; a second call reports "already there". +pub fn create_path_idempotent(z: &Zip) -> bool { + if z.at_root() { + return true; + } + let mut z = z.clone(); + z.create_path(); + let existed = z.path_exists(); + let again = z.create_path(); + existed && !again && z.path_exists() +} + +/// `prune_path` undoes a `create_path` that dangled off an existing location. +/// +/// **Precondition: the location it dangles off must survive the prune.** +/// `create_path` adds one child to the deepest existing ancestor `a` of the +/// focus, and `prune_path` then walks back up to the first location that +/// carries a value, branches, or is the *map* root — it does not stop at the +/// zipper root (see [`Zip::prune_path`]). So when `a` is itself a dangling +/// tip, the prune sweeps `a` up as well and the map does not come back. With +/// `{[] , [2]}` and a zipper rooted at `[2]` focused at `[2,0]`, create then +/// prune leaves `{[]}`: `[2]` is gone. Verified to hold in the Lean model +/// too, so this is the law's scope, not a divergence. +/// +/// `Check.lean` never sees it because every zipper in its battery is rooted +/// at `[]`; check [`create_then_prune_applies`] before asserting this one. +pub fn create_then_prune(ops: &impl ValOps, z: &Zip) -> bool { + if z.at_root() || z.path_exists() { + return true; + } + let mut w = z.clone(); + w.create_path(); + w.prune_path(); + w.trie.beq_t(ops, &z.trie) +} + +/// Whether [`create_then_prune`]'s precondition holds: the deepest existing +/// ancestor of the focus is the map root, carries a value, or already has a +/// child — in each case it still qualifies as a stop once `create_path` has +/// hung one more child off it. +pub fn create_then_prune_applies(z: &Zip) -> bool { + let f = z.focus(); + match path::deepest_proper_prefix(&f, |a| z.trie.path_exists(a)) { + Some(a) => a.is_empty() || z.trie.val_at(a).is_some() || z.trie.child_count(a) >= 1, + None => true, + } +} + +/// `descend_to_existing` always lands on an existing location, provided the +/// focus existed when it was called. +pub fn descend_to_existing_lands(z: &Zip, k: &[u8]) -> bool { + if !z.path_exists() { + return true; + } + let mut z = z.clone(); + z.descend_to_existing(k); + z.path_exists() +} + +/// `descend_indexed_byte` lands on an existing location for every valid index, +/// and the byte it reports is the byte it landed on. +pub fn descend_indexed_lands(z: &Zip) -> bool { + (0..z.child_count()).all(|i| { + let mut w = z.clone(); + match w.descend_indexed_byte(i) { + Some(b) => w.path_exists() && w.focus_byte() == Some(b), + None => false, + } + }) +} + +/// `to_next_sibling_byte` and `to_prev_sibling_byte` are mutually inverse — +/// **but only from an existing location**. +/// +/// From an off-map focus the round trip fails, and correctly so: the sibling +/// moves are defined by the parent's `child_mask`, which does not contain the +/// current byte, so `next_bit` jumps to some larger set byte and `prev_bit` +/// comes back to a *different* one. A caller that steps sideways from a +/// non-existent path cannot expect to step back. +pub fn sibling_round_trip(z: &Zip) -> bool { + if !z.path_exists() { + return true; + } + let mut w = z.clone(); + match w.to_next_sibling_byte() { + None => true, + Some(b) => { + w.focus_byte() == Some(b) + && w.to_prev_sibling_byte().is_some() + && w.path == z.path + } + } +} + +/// `descend_until_observed` reports exactly the bytes it descended. +/// +/// A blind zipper has no `path()`, so this sequence is its only account of +/// where it went; it must equal the path delta. +#[allow(clippy::nonminimal_bool)] // stated as the Lean law is, not simplified +pub fn descend_until_observed_exact(z: &Zip) -> bool { + let mut w = z.clone(); + let (moved, obs) = w.descend_until_observed(); + w.path == path::cat(&z.path, &obs) && moved == !obs.is_empty() +} + +/// `ascend_until` never ascends past the zipper's root, and reports the exact +/// distance travelled. +pub fn ascend_until_accounts(z: &Zip) -> bool { + let mut w = z.clone(); + let n = w.ascend_until(); + w.path.len() + n == z.path.len() && z.path.len() >= n +} + +/// `to_next_val` enumerates values in strictly increasing depth-first order +/// and finishes at the root. +pub fn to_next_val_monotone(z: &Zip) -> bool { + let mut w = z.clone(); + if w.to_next_val() { + z.path < w.path && w.is_val() + } else { + w.at_root() + } +} + +/// `join` is commutative **on locations**, but not on values. +/// +/// `pathmap`'s `u64` (and `usize`, `u32`, `u16`, `u8`) `Lattice` instances +/// define `pjoin` as `Identity(SELF_IDENT)` — a left-biased projection. So +/// joining two maps that disagree at a key keeps whichever value belongs to +/// the receiver, and `a.join(b)` and `b.join(a)` differ. The *set of paths* +/// is still symmetric, and that is what this law asserts. Any test that +/// assumes value-level commutativity is asserting something the crate does not +/// promise for these value types. +pub fn join_comm_on_paths( + ops: &impl ValOps, + a: &PathMap, + b: &PathMap, +) -> bool { + let ab = PathMap::join(ops, a, b); + let ba = PathMap::join(ops, b, a); + ab.paths().collect::>() == ba.paths().collect::>() + && ab.vals().map(|(k, _)| k).collect::>() + == ba.vals().map(|(k, _)| k).collect::>() +} + +/// `join` is idempotent. +pub fn join_idem(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::join(ops, a, a).beq_t(ops, a) +} + +/// `join` is associative, values included: left-biasing is itself associative. +pub fn join_assoc( + ops: &impl ValOps, + a: &PathMap, + b: &PathMap, + c: &PathMap, +) -> bool { + let l = PathMap::join(ops, &PathMap::join(ops, a, b), c); + let r = PathMap::join(ops, a, &PathMap::join(ops, b, c)); + l.beq_t(ops, &r) +} + +/// `meet` is idempotent *on values*. It is not idempotent on locations: a +/// meet discards dangling paths, so `meet a a` keeps only the value-bearing +/// skeleton. +pub fn meet_idem_on_vals(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::meet(ops, a, a).vals().map(|(k, _)| k.clone()).collect::>() + == a.vals().map(|(k, _)| k.clone()).collect::>() +} + +/// Subtracting a map from itself leaves no values. +pub fn sub_self_empty_vals(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::sub(ops, a, a).vals().next().is_none() +} + +/// `restrict a a = a`. This is the law that caught a real `prestrict` bug — +/// see `tests/pathmap_algebra_differential.rs`. +/// +/// **Precondition: every location of `a` leads to a value.** The Lean +/// model's justification — "every path of `a` is validated by the value at +/// its own end" — assumes every path *ends* at a value, which is exactly what +/// `create_path` and `remove_val(false)` make false. On +/// `PathMap::empty().add_path(&[0])` the law fails, in this model and in the +/// Lean one alike: `[0]` has no valued prefix, so `restrict` drops it. +/// `Check.lean`'s six fixtures all happen to satisfy the precondition, which +/// is why the `#guard` holds there; a fuzzer reaching a dangling path does +/// not, so check [`restrict_self_applies`] before asserting this one. +pub fn restrict_self(ops: &impl ValOps, a: &PathMap) -> bool { + map::restrict(a, a).beq_t(ops, a) +} + +/// Whether [`restrict_self`]'s precondition holds: every non-root location of +/// `a` has a value at or above it. +pub fn restrict_self_applies(a: &PathMap) -> bool { + a.paths().all(|q| q.is_empty() || PathMap::validated_by(a, q)) +} + +/// `take_map` followed by `graft_map` restores the map exactly. +pub fn take_then_graft(ops: &impl ValOps, z: &Zip) -> bool { + let mut w = z.clone(); + let m = w.take_map(false).unwrap_or_else(PathMap::empty); + w.graft_map(&m); + w.trie.beq_t(ops, &z.trie) +} + +/// Grafting one submap into two places makes two *independent* copies: +/// writing under one leaves the other exactly as it was. +/// +/// In the model this cannot fail — a [`PathMap`] is a flat map of entries, so +/// the two copies are separate elements and there is no aliasing to leak +/// through. The law is stated anyway for two reasons: it is what the crate +/// must do (`graft` clones a refcounted pointer, so the copies really are +/// shared until copy-on-write separates them), and it would catch a model +/// that grew sharing of its own. +/// +/// See FINDINGS.md #16: the crate gets this right where it completes, but +/// aborts when the shared submap contains a dangling path. +pub fn grafted_copies_independent( + ops: &impl ValOps, + s: &PathMap, + a: &[u8], + b: &[u8], + k: &[u8], + v: V, +) -> bool { + let mut z = Zip::at_path(PathMap::empty(), &[], a); + z.graft_map(s); + let mut z = Zip::at_path(z.trie, &[], b); + z.graft_map(s); + let both = z.trie.clone(); + let mut z = Zip::at_path(z.trie, &[], &path::cat(a, k)); + z.set_val(v); + both.subtrie(b).beq_t(ops, &z.trie.subtrie(b)) +} + +/// `graft` copies the source submap: after grafting, `make_map` at the +/// destination equals `make_map` at the source. +pub fn graft_then_make_map( + ops: &impl ValOps, + dst: &Zip, + src: &Zip, +) -> bool { + let mut d = dst.clone(); + d.graft(src); + d.make_map().beq_t(ops, &src.make_map()) +} + +/// `drop_head k` undoes `insert_prefix` of a `k`-byte prefix, as documented +/// on `ZipperWriting::insert_prefix`. +/// +/// Only the *branches* are compared: `insert_prefix` does not move the focus +/// value, and `drop_head` discards values at depth exactly `k`, so the focus +/// value plays no part on either side. +pub fn drop_head_undoes_insert_prefix( + ops: &impl ValOps, + z: &Zip, + pre: &[u8], +) -> bool { + if z.focus_node_is_empty() || pre.is_empty() { + return true; + } + let mut w = z.clone(); + w.insert_prefix(pre); + w.join_k_path_into(ops, pre.len(), false); + w.focus_node().beq_t(ops, &z.focus_node()) +} + +/// `join_into` with an empty source is a no-op and reports `Identity` (or +/// `None` when the destination is empty too). +pub fn join_empty_identity(ops: &impl ValOps, z: &Zip) -> bool { + let src = Zip::new(PathMap::empty()); + let mut w = z.clone(); + let st = w.join_into(ops, &src); + let want = if z.focus_node_is_empty() { AlgStatus::None } else { AlgStatus::Identity }; + w.trie.beq_t(ops, &z.trie) && st == want +} + +/// Joining a zipper into itself is the identity, and reports it. +pub fn join_self_identity(ops: &impl ValOps, z: &Zip) -> bool { + let src = z.clone(); + let mut w = z.clone(); + let st = w.join_into(ops, &src); + let want = if z.focus_node_is_empty() { AlgStatus::None } else { AlgStatus::Identity }; + w.trie.beq_t(ops, &z.trie) && st == want +} + +/// `remove_branches` empties the focus but preserves its value. +pub fn remove_branches_keeps_val(ops: &impl ValOps, z: &Zip) -> bool { + let mut w = z.clone(); + w.remove_branches(false); + w.focus_node_is_empty() + && match (z.val(), w.val()) { + (None, None) => true, + (Some(a), Some(b)) => ops.beq(a, b), + _ => false, + } +} + +/// `remove_unmasked_branches` with a full mask changes nothing. +pub fn remove_unmasked_full_mask(ops: &impl ValOps, z: &Zip) -> bool { + let mut w = z.clone(); + w.remove_unmasked_branches(&ByteMask::full(), false); + w.trie.beq_t(ops, &z.trie) +} + +/// `remove_unmasked_branches` with an empty mask equals `remove_branches`. +pub fn remove_unmasked_empty_mask(ops: &impl ValOps, z: &Zip) -> bool { + let mut a = z.clone(); + a.remove_unmasked_branches(&ByteMask::of_list([]), false); + let mut b = z.clone(); + b.remove_branches(false); + a.trie.beq_t(ops, &b.trie) +} diff --git a/differential/src/reference/map.rs b/differential/src/reference/map.rs new file mode 100644 index 00000000..001488ad --- /dev/null +++ b/differential/src/reference/map.rs @@ -0,0 +1,153 @@ +//! `Map.lean` — the `PathMap` surface. +//! +//! A `pathmap::PathMap` *is* a map: the value at the empty path is the map's +//! root value. Almost every `PathMap` method is implemented in `pathmap` by +//! opening a temporary zipper at the root and delegating, and the model does the +//! same, so the two layers cannot drift apart. +//! +//! The one genuinely map-level operation is [`restrict`], which differs from +//! [`Zip::restrict`]: it consults the *root value* of the right-hand map, which a +//! node-level `prestrict` cannot see. + +use crate::reference::basic::{ByteMask, ValOps}; +use crate::reference::pathmap::{Entry, PathMap}; +use crate::reference::zipper::Zip; + +/// `PathMap::new`. +pub fn new() -> PathMap { + PathMap::empty() +} + +/// `PathMap::single`. +pub fn single(p: &[u8], v: V) -> PathMap { + let mut t = PathMap::empty(); + t.set_val(p, v); + t +} + +/// A read/write zipper at the root of the map. +pub fn zipper(t: PathMap) -> Zip { + Zip::new(t) +} + +/// A zipper rooted at `p` (`read_zipper_at_path` / `write_zipper_at_path`). +/// The zipper cannot ascend above `p`, and its `val()` at the root is the +/// map's value at `p`. +pub fn zipper_at(t: PathMap, p: &[u8]) -> Zip { + Zip::at(t, p) +} + +/// `PathMap::get_val_at` / `get`. +pub fn get_val_at<'a, V: Clone>(t: &'a PathMap, p: &[u8]) -> Option<&'a V> { + t.val_at(p) +} + +/// `PathMap::contains`: whether there is a **value** at `p`. +/// +/// Not the same question as `path_exists_at`, and the gap between them is the +/// bare case: a path created by `create_path`, or left behind by +/// `remove_val(false)`, exists but is not contained. +pub fn contains(t: &PathMap, p: &[u8]) -> bool { + match t.entry_at(p) { + Entry::Valued(_) => true, + Entry::Bare | Entry::Absent => false, + } +} + +/// `PathMap::path_exists_at`. +pub fn path_exists_at(t: &PathMap, p: &[u8]) -> bool { + t.path_exists(p) +} + +/// `PathMap::set_val_at` / `insert`. +pub fn set_val_at(t: &mut PathMap, p: &[u8], v: V) -> Option { + t.set_val(p, v) +} + +/// `PathMap::remove_val_at` / `remove` (`remove` passes `prune = true`). +pub fn remove_val_at(t: &mut PathMap, p: &[u8], prune: bool) -> Option { + with_zipper_at(t, p, |z| z.remove_val(prune)) +} + +/// `PathMap::val_count`. +pub fn val_count(t: &PathMap) -> usize { + t.val_count(&[]) +} + +/// `PathMap::is_empty`. +pub fn is_empty(t: &PathMap) -> bool { + t.is_empty_map() +} + +/// `PathMap::create_path`. +pub fn create_path(t: &mut PathMap, p: &[u8]) -> bool { + with_zipper_at(t, p, |z| z.create_path()) +} + +/// `PathMap::prune_path`. +pub fn prune_path(t: &mut PathMap, p: &[u8]) -> usize { + with_zipper_at(t, p, |z| z.prune_path()) +} + +/// `PathMap::remove_branches_at`. +pub fn remove_branches_at(t: &mut PathMap, p: &[u8], prune: bool) -> bool { + with_zipper_at(t, p, |z| z.remove_branches(prune)) +} + +/// All key/value pairs, in depth-first (lexicographic) key order — +/// `PathMap::iter`. +pub fn iter(t: &PathMap) -> Vec<(Vec, V)> { + t.vals().map(|(k, v)| (k.clone(), v.clone())).collect() +} + +/// `PathMap::join`: union, root values included. +pub fn join(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + PathMap::join(ops, a, b) +} + +/// `PathMap::meet`: intersection, root values included. +pub fn meet(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + PathMap::meet(ops, a, b) +} + +/// `PathMap::subtract`: difference, root values included. +pub fn subtract(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + PathMap::sub(ops, a, b) +} + +/// `PathMap::restrict`: keep the paths of `a` that have *some* prefix +/// carrying a value in `b`. +/// +/// The empty prefix counts here, so a root value in `b` validates everything +/// and the result is `a` unchanged (root value included). Otherwise the +/// result never has a root value, because a node-level `prestrict` has no +/// slot for one. This is the documented behaviour of `PathMap::restrict`, +/// and it is *not* what [`Zip::restrict`] does. +pub fn restrict(a: &PathMap, b: &PathMap) -> PathMap { + if b.val_at(&[]).is_some() { + a.clone() + } else { + PathMap::restrict_below_root(a, b) + } +} + +/// Every `PathMap` method above that mutates is `pathmap`'s "open a temporary +/// zipper at the root, descend, delegate" — spelled out once here so the two +/// layers cannot drift apart. +fn with_zipper_at( + t: &mut PathMap, + p: &[u8], + f: impl FnOnce(&mut Zip) -> R, +) -> R { + let mut z = Zip::new(std::mem::take(t)); + z.descend_to(p); + let r = f(&mut z); + *t = z.trie; + r +} + +/// Re-exported so callers of [`Zip::remove_unmasked_branches`] have a mask to +/// hand without reaching for the crate's own. +pub fn full_mask() -> ByteMask { + ByteMask::full() +} diff --git a/differential/src/reference/mod.rs b/differential/src/reference/mod.rs new file mode 100644 index 00000000..3e40c990 --- /dev/null +++ b/differential/src/reference/mod.rs @@ -0,0 +1,159 @@ +//! An executable reference model of the `pathmap` trie and zipper API: a Rust +//! transcription of the Lean 4 specification in `lean/PathMapModel/`. +//! +//! See `lean/README.md` for the reasoning behind the specification and +//! `lean/FINDINGS.md` for what it found. Every item names the `pathmap` item it +//! specifies, and the file layout mirrors the Lean one one-for-one, so drift +//! between the two is visible as a diff rather than hidden inside a module: +//! +//! | Lean | here | +//! |--------------------------|-----------------| +//! | `Basic.lean` | `basic.rs` | +//! | `PathMap.lean` | `pathmap.rs` | +//! | `Zipper.lean` | `zipper.rs` | +//! | `Write.lean` | `write.rs` | +//! | `Map.lean` | `map.rs` | +//! | `Spec.lean` §2 | `laws.rs` | +//! | `Check.lean` | `check.rs` | +//! | `Fuzz.lean` | `fuzz.rs` | +//! +//! It lives in the `differential` crate rather than in `pathmap/src/` because +//! nothing in the library may depend on it: it is a testing oracle, not a data +//! structure, and keeping it out of the crate proper is what makes "the model +//! shares no code with the implementation" a fact about the build rather than a +//! promise in a comment. +//! +//! # This is not a trie, and that is the point +//! +//! A trie is a prefix tree, which `pathmap` implements with four node types and +//! a great deal of care. The model is a flat `BTreeMap, Option>` of +//! whole paths. It is not a prefix tree and does not try to be. +//! +//! That is deliberate. This is a *specification*, and what it specifies is the +//! meaning a trie carries, not the trie. A model shaped like the implementation +//! would inherit the implementation's structure, and then a bug in how that +//! structure is handled — a node type promoted wrongly, a child index off by +//! one, an empty node where a real one was expected — could be present in both +//! and cancel out. Findings 14, 15 and 16 are bugs of exactly that kind, and +//! they are visible only because the model has no nodes to get wrong. +//! +//! # The model must never reach into the crate +//! +//! No file in this directory may `use pathmap::...`; not `utils::ByteMask`, not +//! `ring::AlgebraicStatus`, not a helper. Shared code is shared risk. In the +//! archive this was enforced by the build — the model was its own cargo example +//! target, which did not depend on `pathmap` at all — and it now lives beside +//! [`crate::harness`], which does. So the rule is checked instead, by the +//! `model_does_not_touch_the_crate` test below, which reads these files and +//! fails if any of them names the crate. +//! +//! The comparison between model and crate belongs in the harness and in +//! `bin/in_process.rs`, not here. +//! +//! # Why `BTreeMap, Option>` +//! +//! The Lean model keeps a list of `(Path, Option V)` pairs held canonical by +//! hand: sorted by the lexicographic path order, duplicate-free, prefix-closed, +//! and containing the empty path. Rust's `Ord for Vec` **is** that order — +//! `[] < [0] < [0,0] < [0,255] < [1]` — which is the order a depth-first +//! traversal of a radix trie visits paths in. So a `BTreeMap` discharges +//! "sorted" and "duplicate-free" structurally, its iteration order is depth-first +//! order for free, and every "the least existing path after the focus such +//! that ..." in the specification becomes a range query. Prefix-closure and the +//! presence of the root remain the model's own responsibility; +//! `PathMap::mk` establishes them and [`laws::prefix_closed`] checks +//! them. +//! +//! Structural equality of two canonical maps is therefore observational +//! equality, which is what lets the model decide `AlgebraicStatus::Identity` +//! against `Element` (see `PathMap::beq_t`). +//! +//! # Deviations from the Lean model +//! +//! * The Lean model is purely functional: every operation returns a new `Zip`. +//! Here the mutating operations take `&mut self` and return only what the +//! corresponding `pathmap` method returns, so the trace front end can call the +//! two side by side. Where a law needs the prior state, it clones. +//! * `Zip` owns its `PathMap` (as in Lean, where a read zipper holds a snapshot +//! and a write zipper holds the live map). Operations that read one zipper +//! and write another take the source by reference. +//! * The proved theorems of `Spec.lean` §1 have no counterpart: they are proofs, +//! not tests. The checkable laws of §2 are in [`laws`]. +//! * `V: Clone` is required throughout; the Lean model is generic over any `V`. +//! +//! # The three front ends +//! +//! | binary | drives | +//! |---|---| +//! | `lean/.lake/build/bin/pathmap-oracle` | the Lean model (`lean/PathMapModel/Fuzz.lean`) | +//! | `target/release/pathmap_trace` | the real crate ([`crate::harness`]) | +//! | `target/release/reference` | the Rust model in this directory | +//! +//! `lean/differential.py` diffs any two of them. Model-against-model — +//! `--model` — is the acceptance test for this port: the two are independent +//! transcriptions of the same specification in different languages, so a diff +//! means one of them is wrong and nothing about the crate is in question. +//! +//! Once that holds, `target/release/in_process` drops the pipes: it runs +//! [`fuzz::run`] and [`crate::harness::run`] on the same bytes in one process +//! and compares the traces in memory. + +// The model is a specification: it defines the whole API surface whether or not +// any particular front end happens to call each item. `laws` and `map` are +// reached only from `check.rs`, under `cfg(test)`. +#![allow(dead_code)] + +pub mod basic; +pub mod pathmap; +pub mod zipper; +pub mod write; +pub mod map; +pub mod laws; + +#[cfg(test)] +mod check; + +pub mod fuzz; + +#[cfg(test)] +mod tests { + /// The model may not name the crate it is a model of. + /// + /// In the archive this was a property of the build: `examples/reference/` + /// was its own cargo target and `pathmap` was not in scope for it at all. + /// Here the model sits in a crate that does depend on `pathmap`, so the + /// invariant has to be asserted rather than obtained for free — otherwise a + /// single `use pathmap::utils::ByteMask` would quietly make the model share + /// the implementation's bugs, and the differential would agree for the wrong + /// reason. + #[test] + fn model_does_not_touch_the_crate() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/reference"); + let mut offenders = Vec::new(); + for entry in std::fs::read_dir(&dir).expect("src/reference must exist") { + let path = entry.expect("readable dir entry").path(); + if path.extension().and_then(|e| e.to_str()) != Some("rs") { + continue; + } + let src = std::fs::read_to_string(&path).expect("readable model file"); + for (n, line) in src.lines().enumerate() { + // Doc comments talk *about* the crate; code may not name it. + let code = line.trim_start(); + if code.starts_with("//") { + continue; + } + // The model's *own* map module is also called `pathmap`, so + // strip its path before looking for the crate's. + let code = code.replace("crate::reference::pathmap", ""); + if code.contains("pathmap::") || code.contains("extern crate pathmap") { + offenders.push(format!("{}:{}: {}", path.display(), n + 1, line.trim())); + } + } + } + assert!( + offenders.is_empty(), + "the reference model must not reach into the crate it models:\n{}", + offenders.join("\n") + ); + } +} diff --git a/differential/src/reference/pathmap.rs b/differential/src/reference/pathmap.rs new file mode 100644 index 00000000..9b3c5789 --- /dev/null +++ b/differential/src/reference/pathmap.rs @@ -0,0 +1,575 @@ +//! `PathMap.lean` — what a `pathmap` trie means. +//! +//! See the module docs in `main.rs` for why the representation is a flat +//! `BTreeMap, Option>` and not a prefix tree. + +use std::collections::BTreeMap; +use std::collections::btree_map::Entry as BEntry; +use std::ops::Bound; + +use crate::reference::basic::{ByteMask, ValOps, path}; + +// =========================================================================== +// PathMap.lean +// =========================================================================== + +/// What a map holds at a path — the whole answer, in one value. +/// +/// Named for `HashMap`/`BTreeMap`'s `Entry`, and standing in the same relation +/// to the storage: an element of [`PathMap::entries`] is a location that is +/// *there*, while an `Entry` also answers for a path that is not — Rust's +/// `Occupied` and `Vacant`. The difference is that a `pathmap` location has +/// **three** states rather than two, and that third one is the whole reason this +/// type exists. +/// +/// `create_path` produces it, `remove_val(false)` leaves it behind, and findings +/// 7, 8 and 15 are all about operations that mishandle it. Asking through +/// `Entry` rather than through `path_exists` and `val` separately means a +/// definition cannot quietly forget that case: the `match` will not compile +/// until it says what happens. [`Entry::Valued`] implies the location exists, so +/// the impossible combination — a value at a path that is not there — is +/// unrepresentable rather than merely untrue. +/// +/// Generic over the value slot so it can be returned borrowed (`Entry<&V>`) or +/// owned (`Entry`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Entry { + /// The path is not in the map. Rust's `Vacant`. + Absent, + /// The path is in the map but carries no value — a location `path_exists` + /// reports `true` for and `val` reports `None` for. + Bare, + /// The path is in the map and carries this value. Rust's `Occupied`. + Valued(V), +} + +impl Entry { + /// The value, if any — what `ZipperValues::val` reports. + pub fn val(self) -> Option { + match self { + Entry::Valued(v) => Some(v), + _ => None, + } + } + + /// Whether the location is in the map — what `Zipper::path_exists` reports. + /// + /// Holding a value entails being there: there is no constructor for the + /// combination, so the Lean model's `present_of_val` needs no counterpart. + pub fn present(&self) -> bool { + !matches!(self, Entry::Absent) + } + + /// Whether the location is there but holds nothing. + pub fn is_bare(&self) -> bool { + matches!(self, Entry::Bare) + } +} + +/// A `pathmap` trie: a finite path→value map *plus* the prefix-closed set of +/// existing locations. +/// +/// Everything rests on one observation. A `pathmap` trie is not just a +/// path→value map: `create_path` makes a location that exists *without* carrying +/// a value, and `remove_val(false)` leaves one behind. So a location, and a +/// value at that location, are separate facts, and both are recorded — in one +/// map rather than a value map beside a path set, so that a value cannot be +/// recorded at a location that does not exist. A `None` here is exactly a +/// dangling path. +/// +/// # Canonical form +/// +/// * sorted and duplicate-free — structural, from `BTreeMap`; +/// * prefix-closed, and containing the empty path — maintained by the +/// constructors, checked by [`laws::prefix_closed`]. +/// +/// Canonical form makes structural equality observational equality, which is +/// what lets [`Zip`] decide `AlgebraicStatus::Identity` against `Element`. +#[derive(Clone, Debug)] +pub struct PathMap { + /// Every location that exists, in depth-first order, each carrying its value + /// if it has one. + pub entries: BTreeMap, Option>, +} + +impl Default for PathMap { + fn default() -> Self { + Self::empty() + } +} + +impl PathMap { + // -- construction ------------------------------------------------------- + + /// The empty trie: the root exists, nothing else does. + pub fn empty() -> Self { + let mut entries = BTreeMap::new(); + entries.insert(Vec::new(), None); + PathMap { entries } + } + + /// Build a canonical map from raw components — the Lean model's `mk'`. + /// + /// `paths` is completed with every prefix of every raw path and of every key + /// carrying a value, so callers never have to maintain closure by hand. + /// + /// **Left-biased**, matching the Lean `dedupVals` and `pathmap`'s + /// `Identity(SELF_IDENT)` value instances: the *first* binding offered for a + /// key wins. (`BTreeMap::insert` keeps the last, which is why this goes + /// through `or_insert`.) + pub fn mk( + vals: impl IntoIterator, V)>, + paths: impl IntoIterator>, + ) -> Self { + let mut vs: BTreeMap, V> = BTreeMap::new(); + for (k, v) in vals { + if let BEntry::Vacant(e) = vs.entry(k) { + e.insert(v); + } + } + let mut t = PathMap::empty(); + for p in paths { + t.ensure_path(&p); + } + for (k, v) in vs { + t.ensure_path(&k); + t.entries.insert(k, Some(v)); + } + t + } + + /// Make `p` and every prefix of it exist, without disturbing any value + /// already recorded along the way. + fn ensure_path(&mut self, p: &[u8]) { + for n in 0..=p.len() { + self.entries.entry(p[..n].to_vec()).or_insert(None); + } + } + + // -- ranges ------------------------------------------------------------- + // + // The three shapes of query the specification needs. Locations sharing a + // prefix are contiguous in `BTreeMap` order, so each is a range plus a + // `take_while`, and each stops as soon as it leaves the subtree. + + /// Every location at or below `p`, in depth-first order. + pub(crate) fn at_or_below<'a>(&'a self, p: &'a [u8]) -> impl Iterator, &'a Option)> { + self.entries + .range::<[u8], _>((Bound::Included(p), Bound::Unbounded)) + .take_while(move |(q, _)| q.starts_with(p)) + } + + /// Every location strictly below `p`, in depth-first order. + pub(crate) fn strictly_below<'a>( + &'a self, + p: &'a [u8], + ) -> impl Iterator, &'a Option)> { + self.entries + .range::<[u8], _>((Bound::Excluded(p), Bound::Unbounded)) + .take_while(move |(q, _)| q.starts_with(p)) + } + + /// Every location strictly after `from` and at or below `within`, in + /// depth-first order. `within` must be a prefix of `from`. + pub(crate) fn after_within<'a>( + &'a self, + from: &'a [u8], + within: &'a [u8], + ) -> impl Iterator, &'a Option)> { + debug_assert!(from.starts_with(within)); + self.entries + .range::<[u8], _>((Bound::Excluded(from), Bound::Unbounded)) + .take_while(move |(q, _)| q.starts_with(within)) + } + + // -- observations ------------------------------------------------------- + // + // These are the entire observable interface of a map; every law is phrased + // in terms of them. + + /// What the map holds at `p`. + pub fn entry_at(&self, p: &[u8]) -> Entry<&V> { + match self.entries.get(p) { + None => Entry::Absent, + Some(None) => Entry::Bare, + Some(Some(v)) => Entry::Valued(v), + } + } + + /// `Zipper::val` / `PathMap::get_val_at`: the value at `p`, if any. + pub fn val_at(&self, p: &[u8]) -> Option<&V> { + self.entry_at(p).val() + } + + /// `Zipper::path_exists`: whether `p` is a location in the map. True for + /// dangling paths (locations with no value and no children). + pub fn path_exists(&self, p: &[u8]) -> bool { + self.entries.contains_key(p) + } + + /// Every existing location, in depth-first order. + pub fn paths(&self) -> impl Iterator> { + self.entries.keys() + } + + /// Every location that carries a value, with it, in depth-first order — + /// `PathMap::iter`. + pub fn vals(&self) -> impl Iterator, &V)> { + self.entries + .iter() + .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))) + } + + /// `Zipper::child_mask`: the bytes `b` for which `p ++ [b]` exists. + pub fn child_mask(&self, p: &[u8]) -> ByteMask { + ByteMask::of_list( + self.strictly_below(p) + .filter(|(q, _)| q.len() == p.len() + 1) + .map(|(q, _)| q[p.len()]), + ) + } + + /// `Zipper::child_count`. + pub fn child_count(&self, p: &[u8]) -> usize { + self.child_mask(p).count_bits() + } + + /// `ZipperMoving::val_count`: values at and below `p`. + pub fn val_count(&self, p: &[u8]) -> usize { + self.at_or_below(p).filter(|(_, v)| v.is_some()).count() + } + + /// The existing locations at or below `p`, in depth-first order. + pub fn paths_below(&self, p: &[u8]) -> Vec> { + self.at_or_below(p).map(|(q, _)| q.clone()).collect() + } + + /// `TrieNode::node_is_empty` applied to the node *below* `p`: no descendants + /// at all. + pub fn below_is_empty(&self, p: &[u8]) -> bool { + self.strictly_below(p).next().is_none() + } + + /// `PathMap::is_empty`: no values and an empty root node. + pub fn is_empty_map(&self) -> bool { + self.vals().next().is_none() && self.below_is_empty(&[]) + } + + /// Whether the map is in canonical form. + /// + /// Sortedness and freedom from duplicates are structural — `BTreeMap` cannot + /// represent a violation — so what is left to check is that the root exists + /// and that the set of locations is prefix-closed. Every constructor here + /// establishes both; this is the assertion a harness can make after each + /// operation to catch one that does not. + pub fn is_canonical(&self) -> bool { + self.path_exists(&[]) + && self + .entries + .keys() + .all(|q| (0..q.len()).all(|n| self.entries.contains_key(&q[..n]))) + } + + /// Structural (hence observational) equality. Used to decide + /// `AlgebraicStatus::Identity` — `pathmap` reports `Identity(SELF_IDENT)` + /// exactly when the operation's output equals `self`. + /// + /// Values are compared with [`ValOps::beq`], not with `PartialEq`: the + /// notion of equality that decides the status is the value type's own. + pub fn beq_t(&self, ops: &impl ValOps, other: &PathMap) -> bool { + self.entries.len() == other.entries.len() + && self + .entries + .iter() + .zip(other.entries.iter()) + .all(|((ka, va), (kb, vb))| { + ka == kb + && match (va, vb) { + (None, None) => true, + (Some(x), Some(y)) => ops.beq(x, y), + _ => false, + } + }) + } + + // -- sub-maps and grafting --------------------------------------------- + + /// The submap rooted at `p`, **including** the value at `p` as its root + /// value. This is `make_map` / `take_map` under the default + /// `graft_root_vals` feature, and also what a zipper rooted at `p` sees. + /// Yields [`PathMap::empty`] when `p` does not exist. + pub fn subtrie(&self, p: &[u8]) -> PathMap { + let mut entries: BTreeMap, Option> = self + .at_or_below(p) + .map(|(q, v)| (q[p.len()..].to_vec(), v.clone())) + .collect(); + entries.entry(Vec::new()).or_insert(None); + PathMap { entries } + } + + /// Remove everything at and below `p`; `p` itself stops existing. + /// + /// The root always survives: a map with no locations at all is not + /// representable, and `mk'` in the Lean model re-adds it the same way. + pub fn remove_at(&mut self, p: &[u8]) { + let doomed: Vec> = self.at_or_below(p).map(|(q, _)| q.clone()).collect(); + for q in doomed { + self.entries.remove(&q); + } + self.entries.entry(Vec::new()).or_insert(None); + } + + /// Remove everything strictly below `p`; `p` itself, and its value, are + /// untouched. This is `ZipperWriting::remove_branches` without pruning. + pub fn remove_below(&mut self, p: &[u8]) { + let doomed: Vec> = self.strictly_below(p).map(|(q, _)| q.clone()).collect(); + for q in doomed { + self.entries.remove(&q); + } + } + + /// Replace everything strictly below `p` with the strictly-below part of `s`. + /// + /// The value at `p` is *not* touched (`graft_internal` only ever replaces a + /// node); `p` is created iff `s` has any non-root content, mirroring the fact + /// that grafting an empty node neither creates nor destroys the location. + pub fn graft_below(&mut self, p: &[u8], s: &PathMap) { + self.remove_below(p); + let mut any = false; + for (q, v) in s.entries.iter() { + if q.is_empty() { + continue; + } + if !any { + // The first non-root entry is what creates `p` (and its own + // ancestors); an empty source leaves the location alone. + self.ensure_path(p); + any = true; + } + self.entries.insert(path::cat(p, q), v.clone()); + } + } + + // -- point updates ------------------------------------------------------ + + /// `ZipperWriting::set_val` / `PathMap::set_val_at`. Creates `p` if needed. + /// Returns the replaced value. + pub fn set_val(&mut self, p: &[u8], v: V) -> Option { + self.ensure_path(p); + self.entries.insert(p.to_vec(), Some(v)).flatten() + } + + /// `ZipperWriting::remove_val` *without* pruning: the location survives as a + /// dangling path. Returns the removed value. Never creates anything. + pub fn remove_val(&mut self, p: &[u8]) -> Option { + self.entries.get_mut(p).and_then(|slot| slot.take()) + } + + /// `ZipperWriting::create_path`: make `p` exist as a dangling path. + pub fn add_path(&mut self, p: &[u8]) { + self.ensure_path(p); + } + + // -- pruning ------------------------------------------------------------ + + /// Is `p` a dangling tip — an existing location with neither value nor + /// children? + pub fn is_dangling_tip(&self, p: &[u8]) -> bool { + self.entry_at(p).is_bare() && self.below_is_empty(p) + } + + /// The number of bytes `prune_path` would remove at focus `p` for a zipper + /// whose root sits at depth `root_len`. `0` means "nothing to prune". + /// + /// The chain is removed back to the deepest strict ancestor that must be + /// kept: one carrying a value, one that branches, or the one at the stop + /// depth. The ancestor at `root_len` always qualifies, so there is always an + /// answer, and `max` makes "never prunes above the stop depth" syntactic. + pub fn prune_count(&self, root_len: usize, p: &[u8]) -> usize { + if p.len() <= root_len || !self.is_dangling_tip(p) { + return 0; + } + let a = path::deepest_proper_prefix(p, |a| { + a.len() <= root_len || self.val_at(a).is_some() || self.child_count(a) >= 2 + }) + .map(|a| a.len()) + .unwrap_or(0); + p.len() - root_len.max(a) + } + + /// `ZipperWriting::prune_path`: returns the number of bytes removed. + pub fn prune_path(&mut self, root_len: usize, p: &[u8]) -> usize { + let n = self.prune_count(root_len, p); + if n != 0 { + self.remove_at(&p[..p.len() - n + 1]); + } + n + } +} + +// --------------------------------------------------------------------------- +// PathMap.lean — algebraic operations +// --------------------------------------------------------------------------- +// +// These lift `pathmap`'s node-level `pjoin` / `pmeet` / `psubtract` / +// `prestrict` to whole maps. The interesting content is *which locations +// survive*, since `pathmap` drops any node that ends up empty. + +impl PathMap { + /// `Option::pjoin` from `src/ring.rs`. + fn join_val(ops: &impl ValOps, a: Option<&V>, b: Option<&V>) -> Option { + match (a, b) { + (None, b) => b.cloned(), + (Some(a), None) => Some(a.clone()), + (Some(a), Some(b)) => ops.pjoin(a, b).resolve(a, b), + } + } + + /// `Option::pmeet`: a value survives only where *both* sides have one. + fn meet_val(ops: &impl ValOps, a: Option<&V>, b: Option<&V>) -> Option { + match (a, b) { + (Some(a), Some(b)) => ops.pmeet(a, b).resolve(a, b), + _ => None, + } + } + + /// `Option::psubtract`. + fn sub_val(ops: &impl ValOps, a: Option<&V>, b: Option<&V>) -> Option { + match (a, b) { + (None, _) => None, + (Some(a), None) => Some(a.clone()), + (Some(a), Some(b)) => ops.psub(a, b).resolve(a, b), + } + } + + /// Join (union). Every location of either side survives; colliding values + /// are combined with [`ValOps::pjoin`]. + pub fn join(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + let mut keys: Vec<&Vec> = a.vals().map(|(k, _)| k).collect(); + keys.extend(b.vals().map(|(k, _)| k)); + keys.sort_unstable(); + keys.dedup(); + let vals = keys.into_iter().filter_map(|k| { + Self::join_val(ops, a.val_at(k), b.val_at(k)).map(|v| (k.clone(), v)) + }); + // Collected first: `mk` takes the value list by value, and the path + // iterators borrow `a` and `b` at the same time. + let vals: Vec<(Vec, V)> = vals.collect(); + PathMap::mk(vals, a.paths().chain(b.paths()).cloned()) + } + + /// Meet (intersection). A location survives only if it lies on the way to a + /// surviving value, so dangling paths never survive a meet. + pub fn meet(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + let vals: Vec<(Vec, V)> = a + .vals() + .filter_map(|(k, av)| { + Self::meet_val(ops, Some(av), b.val_at(k)).map(|v| (k.clone(), v)) + }) + .collect(); + PathMap::mk(vals, std::iter::empty()) + } + + /// Subtract. + /// + /// Two rules interact here. Where `b` has no node at all, `a`'s subtree is + /// kept verbatim — *including its dangling paths*. Where `b` does have a + /// node, only locations leading to a surviving value are kept. `pathmap` + /// gets this from `psubtract_dyn` short-circuiting on absent children; the + /// model reproduces it by splitting on whether the location leaves `b`. + pub fn sub(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + let surviving: Vec<(Vec, V)> = a + .vals() + .filter_map(|(k, av)| { + Self::sub_val(ops, Some(av), b.val_at(k)).map(|v| (k.clone(), v)) + }) + .collect(); + // The locations at which `a` leaves `b` entirely: below one of these, + // `psubtract_dyn` never looks again and `a` is copied verbatim. + let untouched: Vec<&Vec> = a + .paths() + .filter(|q| !q.is_empty() && !b.path_exists(q) && b.path_exists(&q[..q.len() - 1])) + .collect(); + let kept: Vec> = a + .paths() + .filter(|q| { + surviving.iter().any(|(k, _)| k.starts_with(q)) + || untouched.iter().any(|u| q.starts_with(u.as_slice())) + }) + .cloned() + .collect(); + PathMap::mk(surviving, kept) + } + + /// Is `q` *validated* by `b` — does some non-empty prefix of `q` carry a + /// value in `b`? + /// + /// This is the node-level reading of `prestrict`: a node has no root value, + /// so the empty prefix never validates. `PathMap::restrict` adds the empty + /// prefix back in (see [`map::restrict`]), which is why the map-level and + /// zipper-level operations disagree when the source has a root value. + pub fn validated_by(b: &PathMap, q: &[u8]) -> bool { + (1..=q.len()).any(|i| b.val_at(&q[..i]).is_some()) + } + + /// `prestrict` at node level: keep the locations of `a` validated by `b`. + /// Once a location is validated, everything below it is kept verbatim. + pub fn restrict_below_root(a: &PathMap, b: &PathMap) -> PathMap { + let vals: Vec<(Vec, V)> = a + .vals() + .filter(|(k, _)| Self::validated_by(b, k)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let paths: Vec> = a + .paths() + .filter(|q| Self::validated_by(b, q)) + .cloned() + .collect(); + PathMap::mk(vals, paths) + } + + // -- path surgery ------------------------------------------------------- + + /// `ZipperWriting::insert_prefix`: put `k` in front of every path below the + /// root. The root value is dropped — a node has nowhere to keep one. + pub fn insert_prefix_below(&self, k: &[u8]) -> PathMap { + let vals: Vec<(Vec, V)> = self + .vals() + .filter(|(q, _)| !q.is_empty()) + .map(|(q, v)| (path::cat(k, q), v.clone())) + .collect(); + let paths: Vec> = self + .paths() + .filter(|q| !q.is_empty()) + .map(|q| path::cat(k, q)) + .collect(); + PathMap::mk(vals, paths) + } + + /// The existing locations exactly `k` bytes below the root, in depth-first + /// order. + pub fn k_paths(&self, k: usize) -> Vec> { + self.paths().filter(|q| q.len() == k).cloned().collect() + } + + /// `drop_head` / `ZipperWriting::join_k_path_into` at node level: strip the + /// first `k` bytes from every path and join the results. + /// + /// Values sitting at depth *exactly* `k` are **discarded** — the joined node + /// has nowhere to put a root value. (`meet_k_path_into` keeps them, because + /// it routes through `take_map`/`graft_map`, which do carry root values. The + /// asymmetry is real.) + pub fn drop_head(&self, ops: &impl ValOps, k: usize) -> PathMap { + if k == 0 { + return self.clone(); + } + let mut acc = PathMap::empty(); + for q in self.k_paths(k) { + let mut m = self.subtrie(&q); + m.remove_val(&[]); + acc = PathMap::join(ops, &acc, &m); + } + acc + } +} + diff --git a/differential/src/reference/write.rs b/differential/src/reference/write.rs new file mode 100644 index 00000000..0bd5564b --- /dev/null +++ b/differential/src/reference/write.rs @@ -0,0 +1,705 @@ +//! `Write.lean` — the write zipper: everything that mutates `Zip::trie` +//! through the focus `root ++ path`. + +use crate::reference::basic::{AlgStatus, ByteMask, ValOps, path}; +use crate::reference::pathmap::{Entry, PathMap}; +use crate::reference::zipper::Zip; + +// =========================================================================== +// Write.lean — the write zipper +// =========================================================================== +// +// Everything here mutates `Zip::trie` through the focus `root ++ path`. Two +// invariants shape the whole API and are worth stating up front: +// +// 1. **A node is what lies strictly below a location.** `get_focus`, +// `graft_internal`, and every `*_dyn` algebraic primitive operate on nodes, so +// they never see or touch the value *at* the focus. Operations that do affect +// the focus value (`graft`, `graft_map`, `make_map`, `take_map`, +// `join_map_into`, `meet_into`, `subtract_into`) do it in a separate step — +// this is the `graft_root_vals` cargo feature, which is on by default and +// which the model assumes throughout. Note the resulting asymmetry: `graft` +// adopts the source's focus value but `join_into` does **not** join focus +// values. +// +// 2. **Pruning is opt-in and local.** A write leaves dangling paths behind +// unless `prune` is passed, and even then `prune_path` only fires when the +// focus is a dangling tip. + +impl Zip { + // -- values at the focus ------------------------------------------------ + + /// `ZipperWriting::get_val_mut` — same observation as [`Zip::val`], mutably. + /// Returns `Some` exactly when `val` does; it never creates anything. + pub fn get_val_mut(&mut self) -> Option<&mut V> { + let f = self.focus(); + self.trie.entries.get_mut(&f).and_then(|slot| slot.as_mut()) + } + + /// `ZipperWriting::set_val`: sets the value at the focus, **creating the + /// path** if it did not exist. Returns the replaced value. + pub fn set_val(&mut self, v: V) -> Option { + let f = self.focus(); + self.trie.set_val(&f, v) + } + + /// Writing through the reference `get_val_mut` returns. + /// + /// Specified as: when there is a value, this is `set_val`; when there is not, + /// it is a no-op — in particular it must **not** create the path, which is + /// what separates it from `set_val`. + pub fn get_val_mut_write(&mut self, v: V) -> Option { + match self.get_val_mut() { + Some(slot) => Some(std::mem::replace(slot, v)), + None => None, + } + } + + /// `ZipperWriting::get_val_or_set_mut`: the value at the focus, inserting + /// `default` if there is none. Inserting creates the path, as `set_val` does. + pub fn get_val_or_set_mut(&mut self, d: V) -> V { + let (v, _) = self.get_val_or_set_mut_with(d); + v + } + + /// `ZipperWriting::get_val_or_set_mut_with`: as above, but the value is + /// produced by a closure. + /// + /// The documented contract is that the closure supplies the value "if no + /// value exists", so it must be called **exactly when the focus has no + /// value** — calling it otherwise would be observable to any caller whose + /// closure has a side effect (allocating, taking a lock, bumping a counter). + /// The second component of the result records whether it ran, so a harness + /// can compare that too. + pub fn get_val_or_set_mut_with(&mut self, d: V) -> (V, bool) { + match self.val() { + Some(v) => (v.clone(), false), + None => { + self.set_val(d.clone()); + (d, true) + } + } + } + + // -- pruning ------------------------------------------------------------ + + /// `ZipperWriting::prune_path`: delete the dangling chain ending at the + /// focus, stopping at the first location above it that carries a value or + /// branches. The focus does **not** move. + /// + /// Two things about this differ from the doc comment on + /// `ZipperWriting::prune_path`, both verified against `pathmap` 0.3.1: + /// + /// * **It prunes above the zipper's root.** The doc says "This method cannot + /// prune the trie above the zipper's root", but a write zipper rooted at + /// `ab` whose focus is a dangling tip deletes `a` and `ab` too, right up to + /// the nearest branch or value in the *whole map*. The model therefore + /// passes `0`, not `self.root.len()`, as the stop depth. + /// * **The returned count is not well-defined** when the zipper's root is + /// non-empty. `prune_path` returns `max(node_pruned_bytes, + /// trie_pruned_bytes)`, and `node_pruned_bytes` depends on where the + /// internal node holding the focus happens to begin. Empirically, a + /// 40-byte dangling chain under a zipper rooted at depth 5 reports 40 + /// (absolute) while a 100-byte one reports 95 (relative). The *effect* is + /// the same in both cases; only the number differs. The model reports the + /// absolute count, and a harness should compare the count only for zippers + /// rooted at the map root. + pub fn prune_path(&mut self) -> usize { + let f = self.focus(); + self.trie.prune_path(0, &f) + } + + /// `ZipperWriting::prune_ascend`: `prune_path` followed by ascending that far. + pub fn prune_ascend(&mut self) -> usize { + let n = self.prune_path(); + self.ascend(n); + n + } + + /// `ZipperWriting::remove_val`: removes the value, leaving the location as a + /// dangling path unless `prune` reclaims it. + /// + /// Pruning only happens when a value was actually removed: `remove_val` + /// returns early on the `None` branch, so `remove_val(true)` at a location + /// with no value leaves any dangling path in place. + pub fn remove_val(&mut self, prune: bool) -> Option { + let valued = match self.entry() { + // Nothing to remove. Note the `prune` flag does not fire here: + // `remove_val` returns early on this branch, so a dangling path is + // left in place. + Entry::Absent | Entry::Bare => false, + Entry::Valued(_) => true, + }; + if !valued { + return None; + } + let f = self.focus(); + let v = self.trie.remove_val(&f); + if prune { + self.trie.prune_path(0, &f); + } + v + } + + /// `ZipperWriting::create_path`: make the focus exist as a dangling path. + /// Returns whether new bytes were created. + /// + /// The guard is on the *absolute* focus, not on `at_root`: `create_path` + /// bails out when there is no key left to create, which is the map root, not + /// the zipper root. A zipper rooted at `ab` whose root does not yet exist + /// will happily create it. + pub fn create_path(&mut self) -> bool { + let f = self.focus(); + if f.is_empty() { + return false; + } + let create = match self.trie.entry_at(&f) { + Entry::Absent => true, + // Already there, with or without a value: nothing to create, and in + // particular an existing value is never disturbed. + Entry::Bare | Entry::Valued(_) => false, + }; + if create { + self.trie.add_path(&f); + } + create + } + + // -- removing subtries -------------------------------------------------- + + /// `ZipperWriting::remove_branches`: delete everything strictly below the + /// focus. The value at the focus survives. Returns whether anything was + /// removed. + pub fn remove_branches(&mut self, prune: bool) -> bool { + let f = self.focus(); + let removed = !self.trie.below_is_empty(&f); + self.trie.remove_below(&f); + if prune { + self.trie.prune_path(0, &f); + } + removed + } + + /// `ZipperWriting::remove_unmasked_branches`: keep only the child bytes set + /// in `mask`; delete the rest along with their subtries. + pub fn remove_unmasked_branches(&mut self, mask: &ByteMask, prune: bool) { + let f = self.focus(); + let doomed: Vec = self + .child_mask() + .bytes() + .iter() + .copied() + .filter(|&b| !mask.contains(b)) + .collect(); + for b in doomed { + self.trie.remove_at(&path::push(&f, b)); + } + if prune { + self.trie.prune_path(0, &f); + } + } + + // -- grafting ----------------------------------------------------------- + + /// Replace the submap at the focus with `m`, treating `m` as a whole + /// `PathMap`: its root value becomes the focus value (or clears it), and its + /// branches become the focus's branches. This is + /// `ZipperWriting::graft_map`. + pub fn graft_map(&mut self, m: &PathMap) { + let f = self.focus(); + graft_map_at(&mut self.trie, &f, m); + } + + /// `ZipperWriting::graft`: graft the submap at `src`'s focus, root value + /// included. + pub fn graft(&mut self, src: &Zip) { + self.graft_map(&src.make_map()); + } + + /// `ZipperWriting::graft_src_at`: graft the submap `k` bytes below `src`'s + /// focus. + pub fn graft_src_at(&mut self, src: &Zip, k: &[u8]) { + let p = path::cat(&src.focus(), k); + self.graft_map(&src.trie.subtrie(&p)); + } + + /// `ZipperWriting::graft_masked_branches`: graft the source's child branches + /// for each byte set in `mask`. + /// + /// Each set bit is a `graft_src_at` of the source's corresponding child, so + /// the child's *value* travels with it, and a set bit whose branch is absent + /// from the source leaves that branch absent here — grafting nothing removes. + /// With `remove_unset`, branches for clear bits are removed first, so + /// `child_mask` afterwards is a subset of `mask`; without it they are left + /// alone. + /// + /// `WriteZipperCore` overrides the trait's default implementation with a + /// native one, so this op is really comparing two implementations of the same + /// contract. + pub fn graft_masked_branches(&mut self, src: &Zip, mask: &ByteMask, remove_unset: bool) { + if remove_unset { + self.remove_branches(false); + } + let f = self.focus(); + let sf = src.focus(); + for &b in mask.bytes() { + let m = src.trie.subtrie(&path::push(&sf, b)); + graft_map_at(&mut self.trie, &path::push(&f, b), &m); + } + } + + /// `ZipperWriting::graft_child_maps`: as above, but the branches come from an + /// explicit list of maps rather than from a source zipper. + /// + /// Feeding it the source's own child submaps must therefore produce exactly + /// what [`Zip::graft_masked_branches`] produces from that source — a harness + /// should check the two against each other as well as against this + /// definition. + pub fn graft_child_maps(&mut self, maps: &[(ByteMask, PathMap)], remove_unset: bool) { + if remove_unset { + self.remove_branches(false); + } + let f = self.focus(); + for (mask, m) in maps { + if let [b] = mask.bytes() { + graft_map_at(&mut self.trie, &path::push(&f, *b), m); + } + } + } + + /// `ZipperWriting::take_map`: remove the submap at the focus (value included) + /// and return it as a `PathMap`. Returns `None` when there was nothing to + /// take. + pub fn take_map(&mut self, prune: bool) -> Option> { + let f = self.focus(); + let rv = self.trie.remove_val(&f); + if prune { + self.trie.prune_path(0, &f); + } + let below = self.focus_node(); + self.trie.remove_below(&f); + if prune { + self.trie.prune_path(0, &f); + } + let nothing = below.is_empty_map() && rv.is_none(); + let mut taken = below; + if let Some(v) = rv { + taken.set_val(&[], v); + } + if nothing { None } else { Some(taken) } + } + + // -- path surgery ------------------------------------------------------- + + /// `ZipperWriting::insert_prefix`: put `pre` in front of every path below the + /// focus. The focus value is untouched. Returns `false` at a location with + /// no descendants. + /// + /// BUG (`pathmap` 0.3.1): with an **empty** prefix this should be the + /// identity, but `make_parents_in(b"", node)` discards the node — the submap + /// below the focus is destroyed and `true` is still returned. The model + /// specifies the identity; a differential harness should skip + /// `insert_prefix("")` so the known divergence does not mask others. + pub fn insert_prefix(&mut self, pre: &[u8]) -> bool { + if self.focus_node_is_empty() { + return false; + } + let f = self.focus(); + let node = self.focus_node().insert_prefix_below(pre); + self.trie.graft_below(&f, &node); + true + } + + /// `ZipperWriting::remove_prefix`: lift the submap below the focus up by `n` + /// bytes, replacing whatever was below the new (ascended) focus. Returns + /// whether the full `n` bytes could be ascended. + /// + /// Note the value at the old focus is *not* carried up — it belonged to the + /// parent cell, not to the node that gets moved. + pub fn remove_prefix(&mut self, n: usize) -> bool { + let below = self.focus_node(); + // `ascend` reports how far it got, so "were all `n` bytes removed" is a + // comparison rather than the flag it used to return directly. + let ascended = self.ascend(n); + let f = self.focus(); + self.trie.graft_below(&f, &below); + ascended == n + } +} + +/// `graft_map` against a bare map, factored out so the three grafting entry +/// points cannot drift: replace everything below `p` with `m`'s branches, then +/// let `m`'s root value set or clear the value at `p`. +fn graft_map_at(t: &mut PathMap, p: &[u8], m: &PathMap) { + t.graft_below(p, m); + match m.val_at(&[]) { + Some(v) => { + t.set_val(p, v.clone()); + } + None => { + t.remove_val(p); + } + } +} + +// --------------------------------------------------------------------------- +// Write.lean — algebraic operations +// --------------------------------------------------------------------------- +// +// `AlgebraicStatus` is decided structurally: `Identity` exactly when the output +// equals the input (`Identity(SELF_IDENT)`), `None` when the output is empty, and +// `Element` otherwise. `Identity(COUNTER_IDENT)` — the output equals the +// *source* — is reported as `Element` by `pathmap`, and the model agrees because +// the output still differs from `self`. + +impl Zip { + /// The status of replacing `before` with `after`. + fn node_status(ops: &impl ValOps, before: &PathMap, after: &PathMap) -> AlgStatus { + if after.is_empty_map() { + AlgStatus::None + } else if after.beq_t(ops, before) { + AlgStatus::Identity + } else { + AlgStatus::Element + } + } + + /// Write `r` below the focus, pruning afterwards when the node annihilated + /// and the caller asked for it. The shape shared by `meet_into`, + /// `subtract_into` and their kin. + fn write_node(&mut self, st: AlgStatus, r: &PathMap, prune: bool) { + if st == AlgStatus::Identity { + return; + } + let f = self.focus(); + self.trie.graft_below(&f, r); + if st == AlgStatus::None && prune { + self.trie.prune_path(0, &f); + } + } + + /// `ZipperWriting::join_into`: union the source's submap into the focus's. + /// + /// The focus **values are not joined** — only the nodes below the focus are. + /// (The map-consuming variant [`Zip::join_map_into`] *does* join root values.) + pub fn join_into(&mut self, ops: &impl ValOps, src: &Zip) -> AlgStatus { + let self_b = self.focus_node(); + let src_b = src.focus_node(); + if src_b.is_empty_map() { + return if self_b.is_empty_map() { AlgStatus::None } else { AlgStatus::Identity }; + } + let r = PathMap::join(ops, &self_b, &src_b); + if r.beq_t(ops, &self_b) { + AlgStatus::Identity + } else { + let f = self.focus(); + self.trie.graft_below(&f, &r); + AlgStatus::Element + } + } + + /// `ZipperWriting::join_map_into`: union a consumed `PathMap` into the focus. + /// + /// Unlike [`Zip::join_into`] this *does* join the map's root value into the + /// focus value. It also short-circuits: when the map has no root node, the + /// node status is returned directly and the value status computed above is + /// discarded — even though the value has already been written. + pub fn join_map_into(&mut self, ops: &impl ValOps, m: &PathMap) -> AlgStatus { + let (val_status, val_was_none) = match (self.val().cloned(), m.val_at(&[]).cloned()) { + (Some(sv), Some(mv)) => { + let r = ops.pjoin(&sv, &mv); + let st = AlgStatus::of_val_res(&r); + match r.resolve(&sv, &mv) { + Some(v) => { + self.set_val(v); + } + None => { + self.remove_val(false); + } + } + (st, false) + } + (None, Some(mv)) => { + self.set_val(mv); + (AlgStatus::Element, true) + } + (Some(_), None) => (AlgStatus::Identity, false), + (None, None) => (AlgStatus::None, true), + }; + let mut src_b = m.clone(); + src_b.remove_val(&[]); + if src_b.is_empty_map() { + // Short-circuit, and note the asymmetry with `join_into`: this branch + // tests `self.get_focus().is_none()` (does a node exist at all?), not + // `node_is_empty()`. So a *bare* focus reports `Identity` here, where + // `join_into` on the same state reports `None`. + return match self.entry() { + Entry::Bare | Entry::Valued(_) => AlgStatus::Identity, + Entry::Absent => AlgStatus::None, + }; + } + let self_b = self.focus_node(); + let r = PathMap::join(ops, &self_b, &src_b); + let node_status = if r.beq_t(ops, &self_b) { AlgStatus::Identity } else { AlgStatus::Element }; + if node_status != AlgStatus::Identity { + let f = self.focus(); + self.trie.graft_below(&f, &r); + } + AlgStatus::merge(node_status, val_status, true, val_was_none) + } + + /// `ZipperWriting::join_into_take`: like [`Zip::join_into`], but the source + /// submap is removed from the source zipper's map. + pub fn join_into_take( + &mut self, + ops: &impl ValOps, + src: &mut Zip, + prune: bool, + ) -> AlgStatus { + let src_b = src.focus_node(); + let sf = src.focus(); + src.trie.remove_below(&sf); + if prune { + src.trie.prune_path(0, &sf); + } + let self_b = self.focus_node(); + if src_b.is_empty_map() { + return if self_b.is_empty_map() { AlgStatus::None } else { AlgStatus::Identity }; + } + let r = PathMap::join(ops, &self_b, &src_b); + let st = if r.beq_t(ops, &self_b) { AlgStatus::Identity } else { AlgStatus::Element }; + let f = self.focus(); + self.trie.graft_below(&f, &r); + st + } + + /// `ZipperWriting::meet_into`: intersect the focus's submap with the + /// source's. + /// + /// The value step runs first and can prune the focus out from under the node + /// step. A meet drops every dangling path, since a location only survives if + /// it leads to a surviving value. + pub fn meet_into(&mut self, ops: &impl ValOps, src: &Zip, prune: bool) -> AlgStatus { + let (val_status, val_was_none) = match (self.val().cloned(), src.val().cloned()) { + (Some(sv), Some(ov)) => { + let r = ops.pmeet(&sv, &ov); + let st = AlgStatus::of_val_res(&r); + match r.resolve(&sv, &ov) { + Some(v) => { + self.set_val(v); + } + None => { + self.remove_val(prune); + } + } + (st, false) + } + (None, Some(_)) => (AlgStatus::None, true), + (Some(_), None) => { + self.remove_val(prune); + (AlgStatus::None, false) + } + (None, None) => (AlgStatus::None, true), + }; + let self_b = self.focus_node(); + let src_b = src.focus_node(); + if self_b.is_empty_map() { + return AlgStatus::merge(AlgStatus::None, val_status, true, val_was_none); + } + if src_b.is_empty_map() { + let f = self.focus(); + self.trie.remove_below(&f); + if prune { + self.trie.prune_path(0, &f); + } + return AlgStatus::merge(AlgStatus::None, val_status, false, val_was_none); + } + let r = PathMap::meet(ops, &self_b, &src_b); + let st = Self::node_status(ops, &self_b, &r); + self.write_node(st, &r, prune); + AlgStatus::merge(st, val_status, false, val_was_none) + } + + /// `ZipperWriting::subtract_into`: remove the source's submap from the + /// focus's. + /// + /// Where the source has no node at all, `self`'s subtree survives untouched — + /// dangling paths included. Where it does, only locations leading to a + /// surviving value are kept. + pub fn subtract_into(&mut self, ops: &impl ValOps, src: &Zip, prune: bool) -> AlgStatus { + let (val_status, val_was_none) = match (self.val().cloned(), src.val().cloned()) { + (Some(sv), Some(ov)) => { + let r = ops.psub(&sv, &ov); + let st = AlgStatus::of_val_res(&r); + match r.resolve(&sv, &ov) { + Some(v) => { + self.set_val(v); + } + None => { + self.remove_val(prune); + } + } + (st, false) + } + (None, Some(_)) => (AlgStatus::None, true), + (Some(_), None) => (AlgStatus::Identity, false), + (None, None) => (AlgStatus::None, true), + }; + let self_b = self.focus_node(); + let src_b = src.focus_node(); + if src_b.is_empty_map() { + let node = if self_b.is_empty_map() { AlgStatus::None } else { AlgStatus::Identity }; + return AlgStatus::merge(node, val_status, self_b.is_empty_map(), val_was_none); + } + if self_b.is_empty_map() { + return AlgStatus::merge(AlgStatus::None, val_status, true, val_was_none); + } + let r = PathMap::sub(ops, &self_b, &src_b); + let st = Self::node_status(ops, &self_b, &r); + self.write_node(st, &r, prune); + AlgStatus::merge(st, val_status, false, val_was_none) + } + + /// `ZipperWriting::meet_2`: meet two *source* submaps and write the result at + /// the focus. + /// + /// Two things separate this from [`Zip::meet_into`]. It does not consult + /// what is already at the focus, so — as the implementation notes — it never + /// reports `Identity`, only `Element` or `None`. And it works on nodes, so + /// neither source's focus value is consulted and the focus value here is left + /// untouched. + pub fn meet_2(&mut self, ops: &impl ValOps, a: &Zip, b: &Zip) -> AlgStatus { + let an = a.focus_node(); + let bn = b.focus_node(); + let f = self.focus(); + if an.is_empty_map() || bn.is_empty_map() { + self.trie.remove_below(&f); + return AlgStatus::None; + } + let r = PathMap::meet(ops, &an, &bn); + if r.is_empty_map() { + self.trie.remove_below(&f); + AlgStatus::None + } else { + self.trie.graft_below(&f, &r); + AlgStatus::Element + } + } + + /// `ZipperWriting::restrict`: keep only the paths below the focus that are + /// prefixed by a path to a value in the source's submap. + /// + /// The empty prefix does **not** validate here: the source's *focus value* is + /// invisible to a node-level `prestrict`. [`map::restrict`] does consult the + /// root value, so the two disagree exactly when the source has a value at its + /// focus. The focus value of `self` is never touched. + pub fn restrict(&mut self, ops: &impl ValOps, src: &Zip) -> AlgStatus { + let src_b = src.focus_node(); + let self_b = self.focus_node(); + if src_b.is_empty_map() { + let f = self.focus(); + self.trie.remove_below(&f); + return AlgStatus::None; + } + if self_b.is_empty_map() { + return AlgStatus::None; + } + let r = PathMap::restrict_below_root(&self_b, &src_b); + let st = Self::node_status(ops, &self_b, &r); + if st == AlgStatus::Identity { + return AlgStatus::Identity; + } + let f = self.focus(); + self.trie.graft_below(&f, &r); + st + } + + /// `ZipperWriting::restricting`: the mirror image — fill in `self`'s "stem" + /// paths with the source's submaps. `self`'s submap is replaced by the + /// source's, restricted by the paths to values in `self`. + /// + /// Returns `false`, leaving `self` untouched, when either side has nothing + /// below its focus. Note this is decided by `get_focus().is_none()`, which + /// is true when there is no node below the focus but *false* when an empty + /// node happens to have been materialised there by `create_path` or + /// `remove_val` — see FINDINGS.md #8. The model specifies the common case. + pub fn restricting(&mut self, src: &Zip) -> bool { + if src.focus_node_is_empty() || self.focus_node_is_empty() { + return false; + } + let r = PathMap::restrict_below_root(&src.focus_node(), &self.focus_node()); + let f = self.focus(); + self.trie.graft_below(&f, &r); + true + } + + // -- collapsing path segments ------------------------------------------- + + /// `ZipperWriting::join_k_path_into` (a.k.a. `drop_head`): strip the leading + /// `k` bytes from every path below the focus and join the results. + /// + /// Values sitting at depth exactly `k` are **lost**: the joined node has no + /// root value slot. Returns whether anything survives below the focus. + /// + /// BUG (`pathmap` 0.3.1): `k = 0` should be the identity — dropping no bytes + /// — but `drop_head_dyn(0)` collapses the submap instead. On + /// `{[] ↦ 0, [0] ↦ 0, [0,0] ↦ 0, [1,0] ↦ 0}` it leaves `{[] ↦ 0, [0] ↦ 0}`. + /// The model specifies the identity; a harness should skip `k = 0`. + pub fn join_k_path_into(&mut self, ops: &impl ValOps, k: usize, prune: bool) -> bool { + let below = self.focus_node(); + let res = if below.is_empty_map() { + false + } else { + let r = below.drop_head(ops, k); + let survives = !r.is_empty_map(); + let f = self.focus(); + self.trie.graft_below(&f, &r); + survives + }; + if prune && !res { + self.prune_path(); + } + res + } + + /// `meet_k_path_into` is **not implementable** for these arguments: its + /// provisional implementation drives `descend_first_k_path` through the + /// `ZipperIteration` *default* loop, which spins forever when the focus has + /// no children, and which escapes the focus's subtree entirely when `k = 0`. + /// Verified against `pathmap` 0.3.1: `meet_k_path_into(1, false)` on a leaf + /// hangs. + pub fn meet_k_path_unspecified(&self, k: usize) -> bool { + k == 0 || self.child_count() == 0 + } + + /// `ZipperWriting::meet_k_path_into`: strip the leading `k` bytes from every + /// path below the focus and meet the results. + /// + /// Unlike [`Zip::join_k_path_into`], this routes through + /// `take_map`/`graft_map`, so values at depth exactly `k` *are* carried — they + /// become the focus value. Only meaningful when + /// [`Zip::meet_k_path_unspecified`] is `false`. + pub fn meet_k_path_into(&mut self, ops: &impl ValOps, k: usize, prune: bool) -> bool { + let f = self.focus(); + let kps = self.trie.subtrie(&f).k_paths(k); + let mut result: Option> = None; + for q in kps { + let m = self.trie.subtrie(&path::cat(&f, &q)); + result = Some(match result { + None => m, + Some(a) => PathMap::meet(ops, &a, &m), + }); + } + match result { + Some(m) if !m.is_empty_map() => { + self.graft_map(&m); + true + } + _ => { + self.remove_branches(prune); + false + } + } + } +} + diff --git a/differential/src/reference/zipper.rs b/differential/src/reference/zipper.rs new file mode 100644 index 00000000..152c66df --- /dev/null +++ b/differential/src/reference/zipper.rs @@ -0,0 +1,578 @@ +//! `Zipper.lean` — the zipper, and its read API. + +use crate::reference::basic::{ByteMask, path}; +use crate::reference::pathmap::{Entry, PathMap}; + +// =========================================================================== +// Zipper.lean — the read API +// =========================================================================== + +/// A cursor into a map. Three things determine everything it can observe or do: +/// +/// * the **map** it is looking at, +/// * its **root** — the absolute path at which it was created +/// (`root_prefix_path`); the zipper can never ascend above it, and can never +/// see anything outside the submap hanging below it, and +/// * its **path** — the relative path from the root to the current **focus** +/// (`path()`); `origin_path() = root ++ path`. +/// +/// Read zippers and write zippers share this state and differ only in which +/// operations are offered, so `Zip` models both. For a read zipper `trie` is a +/// snapshot taken when the zipper was created (`fork_read_zipper` etc.); for a +/// write zipper it is the live map, so mutations write back through +/// `root ++ path`. +/// +/// # The focus may not exist +/// +/// `descend_to` moves the focus anywhere, including off the map. `path_exists` +/// then reports `false` while `path()` still reports the full path and `ascend` +/// still walks back up. `path` is therefore an unconstrained byte string. +/// +/// # The blind-zipper contract +/// +/// `ZipperMoving` no longer provides `path()`: a zipper that does not track its +/// own path is a *blind* zipper, and `path()` / `move_to_path()` live in the +/// separate `ZipperPath: ZipperMoving` trait. The model keeps `path` as a field +/// because it has to represent the location somehow, but it mirrors the split in +/// what each operation is allowed to *observe*: [`Zip::focus_byte`] is the only +/// positional information a blind zipper can read, and its value at the root is +/// deliberately unspecified. +/// +/// The migration also changed what the movement operations report: `ascend`, +/// `ascend_until` and `ascend_until_branch` return the **number of bytes +/// ascended**, and `descend_indexed_byte`, `descend_first_byte`, +/// `descend_last_byte`, `to_next_sibling_byte` and `to_prev_sibling_byte` return +/// `Option` — the byte moved to — rather than a `bool`. +/// +/// # Depth-first order is lexicographic order +/// +/// Every iteration primitive is specified as "the least existing location +/// strictly after the focus, subject to ...", ordered by `Ord for [u8]`. That +/// is equivalent to the implementation's node-by-node walk, and far easier to +/// state and to check. +#[derive(Clone, Debug)] +pub struct Zip { + /// The map: a snapshot, for a read zipper; the live map, for a write zipper. + pub trie: PathMap, + /// `root_prefix_path()`: where the zipper was created. + pub root: Vec, + /// `path()`: the relative path from the root to the focus. + pub path: Vec, +} + +impl Zip { + /// A zipper at the root of `trie`. + pub fn new(trie: PathMap) -> Self { + Zip { trie, root: Vec::new(), path: Vec::new() } + } + + /// A zipper rooted at `root`, focused at its root. + pub fn at(trie: PathMap, root: &[u8]) -> Self { + Zip { trie, root: root.to_vec(), path: Vec::new() } + } + + /// A zipper rooted at `root` with its focus already at `path`. + pub fn at_path(trie: PathMap, root: &[u8], path: &[u8]) -> Self { + Zip { trie, root: root.to_vec(), path: path.to_vec() } + } + + /// `ZipperAbsolutePath::origin_path`: the absolute path of the focus. + pub fn focus(&self) -> Vec { + path::cat(&self.root, &self.path) + } + + /// `ZipperAbsolutePath::root_prefix_path`. + pub fn root_prefix_path(&self) -> &[u8] { + &self.root + } + + // -- trait Zipper ------------------------------------------------------- + + /// What the map holds at the focus: the whole answer to `path_exists` and + /// `val` at once. Definitions that must handle a location existing *without* + /// a value are written against this, so the `match` will not compile until + /// they say what happens to it. + pub fn entry(&self) -> Entry<&V> { + self.trie.entry_at(&self.focus()) + } + + /// `Zipper::path_exists`. + pub fn path_exists(&self) -> bool { + self.trie.path_exists(&self.focus()) + } + + /// `Zipper::is_val`. + pub fn is_val(&self) -> bool { + self.trie.val_at(&self.focus()).is_some() + } + + /// `Zipper::child_mask`. Empty on a leaf or a non-existent path. + pub fn child_mask(&self) -> ByteMask { + self.trie.child_mask(&self.focus()) + } + + /// `Zipper::child_count`. + pub fn child_count(&self) -> usize { + self.child_mask().count_bits() + } + + /// `ZipperMoving::focus_byte`: the byte last descended to reach the focus. + /// + /// **Unspecified at the root.** A zipper that retains knowledge of the map + /// above its root may return the byte leading to that root; one that does + /// not, or one rooted at the map root, returns `None`. So a `Some` here does + /// not mean the zipper has descended, and callers needing that distinction + /// must ask [`Zip::at_root`]. The model returns the last byte of the + /// relative path; a harness should mask the value at the root rather than + /// comparing it. + pub fn focus_byte(&self) -> Option { + self.path.last().copied() + } + + // -- trait ZipperValues / ZipperReadOnlyValues -------------------------- + + /// `ZipperValues::val` (and `ZipperReadOnlyValues::get_val`, which differs + /// only in the lifetime of the returned reference). + pub fn val(&self) -> Option<&V> { + self.trie.val_at(&self.focus()) + } + + /// `ZipperValues::val_at`: the value at `k`, relative to the focus. + pub fn val_at(&self, k: &[u8]) -> Option<&V> { + self.trie.val_at(&path::cat(&self.focus(), k)) + } + + // -- trait ZipperSubtries / ZipperInfallibleSubtries -------------------- + + /// `ZipperInfallibleSubtries::make_map`. Under the default `graft_root_vals` + /// feature the value at the focus becomes the new map's root value. + pub fn make_map(&self) -> PathMap { + self.trie.subtrie(&self.focus()) + } + + /// The node below the focus — what `get_focus` returns, i.e. `make_map` with + /// the focus value stripped. This, not `make_map`, is what the algebraic + /// operations and `graft_internal` consume. + pub fn focus_node(&self) -> PathMap { + let mut m = self.make_map(); + m.remove_val(&[]); + m + } + + /// `get_focus().is_none()`: the focus has no descendants. + pub fn focus_node_is_empty(&self) -> bool { + self.trie.below_is_empty(&self.focus()) + } + + // -- trait ZipperMoving — position -------------------------------------- + + /// `ZipperMoving::at_root`. + pub fn at_root(&self) -> bool { + self.path.is_empty() + } + + /// `ZipperMoving::reset`. + pub fn reset(&mut self) { + self.path.clear(); + } + + /// `ZipperMoving::val_count`: values at and below the focus. + pub fn val_count(&self) -> usize { + self.trie.val_count(&self.focus()) + } + + /// The absolute path of the relative location `q` — the model's `atPath`, + /// which lets an ancestor or descendant be *named* and then asked about, so + /// the specifications can say "the deepest ancestor such that ..." instead of + /// walking there step by step. + fn abs(&self, q: &[u8]) -> Vec { + path::cat(&self.root, q) + } + + // -- trait ZipperMoving — descent --------------------------------------- + + /// `ZipperMoving::descend_to`. Never fails; the focus may end up off-map. + pub fn descend_to(&mut self, k: &[u8]) { + self.path.extend_from_slice(k); + } + + /// `ZipperMoving::descend_to_byte`. + pub fn descend_to_byte(&mut self, b: u8) { + self.path.push(b); + } + + /// `ZipperMoving::descend_to_check`: descend, then report existence. + pub fn descend_to_check(&mut self, k: &[u8]) -> bool { + self.descend_to(k); + self.path_exists() + } + + /// How far along `k` the path still exists, starting from the focus. + /// + /// Existence is prefix-closed, so the prefixes of `k` that still exist form + /// an initial segment: the answer is the longest prefix of `k` that exists. + fn reach(&self, k: &[u8]) -> usize { + let f = self.focus(); + let mut probe = f.clone(); + let mut n = 0; + if !self.trie.path_exists(&probe) { + return 0; + } + for &b in k { + probe.push(b); + if !self.trie.path_exists(&probe) { + break; + } + n += 1; + } + n + } + + /// `ZipperMoving::descend_to_existing`: descend byte by byte, stopping where + /// the path stops existing. Returns the number of bytes actually descended. + pub fn descend_to_existing(&mut self, k: &[u8]) -> usize { + let n = self.reach(k); + self.descend_to(&k[..n]); + n + } + + /// `ZipperMoving::descend_to_val`: descend byte by byte, stopping at the + /// first value encountered *below* the starting focus, or where the path + /// stops existing. + pub fn descend_to_val(&mut self, k: &[u8]) -> usize { + let reach = self.reach(k); + let f = self.focus(); + // A value already at the focus does not stop it, so the scan starts at 1. + let stop = (1..=reach) + .find(|&j| self.trie.val_at(&path::cat(&f, &k[..j])).is_some()) + .unwrap_or(reach); + self.descend_to(&k[..stop]); + stop + } + + /// `ZipperMoving::descend_to_existing_byte`. + pub fn descend_to_existing_byte(&mut self, b: u8) -> bool { + self.path.push(b); + if self.path_exists() { + true + } else { + self.path.pop(); + false + } + } + + /// `ZipperMoving::descend_indexed_byte`: descend into the `idx`-th child in + /// ascending byte order, returning the byte moved to. Out-of-range indices + /// do nothing and return `None`. + pub fn descend_indexed_byte(&mut self, idx: usize) -> Option { + let b = self.child_mask().indexed_bit(idx)?; + self.path.push(b); + Some(b) + } + + /// `ZipperMoving::descend_first_byte`. + pub fn descend_first_byte(&mut self) -> Option { + self.descend_indexed_byte(0) + } + + /// `ZipperMoving::descend_last_byte`. + pub fn descend_last_byte(&mut self) -> Option { + let c = self.child_count(); + if c == 0 { None } else { self.descend_indexed_byte(c - 1) } + } + + /// `ZipperMoving::descend_until`: descend while there is exactly one child, + /// stopping on a value. A no-op on a branch, a leaf, or a non-existent path. + /// + /// Nothing happens unless the focus has exactly one child. When it does, the + /// locations below it form a chain until the first one that branches, ends, + /// or carries a value — so the destination is simply the *nearest* descendant + /// that is a value or is not single-childed. Depth-first order along a chain + /// is order of increasing depth, so the first hit is the nearest one. + pub fn descend_until(&mut self) -> bool { + if self.child_count() != 1 { + return false; + } + let f = self.focus(); + let hit = self + .trie + .strictly_below(&f) + .find(|(q, v)| v.is_some() || self.trie.child_count(q) != 1) + .map(|(q, _)| q.clone()); + match hit { + Some(q) => { + self.path = q[self.root.len()..].to_vec(); + true + } + None => false, + } + } + + /// `ZipperMoving::descend_until_observed`: `descend_until`, reporting each + /// byte it descends to a `PathObserver`. + /// + /// For the `Vec` observer — the one the harness uses — the reported + /// sequence is exactly the path delta, which is the only way a blind zipper + /// can learn where it ended up. That equality is the property worth checking. + pub fn descend_until_observed(&mut self) -> (bool, Vec) { + let before = self.path.len(); + let moved = self.descend_until(); + (moved, self.path[before..].to_vec()) + } + + /// `ZipperMoving::descend_until_max_bytes`: `descend_until`, then ascend back + /// to at most `max_bytes` below the starting depth. + pub fn descend_until_max_bytes(&mut self, max_bytes: usize) -> bool { + if max_bytes == 0 { + return false; + } + let target = self.path.len() + max_bytes; + let moved = self.descend_until(); + if self.path.len() > target { + self.path.truncate(target); + } + moved + } + + // -- trait ZipperMoving — ascent ---------------------------------------- + + /// `ZipperMoving::ascend`: ascend `steps` bytes, clamping at the zipper root. + /// Returns the **number of bytes actually ascended**, which is smaller than + /// `steps` when the root was closer than that. + pub fn ascend(&mut self, steps: usize) -> usize { + let n = steps.min(self.path.len()); + self.path.truncate(self.path.len() - n); + n + } + + /// `ZipperMoving::ascend_byte`: still a `bool`, defined as `ascend(1) == 1`. + pub fn ascend_byte(&mut self) -> bool { + self.ascend(1) == 1 + } + + /// `ZipperMoving::ascend_until`: ascend to the nearest strict ancestor that + /// carries a value or branches, or to the root. Returns the number of bytes + /// ascended; `0` means the zipper was already at its root. + pub fn ascend_until(&mut self) -> usize { + self.ascend_until_with(|z, a| { + z.trie.val_at(a).is_some() || z.trie.child_count(a) > 1 + }) + } + + /// `ZipperMoving::ascend_until_branch`: like `ascend_until`, but values do + /// not stop the ascent. Returns the number of bytes ascended. + pub fn ascend_until_branch(&mut self) -> usize { + self.ascend_until_with(|z, a| z.trie.child_count(a) > 1) + } + + /// The shared core: the destination is the deepest strict ancestor + /// satisfying `stop`, or the zipper root, which always qualifies — so there + /// is always an answer. + fn ascend_until_with(&mut self, stop: impl Fn(&Self, &[u8]) -> bool) -> usize { + if self.at_root() { + return 0; + } + let mut dest = 0; + for n in (1..self.path.len()).rev() { + let anc = self.abs(&self.path[..n]); + if stop(self, &anc) { + dest = n; + break; + } + } + let moved = self.path.len() - dest; + self.path.truncate(dest); + moved + } + + // -- trait ZipperMoving — lateral movement ------------------------------ + + /// `ZipperMoving::to_next_sibling_byte`. + /// + /// At the zipper root there is no last byte, so the documented answer — and + /// the `ZipperMoving` default implementation's answer — is "did not move". + /// + /// BUG (`pathmap` 0.3.1): the native `ReadZipper` implementation instead + /// consults the last byte of the *absolute* origin path, so a read zipper + /// whose root does not exist but has a sibling **leaves its own root**. With + /// map `{[0,0,3] ↦ v}` and a zipper rooted at `[0,0,1]`, + /// `to_next_sibling_byte()` returns `true`, `path()` still reports `[]` and + /// `at_root()` still reports `true`, but `origin_path()` is now `[0,0,3]` and + /// the zipper reads `v`. That breaks the containment a `ZipperHead` relies + /// on to hand out non-overlapping zippers. The model specifies the + /// documented behaviour; the differential harness skips the operation at the + /// root so the known bug does not mask others. + /// + /// Transcribed from the *current* `Zipper.lean`. The archive this file came + /// from described the bug as fixed and ran the op at the root; `Fuzz.lean` + /// and `harness.rs` both skip it today, so the model follows them. See the + /// report accompanying this port for whether the skip is still earned. + pub fn to_next_sibling_byte(&mut self) -> Option { + self.to_sibling_byte(true) + } + + /// `ZipperMoving::to_prev_sibling_byte`. + pub fn to_prev_sibling_byte(&mut self) -> Option { + self.to_sibling_byte(false) + } + + /// Both sibling moves are keyed on `focus_byte`, whose value at the root is + /// unspecified, so the `at_root` guard is what keeps the zipper inside its + /// own subtree. + // `to_*` here is `pathmap`'s "move the cursor to", not a conversion, so the + // `&mut self` receiver is right and clippy's naming convention does not apply. + #[allow(clippy::wrong_self_convention)] + fn to_sibling_byte(&mut self, forward: bool) -> Option { + let cur = self.focus_byte()?; + if self.at_root() { + return None; + } + self.path.pop(); + let mask = self.child_mask(); + let hit = if forward { mask.next_bit(cur) } else { mask.prev_bit(cur) }; + match hit { + Some(b) => { + self.path.push(b); + Some(b) + } + None => { + self.path.push(cur); + None + } + } + } + + /// `ZipperMoving::move_to_path`: jump to `p` relative to the zipper root. + /// Returns the number of bytes shared between the old and the new location. + pub fn move_to_path(&mut self, p: &[u8]) -> usize { + let overlap = p + .iter() + .zip(self.path.iter()) + .take_while(|(a, b)| a == b) + .count(); + self.path = p.to_vec(); + overlap + } + + // -- trait ZipperMoving — depth-first stepping -------------------------- + + /// `ZipperMoving::to_next_step`: the next existing location in depth-first + /// order. On exhaustion the focus returns to the root and the result is + /// `false`. + pub fn to_next_step(&mut self) -> bool { + self.step_to(|_, _| true) + } + + // -- trait ZipperIteration ---------------------------------------------- + + /// `ZipperIteration::to_next_val`: the next existing location carrying a + /// value, in depth-first order. Never reports the value at the starting + /// focus. On exhaustion the focus returns to the root and the result is + /// `false`. + pub fn to_next_val(&mut self) -> bool { + self.step_to(|_, v| v.is_some()) + } + + /// The shared core of the two: the least existing location strictly after the + /// focus and within the zipper's own subtree that satisfies `pred`. + fn step_to(&mut self, pred: impl Fn(&[u8], &Option) -> bool) -> bool { + let f = self.focus(); + let root = self.root.clone(); + let hit = self + .trie + .after_within(&f, &root) + .find(|(q, v)| pred(q, v)) + .map(|(q, _)| q.clone()); + match hit { + Some(q) => { + self.path = q[self.root.len()..].to_vec(); + true + } + None => { + self.reset(); + false + } + } + } + + /// `ZipperReadOnlyIteration::to_next_get_val`. + pub fn to_next_get_val(&mut self) -> Option<&V> { + if self.to_next_val() { self.val() } else { None } + } + + /// `ZipperIteration::descend_last_path`: follow the last child to the end of + /// the depth-first-greatest path below the focus. + pub fn descend_last_path(&mut self) -> bool { + let f = self.focus(); + match self.trie.at_or_below(&f).last().map(|(q, _)| q.clone()) { + Some(q) if q.len() > f.len() => { + self.path = q[self.root.len()..].to_vec(); + true + } + _ => false, + } + } + + /// The shared core of `descend_first_k_path` and `to_next_k_path` + /// (`k_path_internal`): the depth-first-least existing location that is + /// exactly `k` bytes below the common ancestor at depth `base`, and that + /// comes strictly after the current focus. On failure the focus moves to + /// that ancestor. + fn k_path_from(&mut self, base: usize, k: usize) -> bool { + let anc_rel = self.path[..base].to_vec(); + let anc = self.abs(&anc_rel); + let f = self.focus(); + let want = anc.len() + k; + let hit = self + .trie + .after_within(&f, &anc) + .find(|(q, _)| q.len() == want) + .map(|(q, _)| q.clone()); + match hit { + Some(q) => { + self.path = q[self.root.len()..].to_vec(); + true + } + None => { + self.path = anc_rel; + false + } + } + } + + /// `ZipperIteration::descend_first_k_path`: descend to the depth-first-first + /// existing location exactly `k` bytes below the focus. Leaves the focus + /// untouched and returns `false` when there is none. + pub fn descend_first_k_path(&mut self, k: usize) -> bool { + let base = self.path.len(); + self.k_path_from(base, k) + } + + /// `ZipperIteration::to_next_k_path`: the next existing location at the same + /// depth, under the common ancestor `k` bytes above the focus. On exhaustion + /// the focus moves to that ancestor and the result is `false`. + /// + /// NOTE: when the focus is shallower than `k`, the *native* `ReadZipper` falls + /// back to the **zipper root** as the common ancestor — so the call behaves + /// like `descend_first_k_path(k)` from the root and can succeed. The + /// `ZipperIteration` default implementation instead returns `false` without + /// moving. The model follows the native `ReadZipper`, which is what the + /// public API reaches. + pub fn to_next_k_path(&mut self, k: usize) -> bool { + if k <= self.path.len() { + let base = self.path.len() - k; + self.k_path_from(base, k) + } else { + self.k_path_from(0, k) + } + } + + // -- trait ZipperForking ------------------------------------------------ + + /// `ZipperForking::fork_read_zipper`: a new zipper rooted at the current + /// focus, over a snapshot of the same map. + pub fn fork_read_zipper(&self) -> Zip { + Zip { trie: self.trie.clone(), root: self.focus(), path: Vec::new() } + } +} + diff --git a/lean/differential.py b/lean/differential.py index ac20e208..b43c71a8 100755 --- a/lean/differential.py +++ b/lean/differential.py @@ -7,6 +7,7 @@ lean/.lake/build/bin/pathmap-oracle the Lean model (always) target/*/pathmap_trace the real crate (default) + target/*/reference the Rust model (--model) target/*/act_trace ACT read source (--act) Each child is spawned once with `--server` and stays resident, taking inputs as @@ -17,6 +18,14 @@ ./lean/differential.py corpus/* # check a corpus ./lean/differential.py --random 500 # generate and check + ./lean/differential.py --model --random 500 # check the Rust port + +`--model` is the acceptance test for `differential/src/reference/`: it compares +two independent transcriptions of the same specification, in different +languages, with the crate not involved at all. So the KNOWN table below does not +apply -- every divergence is a bug in one of the two models, and none may be +tolerated. Once it is clean, `target/release/in_process` compares the Rust model +against the crate in one process, with no pipes, about 4x faster. """ import argparse import multiprocessing @@ -41,6 +50,12 @@ os.path.join(ROOT, "target", "release", "act_trace"), os.path.join(ROOT, "target", "debug", "act_trace"), ] +# The Rust port of the Lean model (differential/src/reference/), driven by +# `--model`. Its trace front end is `differential/src/bin/reference.rs`. +MODEL_CANDIDATES = [os.environ.get("PATHMAP_REFERENCE", "")] + [ + os.path.join(ROOT, "target", "release", "reference"), + os.path.join(ROOT, "target", "debug", "reference"), +] # Seconds a single input may take. Measured over 2000 random programs against # the real crate, the non-hanging ones run in p50 0.19ms / p100 1.21ms, so this # is ~1600x the worst legitimate case and still cuts the cost of a hang by 15x @@ -48,10 +63,18 @@ TIMEOUT = 2.0 -def find_trace_bin(act): - for c in (ACT_CANDIDATES if act else TRACE_CANDIDATES): +def find_trace_bin(act, model=False): + if model: + candidates = MODEL_CANDIDATES + elif act: + candidates = ACT_CANDIDATES + else: + candidates = TRACE_CANDIDATES + for c in candidates: if c and os.path.exists(c): return c + if model: + sys.exit("build the Rust model first: cargo build --release -p differential") if act: sys.exit("build the ACT side first: " "cargo build --release -p differential") @@ -604,6 +627,9 @@ def main(): ap.add_argument("-v", "--verbose", action="store_true") ap.add_argument("--act", action="store_true", help="use an ArenaCompactTree as the read source") + ap.add_argument("--model", action="store_true", + help="compare the Lean model against the Rust model " + "(differential/src/reference/) instead of against the crate") ap.add_argument("-j", "--jobs", type=int, default=1, help="run this many worker processes in parallel " "(each owns its own pair of children)") @@ -614,8 +640,8 @@ def main(): args = ap.parse_args() TIMEOUT = args.timeout - trace_bin = find_trace_bin(args.act) - other_label = "crate" + trace_bin = find_trace_bin(args.act, args.model) + other_label = "rust" if args.model else "crate" # Inputs are produced by an `InputSource`, in whichever worker picks the # index up -- see the class docs. Nothing is written to disk and no blob is @@ -632,7 +658,7 @@ def main(): n_inputs = len(source) oracle_argv = [ORACLE] + (["--act"] if args.act else []) - other_argv = [trace_bin] + other_argv = [trace_bin] + (["--act"] if (args.act and args.model) else []) faildir = [] # created on first failure only def save(idx): @@ -659,7 +685,9 @@ def record(idx, msg): if args.verbose: print("ok %s" % name) return False - note = classify(msg) + # Model against model: the KNOWN table is a list of *crate* defects, and + # the crate is not involved. Every divergence is new. + note = None if args.model else classify(msg) if note: known[note] = known.get(note, 0) + 1 if args.verbose: From 9833f0fcc41ba1bbe3a2f4212e9a105fcb61539b Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 04:38:07 +0000 Subject: [PATCH 17/73] Run the reference model against the crate in one process, and under AFL Two front ends over the revived model, both reusing the existing op tables by module path -- the model side `reference::fuzz::run`, the crate side `harness::run` / `act::run_act` -- so there is still no fourth transcription. `bin/in_process.rs` is the archived comparator, brought forward. It runs both on the same bytes and compares the traces in memory: no subprocesses, no pipes, no hex on stdin, no Python. The reason it was shelved is gone. It was written against a crate that panicked on roughly one random input in eight, and it deliberately does not `catch_unwind`, because catching a panic out of the middle of a trie mutation drops half-updated refcounted nodes while the stack unwinds and corrupts the heap. So a run used to end within a handful of inputs. The panicking findings have since been fixed; the design is unchanged, there is simply nothing left to end the run. The panic hook stays, and if a panic reappears it stops and names the input, which is the right outcome. Two deliberate changes from the archive: * the random source now draws lengths uniformly from `[8, maxlen)`, which is what `RandomInputs.get` in differential.py does. The archive drew from `[8, 8+maxlen)`, so a divergence rate measured here was not comparable with one measured there, which is the whole reason to have both. * `--act` runs the ArenaCompactTree read source, so the in-process fuzzer covers the same two configurations differential.py does. It does not classify. The `KNOWN` table of tolerated crate defects lives in lean/differential.py and there is exactly one of it, so `--save DIR` writes the diverging inputs out and `./lean/differential.py DIR/*` does the triage -- which also re-checks each one against the Lean oracle rather than against this binary's own model. The run's own summary buckets by first differing *op*, which needs no taxonomy. `bin/afl_differential.rs` is the same comparison under AFL++, behind the `afl` feature so a plain `cargo build -p differential` neither needs the dependency nor builds the target. Uniform random bytes are the right shape for measuring a rate and the wrong shape for finding: the residual divergence classes sit between 1 in 30,000 and 1 in 2,250,000, and a blind sampler pays full price for every one. AFL keeps what reached new edges and mutates that. The wire format suits it -- every operand is a byte reduced mod a small number at the point of use, there is no checksum and no whole-input length prefix, so a truncated input is a valid shorter program and a byte flip lands on an op selector rather than being rejected by a parser. AFL also disposes of the unwind-safety question rather than working around it: each input runs in a child forked from the fork server and `afl::fuzz!` aborts instead of unwinding, so corruption cannot outlive the input that caused it. That is the property the subprocess design bought, at a fraction of the cost. `in_process --emit-corpus DIR` writes the seed corpus from the same generator. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/Cargo.toml | 16 ++ differential/src/bin/afl_differential.rs | 90 ++++++ differential/src/bin/in_process.rs | 344 +++++++++++++++++++++++ differential/src/reference/mod.rs | 9 +- 4 files changed, 457 insertions(+), 2 deletions(-) create mode 100644 differential/src/bin/afl_differential.rs create mode 100644 differential/src/bin/in_process.rs diff --git a/differential/Cargo.toml b/differential/Cargo.toml index 4218b74a..cfbb3b4c 100644 --- a/differential/Cargo.toml +++ b/differential/Cargo.toml @@ -7,3 +7,19 @@ description = "The Rust side of the differential fuzzing harness for pathmap's z [dependencies] pathmap = { path = "..", features = ["arena_compact"] } +# Only for `bin/afl_differential.rs`, behind the `afl` feature, so a plain +# `cargo build -p differential` neither needs it nor builds it. See that file. +afl = { version = "0.18", optional = true } + +[features] +afl = ["dep:afl"] + +# Coverage-guided front end for the in-process comparator. Declared explicitly +# rather than left to src/bin autodiscovery so it can carry `required-features`: +# without it, `cargo build -p differential` would try to compile a file that +# needs the `afl` crate. Build it with `cargo afl build`, not `cargo build` -- +# the instrumentation is a rustc flag `cargo afl` sets. +[[bin]] +name = "afl_differential" +path = "src/bin/afl_differential.rs" +required-features = ["afl"] diff --git a/differential/src/bin/afl_differential.rs b/differential/src/bin/afl_differential.rs new file mode 100644 index 00000000..c6f1b5de --- /dev/null +++ b/differential/src/bin/afl_differential.rs @@ -0,0 +1,90 @@ +//! Coverage-guided differential fuzzing: the Rust reference model against the +//! real crate, driven by AFL++ instead of by a random byte generator. +//! +//! ```text +//! cargo install cargo-afl # once +//! cargo afl build --release -p differential --features afl +//! ./lean/afl-seed.sh out/afl-in # or any corpus +//! cargo afl fuzz -i out/afl-in -o out/afl-out \ +//! target/release/afl_differential +//! ``` +//! +//! # Why a second front end at all +//! +//! `in_process` generates uniformly random bytes. That is the right shape for +//! *measuring* — a rate over a known distribution, comparable with +//! `differential.py`'s — and the wrong shape for *finding*: the divergence +//! classes it turns up sit between 1 in 30,000 and 1 in 2,250,000 inputs, which +//! is what a blind sampler costs when the interesting programs are a thin set. +//! AFL keeps the inputs that reached new edges and mutates those, so a 40-op +//! program that got a write zipper into an unusual node representation becomes +//! the stem for the next thousand. Same comparison, same two op tables, better +//! search. +//! +//! The wire format suits it: every operand is a byte, reduced mod a small number +//! at the point of use, so AFL's byte flips, arithmetic and splices all land on +//! op selectors and path bytes rather than being rejected by a parser. There is +//! no checksum, no length prefix over the whole input, and a truncated input is +//! a valid shorter program. +//! +//! # Why this is safe where `in_process` has to be careful +//! +//! `in_process` runs every input in one long-lived process, so a panic out of +//! the middle of a trie mutation cannot be caught and recovered from — +//! `pathmap` is not unwind-safe, and dropping half-updated refcounted nodes +//! while the stack unwinds corrupts the heap. It therefore reports from a panic +//! *hook* and exits. +//! +//! AFL removes the problem rather than working around it. Each input runs in a +//! child forked from the fork server, and `afl::fuzz!` installs a hook that +//! **aborts** rather than unwinds. So a panicking input kills one child, is +//! written to `out/afl-out/default/crashes/`, and fuzzing continues from the +//! next one. Corruption cannot outlive the input that caused it. That is the +//! same property the subprocess design in `differential.py` bought, at a +//! fraction of the cost, and it is why this file does not reproduce +//! `in_process`'s panic-hook dance. +//! +//! A *divergence* is reported the same way a panic is — by panicking — so AFL +//! files it as a crash. Replay one with the plain comparator, which prints the +//! differing line rather than a backtrace: +//! +//! ```text +//! target/release/in_process out/afl-out/default/crashes/id:000000* +//! ./lean/differential.py out/afl-out/default/crashes/* # KNOWN-table breakdown +//! ``` +//! +//! Note that `crashes/` will fill up with the *known* residual defects +//! (`meet_keeps_dangling` and friends) within the first minute, because to this +//! target they are indistinguishable from a new finding. Triage is +//! `differential.py`'s job: it owns the one `KNOWN` table. + +use differential::harness::run as crate_run; +use differential::reference::fuzz::run as model_run; + +fn main() { + afl::fuzz!(|data: &[u8]| { + // An empty or near-empty input decodes to `EMPTY` on both sides; let AFL + // keep it as a seed anyway, it costs one comparison. + let model = model_run(data, false); + let real = crate_run(data, false); + if model == real { + return; + } + // Panicking is the reporting channel: `afl::fuzz!`'s hook turns it into + // an abort, which AFL records as a crash and saves the input for. + let first = model + .lines() + .zip(real.lines()) + .enumerate() + .find(|(_, (a, b))| a != b) + .map(|(i, (a, b))| format!("line {i}\n model: {a}\n crate: {b}")) + .unwrap_or_else(|| { + format!( + "length {} (model) vs {} (crate) lines", + model.lines().count(), + real.lines().count() + ) + }); + panic!("model/crate divergence:\n{first}"); + }); +} diff --git a/differential/src/bin/in_process.rs b/differential/src/bin/in_process.rs new file mode 100644 index 00000000..ac571d25 --- /dev/null +++ b/differential/src/bin/in_process.rs @@ -0,0 +1,344 @@ +//! In-process differential fuzzer: the Rust reference model against the real crate. +//! +//! in_process --random 100000 +//! in_process --random 10000000 -j 56 --max-fails 0 --save runs/diverged +//! in_process --random 2000000 -j 56 --act +//! in_process corpus/*.bin +//! +//! `lean/differential.py` compares the *Lean* model against something else, and +//! pays for it: two child processes, a hex-encoded input over a pipe, a rendered +//! trace back, and a Python driver in the middle. That buys language +//! independence — two transcriptions of one specification, written in different +//! languages — and it is the right tool for validating the port +//! (`differential.py --model`). +//! +//! It is the wrong tool for *volume*. Once the Rust model is known to agree with +//! the Lean one, the crate can be checked against the Rust model with no +//! processes, no pipes and no serialisation at all: both run in this binary, on +//! the same input, and the results are compared in memory. Nothing is rendered +//! unless something diverges. +//! +//! # What is being compared +//! +//! Exactly what `differential.py` compares, through exactly the same two op +//! tables — there is no fourth transcription here: +//! +//! * the model side is [`differential::reference::fuzz`], the port of `Fuzz.lean`; +//! * the crate side is [`differential::harness`], shared with `pathmap_trace`, +//! `act_trace` and the repro generator. +//! +//! Both are driven from the same bytes and must produce the same trace. Sharing +//! the op tables rather than copying them is the point: a fourth copy would drift. +//! +//! # Why this could not run before, and can now +//! +//! It was written against a crate that was **not unwind-safe** and panicked on +//! roughly one random input in eight. Catching such a panic with `catch_unwind` +//! and carrying on corrupts the heap — `malloc(): unaligned tcache chunk +//! detected` — because the half-updated refcounted nodes are dropped while the +//! stack unwinds. So this does **not** catch panics, and never should: a panic +//! *hook* runs before unwinding starts, which makes it the last safe moment to +//! say which input was responsible; it reports and `exit`s, and nothing unwinds +//! through the crate's internals. With a 1-in-8 panic rate that ended a run +//! within a handful of inputs, which is why the subprocess design existed at all: +//! it is not overhead, it is panic tolerance. A dead child costs one input. +//! +//! The panicking findings have since been fixed. The design here is unchanged — +//! there are simply no panics left to end the run. If one reappears, this will +//! stop on it and name it, which is the correct outcome and a finding in its own +//! right. +//! +//! # Reporting +//! +//! A divergence is printed as the first differing trace line, and `--save DIR` +//! writes the input beside it. Classification into the known-defect buckets is +//! deliberately *not* reimplemented here: the `KNOWN` table lives in +//! `lean/differential.py` and there is exactly one of it. Run +//! `./lean/differential.py DIR/*` over the saved inputs to get the breakdown — +//! which also re-checks each one against the Lean oracle rather than against this +//! binary's own model. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use differential::act::run_act as crate_run_act; +use differential::harness::run as crate_run; +use differential::reference::fuzz::run as model_run; + +/// Where inputs come from. +/// +/// `get` is deterministic in `idx` and holds no state between calls, so which +/// thread runs an input cannot change it and a failing input can be re-derived +/// from its index alone. Mirrors `InputSource` in `lean/differential.py`; a +/// queue-backed source drops in here the same way. +enum Source { + Random { seed: u64, count: usize, maxlen: usize }, + Files(Vec), +} + +impl Source { + fn len(&self) -> usize { + match self { + Source::Random { count, .. } => *count, + Source::Files(v) => v.len(), + } + } + + fn name(&self, idx: usize) -> String { + match self { + Source::Random { .. } => format!("random#{idx:06}"), + Source::Files(v) => v[idx].clone(), + } + } + + fn get(&self, idx: usize) -> Vec { + match *self { + Source::Random { seed, maxlen, .. } => { + // splitmix64, seeded per index so generation parallelises without + // changing what gets tested. + let mut s = seed.wrapping_mul(0x9E3779B97F4A7C15) + ^ (idx as u64).wrapping_mul(0xBF58476D1CE4E5B9); + let mut next = || { + s = s.wrapping_add(0x9E3779B97F4A7C15); + let mut z = s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + }; + // The same *distribution* as `RandomInputs.get` in + // `lean/differential.py` -- `randrange(8, maxlen)`, i.e. uniform + // on `[8, maxlen)` -- so a divergence rate measured here is + // directly comparable to one measured there. The bit stream + // differs (splitmix64 against Python's Mersenne Twister) and is + // meant to: two independent samples of the same population. + let span = maxlen.saturating_sub(8).max(1); + let n = 8 + (next() as usize) % span; + (0..n).map(|_| next() as u8).collect() + } + Source::Files(ref v) => std::fs::read(&v[idx]).expect("cannot read input"), + } + } +} + +/// The index each thread is currently working on, so a panic or an abort can be +/// attributed. +/// +/// A panic ends the run by design (see the module docs); an *abort* is not +/// catchable in-process at all. Either way the fuzzer prints what every thread +/// had in flight on the way down, and `--from` resumes past it. +static IN_FLIGHT: [AtomicUsize; 256] = [const { AtomicUsize::new(usize::MAX) }; 256]; + +/// Compare one input. Returns the rendered report on divergence, `None` on +/// agreement. +/// +/// The traces are only *rendered* because both op tables render them today; the +/// comparison itself is a string equality, and neither side leaves this process. +fn compare(blob: &[u8], act: bool) -> Option { + let model = model_run(blob, act); + // NOT wrapped in `catch_unwind`, and that is deliberate. See the module + // docs: catching a panic out of the middle of a trie mutation and carrying + // on corrupts the heap. The panic hook installed in `main` reports and + // exits instead of letting the stack unwind at all. + let real = if act { crate_run_act(blob, false) } else { crate_run(blob, false) }; + // The fast path is one `memcmp` over two buffers. Individual lines are only + // needed to *report* a divergence, so they are only split out then. + if model == real { + return None; + } + for (i, (a, b)) in model.lines().zip(real.lines()).enumerate() { + if a != b { + return Some(format!("line {i}\n model: {a}\n crate: {b}")); + } + } + Some(format!( + "length {} (model) vs {} (crate) lines", + model.lines().count(), + real.lines().count() + )) +} + +/// The operation name of the first differing line, used only for the run's own +/// summary. Buckets by *op*, not by defect: the defect taxonomy is the `KNOWN` +/// table in `lean/differential.py`, and there is one of it. +fn first_diff_op(msg: &str) -> String { + msg.lines() + .find_map(|l| l.trim_start().strip_prefix("model: ")) + .and_then(|l| l.split_whitespace().nth(1)) + .unwrap_or("?") + .to_string() +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let flag = |name: &str, default: usize| -> usize { + args.iter() + .position(|a| a == name) + .and_then(|i| args.get(i + 1)) + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + }; + let str_flag = |name: &str| -> Option { + args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).cloned() + }; + let count = flag("--random", 0); + // Index to start at. Inputs are deterministic in their index, so a run that + // dies can be resumed past the offending one. + let from = flag("--from", 0); + let seed = flag("--seed", 1) as u64; + let maxlen = flag("--maxlen", 300); + let jobs = flag("-j", 1).max(1); + // 0 means "never stop": a long survey wants every divergence, not the first ten. + let max_fails = flag("--max-fails", 10); + let save_dir = str_flag("--save"); + // Read source is an `ArenaCompactTree` built from map1 rather than the + // `PathMap` itself, exactly as `act_trace` does it. The model takes the + // same flag and skips the operations ACT cannot serve. + let act = args.iter().any(|a| a == "--act"); + let value_flags = [ + "--random", "--from", "--seed", "--maxlen", "-j", "--max-fails", "--save", "--dump", + "--emit-corpus", + ]; + let files: Vec = { + let mut v = Vec::new(); + let mut it = args.iter().peekable(); + while let Some(a) = it.next() { + if a.starts_with('-') { + if value_flags.contains(&a.as_str()) { + it.next(); // its value + } + } else { + v.push(a.clone()); + } + } + v + }; + let source = if !files.is_empty() { + Source::Files(files) + } else if count > 0 { + Source::Random { seed, count, maxlen } + } else { + eprintln!( + "usage: in_process [--random N] [--seed S] [--maxlen L] [--from I] \ + [-j N] [--max-fails N] [--save DIR] [--emit-corpus DIR] [--act] [FILES...]" + ); + std::process::exit(2); + }; + + // `--dump IDX` writes one input to stdout and exits, so an input that ends a + // run can still be extracted and replayed against the crate alone. + if let Some(i) = args.iter().position(|a| a == "--dump") { + let idx: usize = args[i + 1].parse().expect("--dump IDX"); + use std::io::Write; + std::io::stdout().write_all(&source.get(idx)).unwrap(); + return; + } + + // `--emit-corpus DIR` writes the whole source out as one file per input, and + // runs nothing. That is the seed corpus for `afl_differential`: AFL needs + // starting points that already reach interesting states, and the same + // generator that feeds this binary is the obvious source of them. + if let Some(d) = str_flag("--emit-corpus") { + std::fs::create_dir_all(&d).expect("cannot create --emit-corpus directory"); + for idx in 0..source.len() { + let path = std::path::Path::new(&d).join(format!("{idx:06}.bin")); + std::fs::write(&path, source.get(idx)).expect("cannot write seed"); + } + println!("wrote {} seeds to {d}", source.len()); + return; + } + + if let Some(d) = &save_dir { + std::fs::create_dir_all(d).expect("cannot create --save directory"); + } + + let n = source.len(); + let next = AtomicUsize::new(from); + let agreed = AtomicUsize::new(0); + let stop = AtomicBool::new(false); + let reports: Mutex> = Mutex::new(Vec::new()); + // A crate panic ends the run, by design. The hook runs *before* the stack + // unwinds, so it is the last safe moment to say which input did it -- and + // exiting from here means nothing unwinds through `pathmap`'s internals. + std::panic::set_hook(Box::new(|info| { + let live: Vec = IN_FLIGHT + .iter() + .map(|a| a.load(Ordering::Relaxed)) + .filter(|&i| i != usize::MAX) + .collect(); + eprintln!("CRATE PANIC on input index {live:?}: {info}"); + eprintln!(" the crate panics on this input; the model is total and cannot."); + eprintln!(" extract it with --dump IDX, resume past it with --from {}", + live.iter().max().map_or(0, |m| m + 1)); + // Not `abort`: exit runs no destructors and unwinds nothing. + std::process::exit(101); + })); + + let start = std::time::Instant::now(); + let (src, next_r, stop_r, agreed_r, reports_r) = (&source, &next, &stop, &agreed, &reports); + std::thread::scope(|scope| { + for slot in 0..jobs { + scope.spawn(move || { + loop { + if stop_r.load(Ordering::Relaxed) { + return; + } + let idx = next_r.fetch_add(1, Ordering::Relaxed); + if idx >= n { + IN_FLIGHT[slot.min(255)].store(usize::MAX, Ordering::Relaxed); + return; + } + IN_FLIGHT[slot.min(255)].store(idx, Ordering::Relaxed); + match compare(&src.get(idx), act) { + None => { + agreed_r.fetch_add(1, Ordering::Relaxed); + } + Some(msg) => { + let mut r = reports_r.lock().unwrap(); + r.push((idx, src.name(idx), msg)); + if max_fails != 0 && r.len() >= max_fails { + stop_r.store(true, Ordering::Relaxed); + } + } + } + } + }); + } + }); + let elapsed = start.elapsed().as_secs_f64(); + + // Sorted by input index, so a -j run reports in the same order a -j1 run does. + let mut reports = reports.into_inner().unwrap(); + reports.sort_by_key(|(idx, _, _)| *idx); + let mut by_op: std::collections::BTreeMap = Default::default(); + for (idx, name, msg) in &reports { + *by_op.entry(first_diff_op(msg)).or_default() += 1; + match &save_dir { + Some(d) => { + let path = std::path::Path::new(d).join(format!("{idx:08}.bin")); + let _ = std::fs::write(&path, source.get(*idx)); + println!("FAIL {name} [saved {}]: {msg}", path.display()); + } + None => println!("FAIL {name}: {msg}"), + } + } + let ok = agreed.load(Ordering::Relaxed); + let done = ok + reports.len(); + if !by_op.is_empty() { + println!("--- first differing op ---"); + for (op, k) in &by_op { + println!("{k:8} {op} (1 in {})", done / k.max(&1)); + } + println!( + "run `./lean/differential.py {}/*` for the KNOWN-table breakdown", + save_dir.as_deref().unwrap_or("DIR") + ); + } + println!( + "{ok}/{done} inputs agree ({} divergences) in {elapsed:.2}s -> {:.0} inputs/s", + reports.len(), + done as f64 / elapsed + ); + if !reports.is_empty() { + std::process::exit(1); + } +} diff --git a/differential/src/reference/mod.rs b/differential/src/reference/mod.rs index 3e40c990..b65c5758 100644 --- a/differential/src/reference/mod.rs +++ b/differential/src/reference/mod.rs @@ -128,6 +128,11 @@ mod tests { /// reason. #[test] fn model_does_not_touch_the_crate() { + // Spelled in halves so this file, which is itself part of the model + // directory and is scanned like the rest, does not match its own needle. + let crate_path = ["path", "map::"].concat(); + let extern_crate = ["extern crate ", "pathmap"].concat(); + let own_map = ["crate::reference::path", "map"].concat(); let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/reference"); let mut offenders = Vec::new(); for entry in std::fs::read_dir(&dir).expect("src/reference must exist") { @@ -144,8 +149,8 @@ mod tests { } // The model's *own* map module is also called `pathmap`, so // strip its path before looking for the crate's. - let code = code.replace("crate::reference::pathmap", ""); - if code.contains("pathmap::") || code.contains("extern crate pathmap") { + let code = code.replace(&own_map, ""); + if code.contains(&crate_path) || code.contains(&extern_crate) { offenders.push(format!("{}:{}: {}", path.display(), n + 1, line.trim())); } } From 43e023f36755d28f5a87de3bee801114b1c1ac69 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 04:48:08 +0000 Subject: [PATCH 18/73] Name the trailer lines in the in-process fuzzer's own summary A divergence that first shows up in `MAP0`/`MAP1`/`ROOT0`/`ROOT1` has no step number, so reading the op name out of the second token printed the whole trie dump as the bucket label. Those lines carry their name in the *first* token instead, which is what "the step number parses as a number" distinguishes. Cosmetic: it only affects this binary's own per-op tally, not what it flags. The defect breakdown is still `./lean/differential.py`'s. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/bin/in_process.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/differential/src/bin/in_process.rs b/differential/src/bin/in_process.rs index ac571d25..1cf64bb0 100644 --- a/differential/src/bin/in_process.rs +++ b/differential/src/bin/in_process.rs @@ -161,11 +161,18 @@ fn compare(blob: &[u8], act: bool) -> Option { /// summary. Buckets by *op*, not by defect: the defect taxonomy is the `KNOWN` /// table in `lean/differential.py`, and there is one of it. fn first_diff_op(msg: &str) -> String { - msg.lines() + let line = msg + .lines() .find_map(|l| l.trim_start().strip_prefix("model: ")) - .and_then(|l| l.split_whitespace().nth(1)) - .unwrap_or("?") - .to_string() + .unwrap_or("?"); + let mut it = line.split_whitespace(); + match it.next() { + // An operation line is ` ret=...`; the trailer lines are + // `MAP0 ` and `ROOT0 `, whose own first token is the name. + Some(tok) if tok.parse::().is_ok() => it.next().unwrap_or("?").to_string(), + Some(tok) => tok.to_string(), + None => "?".to_string(), + } } fn main() { From acc1bca155f9e8c747596bf260d65a6f615d58af Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 04:52:28 +0000 Subject: [PATCH 19/73] Keep a dangling-child residue the classifier cannot see The revived in-process fuzzer turned up three divergences in 9,000,000 inputs that `differential.py`'s KNOWN table does not match. All three are the `meet_keeps_dangling` defect -- the crate keeps a valueless path an algebraic op should have dropped -- and the Lean and Rust models agree byte for byte on all three, so nothing about the models is in question. Two of them are invisible to `classify()` for a structural reason worth a corpus entry. Its DANGLING-KEPT rule keys on `child_count` or `val_count` moving on the differing *trace* line; here every one of the run's 69 steps agrees exactly, and the extra path appears only in the final `MAP0` dump, which has no such fields. So the defect is real, known, and reported as new. This is the smaller of the two, shrunk from 250 bytes to 65 by lean/shrink.py. The third stops being unclassifiable once shrunk -- the reduced input's diff is narrow enough for the existing rule -- so it needs no entry. `./lean/differential.py lean/corpus/*.bin` now exits non-zero, which the README explains: it means one entry is unclassified, not that the crate regressed. No CI job runs that glob. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- lean/corpus/README.md | 6 ++++++ lean/corpus/dangling-residue-only-in-map0.bin | Bin 0 -> 65 bytes 2 files changed, 6 insertions(+) create mode 100644 lean/corpus/dangling-residue-only-in-map0.bin diff --git a/lean/corpus/README.md b/lean/corpus/README.md index 5862cec2..49a083f5 100644 --- a/lean/corpus/README.md +++ b/lean/corpus/README.md @@ -9,10 +9,16 @@ of them reduce to a snippet small enough to write out as a Rust example. ./lean/shrink.py lean/corpus/ # minimise one further ``` +That exits non-zero, and is meant to: `dangling-residue-only-in-map0.bin` is here +precisely because `classify()` cannot recognise it, so it is reported as a new +divergence every time. The exit code says "one entry is unclassified", not "the +crate regressed". + | file | what it shows | | --- | --- | | `status-imprecise-join_map_into.bin` | `join_map_into` reports `Element` where the trie is provably unchanged (FINDINGS.md #8) | | `status-imprecise-restrict.bin` | `restrict` reports `Element` where the trie is provably unchanged (FINDINGS.md #8) | +| `dangling-residue-only-in-map0.bin` | a kept dangling child that no operation's fingerprint reveals: every one of the run's steps agrees, and the extra valueless path shows up only in the final `MAP0` dump. Same defect as `meet_keeps_dangling`, but `classify()` keys that one on `child_count`/`val_count` moving on the trace line, and a `MAP` line has neither — so this manifestation is reported as a new divergence | Everything else in FINDINGS.md has a standalone reproducer instead; see `cargo run -p differential --bin zipper_bug_repros -- --list`. diff --git a/lean/corpus/dangling-residue-only-in-map0.bin b/lean/corpus/dangling-residue-only-in-map0.bin new file mode 100644 index 0000000000000000000000000000000000000000..25d5d5b96cedee7e6eb1fbce66d8b7fb80adfef2 GIT binary patch literal 65 zcmZSBGL>Q9Yj4Jdii`{l45=W3fuWCq(dn8m1A{q`t;g_=IXaIynSl`~$H;hbBSR&S PV*Y)!m_fSo&U6L<* Date: Wed, 16 Sep 2026 04:55:26 +0000 Subject: [PATCH 20/73] Say where AFL actually files a divergence on a stock Linux box The first real run saved 0 crashes and 18 hangs, and all 18 hangs replay through `in_process` as genuine divergences. Nothing was wrong with the target: AFL decides "crashed" by reaping the child and reading its signal, and when /proc/sys/kernel/core_pattern is a pipe -- apport here, systemd-coredump elsewhere, a distro default either way -- the kernel hands the corpse to that helper first and AFL's wait times out instead. It warns about this at startup, and AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 silences the refusal to start without changing the outcome. So the docs now say to sweep `hangs/` as well as `crashes/`, and give the tell for a *genuine* timeout -- an infinite loop in the crate, which would be a finding in its own right: it is the input in `hangs/` that `in_process` does not flag. The root-only fix is recorded next to it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/bin/afl_differential.rs | 34 +++++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/differential/src/bin/afl_differential.rs b/differential/src/bin/afl_differential.rs index c6f1b5de..33561a3b 100644 --- a/differential/src/bin/afl_differential.rs +++ b/differential/src/bin/afl_differential.rs @@ -45,18 +45,38 @@ //! `in_process`'s panic-hook dance. //! //! A *divergence* is reported the same way a panic is — by panicking — so AFL -//! files it as a crash. Replay one with the plain comparator, which prints the +//! saves the input. Replay one with the plain comparator, which prints the //! differing line rather than a backtrace: //! //! ```text -//! target/release/in_process out/afl-out/default/crashes/id:000000* -//! ./lean/differential.py out/afl-out/default/crashes/* # KNOWN-table breakdown +//! target/release/in_process out/afl-out/default/crashes/id:* out/afl-out/default/hangs/id:* +//! ./lean/differential.py out/afl-out/default/crashes/* # KNOWN-table breakdown //! ``` //! -//! Note that `crashes/` will fill up with the *known* residual defects -//! (`meet_keeps_dangling` and friends) within the first minute, because to this -//! target they are indistinguishable from a new finding. Triage is -//! `differential.py`'s job: it owns the one `KNOWN` table. +//! # Look in `hangs/` as well as `crashes/` +//! +//! Which of the two a divergence lands in is a property of the *machine*, not of +//! the finding. AFL decides "crashed" by reaping the child and reading its +//! signal, and when `/proc/sys/kernel/core_pattern` is a pipe — apport, systemd +//! -coredump, any distro default — the kernel hands the corpse to that helper +//! first, so AFL's wait races the helper and times out instead. It says so at +//! startup ("To avoid having crashes misinterpreted as timeouts...") and +//! `AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1` only silences the refusal to start. +//! +//! Measured here: a 180s run saved **0 crashes and 18 hangs**, and all 18 hangs +//! replay through `in_process` as real divergences. So always sweep both +//! directories; a genuine timeout (an infinite loop in the crate, itself a +//! finding) is then the input in `hangs/` that `in_process` does *not* flag. +//! +//! `echo core | sudo tee /proc/sys/kernel/core_pattern` (or `cargo afl +//! system-config`) puts them back in `crashes/`, and needs root. +//! +//! # `crashes/` is not a list of new bugs +//! +//! It fills up with the *known* residual defects (`meet_keeps_dangling` and +//! friends) within the first minute, because to this target they are +//! indistinguishable from a new finding. Triage is `differential.py`'s job: it +//! owns the one `KNOWN` table. use differential::harness::run as crate_run; use differential::reference::fuzz::run as model_run; From 46da390c4ab3be5923d9a0c58536e3b9bbb656c7 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 05:24:01 +0000 Subject: [PATCH 21/73] Stop skipping the sibling-byte moves at the zipper root `to_next_sibling_byte` and `to_prev_sibling_byte` were skipped on both sides whenever the target zipper sat at its own root, because the native `ReadZipperCore` override guarded on `prefix_buf.len() == 0` -- the *absolute* path length -- and so walked out of the root of a zipper rooted at a non-empty path (FINDINGS.md #3). The override now guards on the zipper's own root. Directly: over read zippers rooted at `[]`, `[0]`, `[0,0]`, a non-existent `[0,0,1]`, an existing `[0,0,3]` and an absent `[9]`, and over write zippers at the same roots, both moves return `None` at the root and leave `origin_path()` untouched. That is what the model already specified, so the guard now only creates a blind spot. Lifted on all three sides at once and A/B'd against the same sweeps with the guard in place: 20M inputs at maxlen 120, 20M at 300, 8M at 600, and 8M each at 300/600 in ACT mode -- 56M in all -- reproduce the baseline divergence report byte for byte, and no trace anywhere carries `ESCAPED-ROOT`. Identity is the expected outcome rather than a coincidence: at the root the operation moves neither side, so lifting it changes no downstream state. `differential.py --model` stays at zero in both modes, and the Lean model against the crate reports no new divergence over 500k inputs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/harness.rs | 24 +++++++----------------- differential/src/reference/fuzz.rs | 25 +++++++------------------ lean/PathMapModel/Fuzz.lean | 22 +++++++++------------- lean/README.md | 13 ++++++++++++- 4 files changed, 35 insertions(+), 49 deletions(-) diff --git a/differential/src/harness.rs b/differential/src/harness.rs index 160b34be..b8e2781e 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -145,8 +145,6 @@ pub fn fingerprint + ZipperAbso /// * `skip:act` — the ACT read source cannot be a merge source /// (`ZipperInfallibleSubtries` is not implemented for it) or does not /// implement the trait the op needs. -/// * `skip:at-root` — `to_next`/`to_prev_sibling_byte` at the zipper root, -/// where the native read zipper escapes its own root. /// * `skip:k0` — a degenerate `k = 0`. /// * `skip:empty-focus` — the focus has nothing below it, where the op's /// behaviour is a function of node materialisation rather than trie state. @@ -157,7 +155,6 @@ pub fn fingerprint + ZipperAbso /// /// Each is recorded in lean/FINDINGS.md and commented at its site. pub const SKIP_ACT: &str = "skip:act"; -pub const SKIP_AT_ROOT: &str = "skip:at-root"; pub const SKIP_K0: &str = "skip:k0"; pub const SKIP_EMPTY_FOCUS: &str = "skip:empty-focus"; pub const SKIP_EMPTY_PATH: &str = "skip:empty-path"; @@ -549,23 +546,16 @@ pub fn run_ops( } 11 => { let t = get!(d.modn(2)); - // Skipped at the zipper root: the native ReadZipper escapes - // its own root there. See `Zip.toNextSiblingByte`. - if tgt!(t, wz, *rz, z, z.at_root()) { - ("to_next_sibling_byte", SKIP_AT_ROOT.to_string()) - } else { - let r = tgt!(t, wz, *rz, z, z.to_next_sibling_byte()); - ("to_next_sibling_byte", show_byte_opt(r)) - } + // Formerly skipped at the zipper root, where the native + // ReadZipper escaped its own root (FINDINGS.md #3). Fixed: + // the root case is now compared like any other. + let r = tgt!(t, wz, *rz, z, z.to_next_sibling_byte()); + ("to_next_sibling_byte", show_byte_opt(r)) } 12 => { let t = get!(d.modn(2)); - if tgt!(t, wz, *rz, z, z.at_root()) { - ("to_prev_sibling_byte", SKIP_AT_ROOT.to_string()) - } else { - let r = tgt!(t, wz, *rz, z, z.to_prev_sibling_byte()); - ("to_prev_sibling_byte", show_byte_opt(r)) - } + let r = tgt!(t, wz, *rz, z, z.to_prev_sibling_byte()); + ("to_prev_sibling_byte", show_byte_opt(r)) } 13 => { let t = get!(d.modn(2)); diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs index 2db64c34..436bf793 100644 --- a/differential/src/reference/fuzz.rs +++ b/differential/src/reference/fuzz.rs @@ -52,9 +52,6 @@ const OPS: U64Ops = U64Ops; /// The ACT read source cannot be a merge source (`ZipperInfallibleSubtries` is /// not implemented for it) or does not implement the trait the op needs. const SKIP_ACT: &str = "skip:act"; -/// `to_next`/`to_prev_sibling_byte` at the zipper root, where the native read -/// zipper escapes its own root. -const SKIP_AT_ROOT: &str = "skip:at-root"; /// A degenerate `k = 0`. const SKIP_K0: &str = "skip:k0"; /// The focus has nothing below it, where the op's behaviour is a function of @@ -324,25 +321,17 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { s.emit("ascend_until_branch", &r.to_string()); } 11 => { - // Skipped at the zipper root: `ReadZipper::to_next_sibling_byte` - // escapes its own root there (see the notes in - // `Zip::to_next_sibling_byte`). + // Formerly skipped at the zipper root, where + // `ReadZipper::to_next_sibling_byte` escaped its own root + // (FINDINGS.md #3). Fixed: the root case is compared like any other. let t = d.modn(2)?; - if s.target_ref(t).at_root() { - s.emit("to_next_sibling_byte", SKIP_AT_ROOT); - } else { - let r = s.target(t).to_next_sibling_byte(); - s.emit("to_next_sibling_byte", &show_byte_opt(r)); - } + let r = s.target(t).to_next_sibling_byte(); + s.emit("to_next_sibling_byte", &show_byte_opt(r)); } 12 => { let t = d.modn(2)?; - if s.target_ref(t).at_root() { - s.emit("to_prev_sibling_byte", SKIP_AT_ROOT); - } else { - let r = s.target(t).to_prev_sibling_byte(); - s.emit("to_prev_sibling_byte", &show_byte_opt(r)); - } + let r = s.target(t).to_prev_sibling_byte(); + s.emit("to_prev_sibling_byte", &show_byte_opt(r)); } 13 => { let t = d.modn(2)?; diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index 67e8ad5d..a5ea4263 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -60,8 +60,6 @@ agree exactly or every input with a skip diverges. * `skip:act` — the ACT read source cannot be a merge source (`ZipperInfallibleSubtries` is not implemented for it) or does not implement the trait the op needs. -* `skip:at-root` — `to_next`/`to_prev_sibling_byte` at the zipper root, where - the native read zipper escapes its own root. * `skip:k0` — a degenerate `k = 0`. * `skip:empty-focus` — the focus has nothing below it, where the op's behaviour is a function of node materialisation rather than trie state. @@ -73,7 +71,6 @@ agree exactly or every input with a skip diverges. Each is recorded in FINDINGS.md and commented at its site. -/ def skipAct : String := "skip:act" -def skipAtRoot : String := "skip:at-root" def skipK0 : String := "skip:k0" def skipEmptyFocus : String := "skip:empty-focus" def skipEmptyPath : String := "skip:empty-path" @@ -290,17 +287,16 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do let (r, s) := onTarget s t (fun z => z.ascendUntilBranch) some (emit s "ascend_until_branch" (toString r), d) | 11 => do let (t, d) ← d.mod 2 - -- Skipped at the zipper root: `ReadZipper::to_next_sibling_byte` - -- escapes its own root there (see the notes in `Zip.toNextSiblingByte`). - if (getTarget s t).atRoot then some (emit s "to_next_sibling_byte" skipAtRoot, d) - else - let (r, s) := onTarget s t (fun z => z.toNextSiblingByte) - some (emit s "to_next_sibling_byte" (showByteOpt r), d) + -- Formerly skipped at the zipper root, where `ReadZipper:: + -- to_next_sibling_byte` used to escape its own root + -- (FINDINGS.md #3). That is fixed: the override now guards on + -- `at_root()`, returns `None` there and leaves `origin_path()` + -- alone, so the root case is compared like any other. + let (r, s) := onTarget s t (fun z => z.toNextSiblingByte) + some (emit s "to_next_sibling_byte" (showByteOpt r), d) | 12 => do let (t, d) ← d.mod 2 - if (getTarget s t).atRoot then some (emit s "to_prev_sibling_byte" skipAtRoot, d) - else - let (r, s) := onTarget s t (fun z => z.toPrevSiblingByte) - some (emit s "to_prev_sibling_byte" (showByteOpt r), d) + let (r, s) := onTarget s t (fun z => z.toPrevSiblingByte) + some (emit s "to_prev_sibling_byte" (showByteOpt r), d) | 13 => do let (t, d) ← d.mod 2 let (r, s) := onTarget s t (fun z => z.toNextStep) some (emit s "to_next_step" (showBool r), d) diff --git a/lean/README.md b/lean/README.md index f3a0cb43..a672f194 100644 --- a/lean/README.md +++ b/lean/README.md @@ -332,11 +332,22 @@ skips diverges: | `skip:k0` | `meet_k_path_into(0)`, `join_k_path_into(0)`, `descend_first_k_path(0)` / `to_next_k_path(0)` — degenerate; the first two should be the identity and destroy the subtrie, the last reports success without moving, forever. | | `skip:empty-focus` | `meet_k_path_into` with no children (it does not terminate), and `restricting` when either side has nothing below its focus (the two branches differ in *effect*, not just in the reported bool). | | `skip:empty-path` | `insert_prefix("")` — should be the identity, destroys the subtrie. | -| `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root — the native read zipper leaves its own root there. | | `skip:off-root-prune` | `prune_path` / `prune_ascend`, and the `prune` flag on every other operation, for a write zipper not rooted at the map root — the depth pruned is a function of internal node layout, so there is nothing to specify. | | `skip:quarantined` | `graft_child_maps` (op 54), disabled outright: it is broken three ways (FINDINGS.md #15) and the node representations it leaves behind degrade the `AlgebraicStatus` that *later* operations report. | | `skip:act` | ACT mode only — the read source cannot be a merge source (`ZipperInfallibleSubtries` is not implemented for it) or does not implement the trait the op needs. | +#### Suppressions that have been lifted + +A skip is only worth keeping while the defect behind it is live. These were +re-tested against the current crate, found stale, and removed — the operation is +now compared like any other. Each was A/B'd by lifting the guard on all three +sides at once and re-running the same sweeps: the divergence report has to come +back unchanged, since a fixed operation changes no state. + +| token | why it is gone | +|---|---| +| `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root (FINDINGS.md #3). The override now guards on `at_root()`: it returns `None` and leaves `origin_path()` alone, for a read zipper and a write zipper alike, whether or not the root exists. A/B over 48M inputs (maxlen 120/300/600, crate and ACT) reproduced the baseline divergence report byte for byte, with no `ESCAPED-ROOT` anywhere. | + Naming these turned up an ordering bug the bare token had hidden: for `restricting` the model tested ACT mode first and the harness tested the empty focus first, so in ACT mode with an empty focus the two took different branches From 29da7ec422a4cc1051c3b5b271c24335c2300b59 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 05:31:11 +0000 Subject: [PATCH 22/73] Stop skipping insert_prefix with an empty prefix `insert_prefix("")` was skipped on both sides because `make_parents_in(b"", node)` discarded the node rather than doing nothing, destroying the subtrie below the focus and still returning `true` (FINDINGS.md #4). That was fixed upstream -- not by this effort -- and the fix carries its own regression test, `write_zipper_insert_prefix_empty_is_identity`. Directly: over seven trie shapes (empty, root value only, a single line, branchy, wide, one with a dangling path, one built by grafting so the node boundaries fall elsewhere), at the map root and at a non-root focus, `insert_prefix(&[])` leaves the trie byte for byte unchanged in all fourteen. The return value needed no special case either: `false` in exactly the shapes where the focus has nothing below it, which is what `Zip.insertPrefix` already specified. Same A/B as the previous commit -- 20M inputs at maxlen 120, 20M at 300, 8M at 600, 8M each at 300/600 in ACT mode -- reproduces the baseline divergence report byte for byte. As with the root case this is the expected outcome rather than luck: an identity operation changes no downstream state, so the programs either side of the lift are the same programs. `differential.py --model` stays at zero in both modes; the Lean model against the crate reports no new divergence over 500k inputs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/harness.rs | 12 ++++-------- differential/src/reference/fuzz.rs | 14 +++++--------- lean/PathMapModel/Fuzz.lean | 15 ++++++--------- lean/README.md | 4 ++-- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/differential/src/harness.rs b/differential/src/harness.rs index b8e2781e..6a932aa0 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -148,7 +148,6 @@ pub fn fingerprint + ZipperAbso /// * `skip:k0` — a degenerate `k = 0`. /// * `skip:empty-focus` — the focus has nothing below it, where the op's /// behaviour is a function of node materialisation rather than trie state. -/// * `skip:empty-path` — `insert_prefix("")`, which destroys the subtrie. /// * `skip:off-root-prune` — a prune on a write zipper not rooted at the map /// root, where the depth pruned is a function of internal node layout. /// * `skip:quarantined` — the op is disabled outright (op 54). @@ -157,7 +156,6 @@ pub fn fingerprint + ZipperAbso pub const SKIP_ACT: &str = "skip:act"; pub const SKIP_K0: &str = "skip:k0"; pub const SKIP_EMPTY_FOCUS: &str = "skip:empty-focus"; -pub const SKIP_EMPTY_PATH: &str = "skip:empty-path"; pub const SKIP_OFF_ROOT_PRUNE: &str = "skip:off-root-prune"; pub const SKIP_QUARANTINED: &str = "skip:quarantined"; @@ -787,12 +785,10 @@ pub fn run_ops( } 43 => { let p = get!(d.path(6)); - // `insert_prefix("")` destroys the subtrie in pathmap 0.3.1. - if p.is_empty() { - ("insert_prefix", SKIP_EMPTY_PATH.to_string()) - } else { - ("insert_prefix", show_bool(wz.insert_prefix(&p)).to_string()) - } + // The empty prefix was skipped while it destroyed the + // subtrie (FINDINGS.md #4); fixed upstream, so it is + // compared like any other prefix. + ("insert_prefix", show_bool(wz.insert_prefix(&p)).to_string()) } 44 => { let n = get!(d.modn(6)); diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs index 436bf793..cf61f4d6 100644 --- a/differential/src/reference/fuzz.rs +++ b/differential/src/reference/fuzz.rs @@ -57,8 +57,6 @@ const SKIP_K0: &str = "skip:k0"; /// The focus has nothing below it, where the op's behaviour is a function of /// node materialisation rather than of trie state. const SKIP_EMPTY_FOCUS: &str = "skip:empty-focus"; -/// `insert_prefix("")`, which destroys the subtrie. -const SKIP_EMPTY_PATH: &str = "skip:empty-path"; /// A prune on a write zipper not rooted at the map root, where the depth pruned /// is a function of internal node layout. const SKIP_OFF_ROOT_PRUNE: &str = "skip:off-root-prune"; @@ -581,13 +579,11 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { } 43 => { let p = d.path(6)?; - // `insert_prefix("")` destroys the subtrie in pathmap 0.3.1. - if p.is_empty() { - s.emit("insert_prefix", SKIP_EMPTY_PATH); - } else { - let r = s.wz.insert_prefix(&p); - s.emit("insert_prefix", show_bool(r)); - } + // The empty prefix was skipped while it destroyed the subtrie + // (FINDINGS.md #4); fixed upstream, so it is compared like any + // other prefix. + let r = s.wz.insert_prefix(&p); + s.emit("insert_prefix", show_bool(r)); } 44 => { let n = d.modn(6)?; diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index a5ea4263..96b5a352 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -63,7 +63,6 @@ agree exactly or every input with a skip diverges. * `skip:k0` — a degenerate `k = 0`. * `skip:empty-focus` — the focus has nothing below it, where the op's behaviour is a function of node materialisation rather than trie state. -* `skip:empty-path` — `insert_prefix("")`, which destroys the subtrie. * `skip:off-root-prune` — a prune on a write zipper not rooted at the map root, where the depth pruned is a function of internal node layout. * `skip:quarantined` — the op is disabled outright (op 54). @@ -73,7 +72,6 @@ Each is recorded in FINDINGS.md and commented at its site. -/ def skipAct : String := "skip:act" def skipK0 : String := "skip:k0" def skipEmptyFocus : String := "skip:empty-focus" -def skipEmptyPath : String := "skip:empty-path" def skipOffRootPrune : String := "skip:off-root-prune" def skipQuarantined : String := "skip:quarantined" @@ -444,13 +442,12 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do some (emit { s with wz := z } "join_k_path_into" (if z.focusNodeIsEmpty then "?" else showBool r), d) | 43 => do let (p, d) ← d.path - -- `insert_prefix("")` destroys the subtrie in pathmap 0.3.1; see - -- `Zip.insertPrefix`. Skipped so the known bug does not mask others. - if p.isEmpty then - some (emit s "insert_prefix" skipEmptyPath, d) - else - let (r, z) := s.wz.insertPrefix p - some (emit { s with wz := z } "insert_prefix" (showBool r), d) + -- The empty prefix was skipped while `make_parents_in(b"", node)` + -- discarded the node instead of doing nothing (FINDINGS.md #4). + -- That is fixed upstream, with a regression test of its own, so + -- the empty prefix is compared like any other. + let (r, z) := s.wz.insertPrefix p + some (emit { s with wz := z } "insert_prefix" (showBool r), d) | 44 => do let (n, d) ← d.mod 6 let (r, z) := s.wz.removePrefix n some (emit { s with wz := z } "remove_prefix" (showBool r), d) diff --git a/lean/README.md b/lean/README.md index a672f194..a9e00a97 100644 --- a/lean/README.md +++ b/lean/README.md @@ -331,7 +331,6 @@ skips diverges: |---|---| | `skip:k0` | `meet_k_path_into(0)`, `join_k_path_into(0)`, `descend_first_k_path(0)` / `to_next_k_path(0)` — degenerate; the first two should be the identity and destroy the subtrie, the last reports success without moving, forever. | | `skip:empty-focus` | `meet_k_path_into` with no children (it does not terminate), and `restricting` when either side has nothing below its focus (the two branches differ in *effect*, not just in the reported bool). | -| `skip:empty-path` | `insert_prefix("")` — should be the identity, destroys the subtrie. | | `skip:off-root-prune` | `prune_path` / `prune_ascend`, and the `prune` flag on every other operation, for a write zipper not rooted at the map root — the depth pruned is a function of internal node layout, so there is nothing to specify. | | `skip:quarantined` | `graft_child_maps` (op 54), disabled outright: it is broken three ways (FINDINGS.md #15) and the node representations it leaves behind degrade the `AlgebraicStatus` that *later* operations report. | | `skip:act` | ACT mode only — the read source cannot be a merge source (`ZipperInfallibleSubtries` is not implemented for it) or does not implement the trait the op needs. | @@ -346,7 +345,8 @@ back unchanged, since a fixed operation changes no state. | token | why it is gone | |---|---| -| `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root (FINDINGS.md #3). The override now guards on `at_root()`: it returns `None` and leaves `origin_path()` alone, for a read zipper and a write zipper alike, whether or not the root exists. A/B over 48M inputs (maxlen 120/300/600, crate and ACT) reproduced the baseline divergence report byte for byte, with no `ESCAPED-ROOT` anywhere. | +| `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root (FINDINGS.md #3). The override now guards on `at_root()`: it returns `None` and leaves `origin_path()` alone, for a read zipper and a write zipper alike, whether or not the root exists. A/B over 56M inputs (maxlen 120/300/600, crate and ACT) reproduced the baseline divergence report byte for byte, with no `ESCAPED-ROOT` anywhere. | +| `skip:empty-path` | `insert_prefix("")` (FINDINGS.md #4). Fixed upstream, with the regression test `write_zipper_insert_prefix_empty_is_identity`. Directly: over seven trie shapes (empty, root value only, one line, branchy, wide, dangling, grafted), at the map root and at a non-root focus, `insert_prefix(&[])` leaves the trie untouched and returns exactly what the model's `focusNodeIsEmpty` rule predicts. Same 56M-input A/B, again byte for byte identical. | Naming these turned up an ordering bug the bare token had hidden: for `restricting` the model tested ACT mode first and the harness tested the empty From 470eea8af90766ec324a8be577d98125e558cb25 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 05:42:36 +0000 Subject: [PATCH 23/73] Compare restrict's status instead of masking it `restrict` was one of five return values masked to `?` whenever the write zipper's focus had nothing below it. The reason was FINDINGS.md #8: those returns branch on `get_focus().is_none()`, which is false at a location where `create_path` or `remove_val` happened to leave an empty node materialised and true at one that only ever carried a value. The model cannot see the difference, so the status was not compared there. It is compared now. Unmasked, over 20M inputs at maxlen 120, 20M at 300 and 8M at 600, the divergence report is byte for byte the one the mask produced -- no `restrict` line differs anywhere. The mask is not a branch that never runs: tracing 1500 inputs, 824 of 947 `restrict` calls hit it, so this is on the order of 30M statuses compared rather than skipped. (ACT mode is unaffected: `restrict` is `skip:act` there, so the ACT sweeps say nothing about it.) The other four stay, and the same experiment is why: unmasked, `remove_branches`, `join_map_into` and `take_map` each diverge on roughly one input in eight, immediately. Whatever decides `restrict`'s status, it is not the materialisation bit that decides theirs. This is evidence that the leak does not reach `restrict`, not a proof that it cannot; the commit is deliberately separate so it can be reverted alone if a counterexample turns up. `differential.py --model` stays at zero in both modes; the Lean model against the crate reports no new divergence over 500k inputs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/harness.rs | 17 +++++++---------- differential/src/reference/fuzz.rs | 5 +++-- lean/PathMapModel/Fuzz.lean | 9 ++++++--- lean/README.md | 19 ++++++++++++++----- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/differential/src/harness.rs b/differential/src/harness.rs index 6a932aa0..70f8d20f 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -162,9 +162,9 @@ pub const SKIP_QUARANTINED: &str = "skip:quarantined"; /// Does the focus have no descendants at all? /// /// Several return values (`remove_branches`, `restricting`, `join_map_into`, -/// `take_map`, `restrict`) hinge on whether an *empty node* happens to be -/// materialised at the focus rather than on the logical state, so the harness -/// masks them here. See lean/README.md. +/// `take_map`) hinge on whether an *empty node* happens to be materialised at +/// the focus rather than on the logical state, so the harness masks them here. +/// `restrict` used to be in that list and no longer is. See lean/README.md. pub fn focus_node_empty(z: &Z) -> bool { z.child_count() == 0 } @@ -745,14 +745,11 @@ pub fn run_ops( ) } 40 => { - let leaky = focus_node_empty(&wz); + // The status used to be masked to `?` at a focus with + // nothing below it (FINDINGS.md #8). No longer: it tracks + // the spec over the whole sweep. let st = (*rz).do_restrict(&mut wz); - let s = if leaky && st.is_some() { - "?".to_string() - } else { - show_status_opt(st) - }; - ("restrict", s) + ("restrict", show_status_opt(st)) } 41 => { // Skipped when either side has nothing below its focus; see diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs index cf61f4d6..ca28e1ae 100644 --- a/differential/src/reference/fuzz.rs +++ b/differential/src/reference/fuzz.rs @@ -535,9 +535,10 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { if s.act { s.emit("restrict", SKIP_ACT); } else { - let leaky = s.wz.focus_node_is_empty(); + // Formerly masked to `?` at an empty focus (FINDINGS.md #8); + // the crate's status tracks the spec, so it is compared. let st = s.wz.restrict(&OPS, &s.rz); - let ret = if leaky { "?".to_string() } else { show_status(st) }; + let ret = show_status(st); s.emit("restrict", &ret); } } diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index 96b5a352..bf5676dc 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -410,10 +410,13 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do some (emit { s with wz := z } "subtract_into" (toString st), d) | 40 => do if s.act then some (emit s "restrict" skipAct, d) else do - let leaky := s.wz.focusNodeIsEmpty + -- The status used to be masked to `?` at a focus with nothing + -- below it, as one of the node-materialisation leaks in + -- FINDINGS.md #8. It is no longer: over 48M inputs, of which + -- roughly seven in eight reach this op with an empty focus, the + -- crate's status matches the spec every time. let (st, z) := s.wz.restrict ops s.rz - some (emit { s with wz := z } "restrict" - (if leaky then "?" else toString st), d) + some (emit { s with wz := z } "restrict" (toString st), d) -- Skipped, not merely masked, when either side has nothing below -- its focus: there `restricting` branches on whether an empty node -- happens to be materialised, and the two branches differ in diff --git a/lean/README.md b/lean/README.md index a9e00a97..f5eb4756 100644 --- a/lean/README.md +++ b/lean/README.md @@ -359,11 +359,20 @@ the harness's order. `k_path_internal` carries iteration state and `pathmap`'s own debug assertions flag calling it cold. -Five return values are compared as `?` when the focus has no descendants -(`remove_branches`, `join_map_into`, `restrict`, `restricting`, `take_map`): -they report on whether an empty node happens to be materialised at the focus, -which is representation state rather than trie state. The *effects* are still -compared in full. +Some return values are compared as `?` when the focus has no descendants +(`remove_branches`, `join_map_into`, `take_map`, and `join_k_path_into` on its +result): they report on whether an empty node happens to be materialised at the +focus, which is representation state rather than trie state. The *effects* are +still compared in full. Each was re-tested by unmasking it alone and re-running +the sweeps; `remove_branches`, `join_map_into` and `take_map` diverge on roughly +one input in eight the moment the mask comes off, which is what keeps them. + +`restrict` was on that list and is not any more. Unmasked, its status matched +the specification over the full 48M-input crate-mode sweep (it is `skip:act` in +ACT mode, so ACT does not exercise it), and the mask fires on about seven of +every eight calls, so that is 30M-odd compared statuses rather than a branch +that never runs. That is evidence the leak does not reach `restrict`, not a +proof that it cannot. ## The blind-zipper contract From f99f40d661f0538349ae01d22505be0f12027514 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 05:57:36 +0000 Subject: [PATCH 24/73] Compare join_k_path_into's bool instead of masking it `join_k_path_into` (for k > 0) had its return masked to `?` whenever the collapse left the focus with no children. The stated reason was another `AbstractNodeRef` leak: an empty node coming back as `Some(...)` from `into_option()` for some representations, so `true` would be reported for a collapse that produced nothing. The sweep does not bear that out. Unmasked, over 20M inputs at maxlen 120, 20M at 300, 8M at 600, and 8M each at 300/600 in ACT mode -- 56M in all -- the divergence report is byte for byte the one the mask produced; no `join_k_path_into` line differs anywhere. The mask is not a branch that never runs: tracing 1500 inputs, 678 of 1003 calls hit it. `k = 0` is untouched and stays `skip:k0`. That one is a live bug, not a suspicion: `join_k_path_into(0)` still destroys the subtrie at HEAD -- on `{[] = 0, [0] = 0, [0,0] = 0, [1,0] = 0}` it returns `true` and leaves `{[] = 0, [0] = 0}`, which is `zipper_bug_repros drop_head_zero` reproducing exactly as FINDINGS.md #5 describes it. As with `restrict`, this is evidence that the leak does not reach this return, not a proof that it cannot, and it is a separate commit so it can be reverted alone. `differential.py --model` stays at zero in both modes; the Lean model against the crate reports no new divergence over 500k inputs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/harness.rs | 12 +++++------- differential/src/reference/fuzz.rs | 9 ++++----- lean/PathMapModel/Fuzz.lean | 13 ++++++------- lean/README.md | 30 ++++++++++++++++-------------- 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/differential/src/harness.rs b/differential/src/harness.rs index 70f8d20f..97b5706f 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -770,14 +770,12 @@ pub fn run_ops( if k == 0 { ("join_k_path_into", SKIP_K0.to_string()) } else { - // The bool leaks node materialisation; see FINDINGS.md #8. + // The bool was masked to `?` on an empty resulting + // focus as a suspected materialisation leak + // (FINDINGS.md #8). It is not one: unmasked it tracks + // the spec over the whole sweep. let r = wz.join_k_path_into(k, no_prune); - let s = if focus_node_empty(&wz) { - "?".to_string() - } else { - show_bool(r).to_string() - }; - ("join_k_path_into", s) + ("join_k_path_into", show_bool(r).to_string()) } } 43 => { diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs index ca28e1ae..f43c7cd6 100644 --- a/differential/src/reference/fuzz.rs +++ b/differential/src/reference/fuzz.rs @@ -569,12 +569,11 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { if k == 0 { s.emit("join_k_path_into", SKIP_K0); } else { - // The bool is another `AbstractNodeRef` leak: an empty node still - // comes back as `Some(...)` from `into_option()` for some - // representations. Compared only when something survived. + // The bool was masked to `?` on an empty resulting focus as a + // suspected `AbstractNodeRef` leak (FINDINGS.md #8). It is not + // one: unmasked it tracks the spec over the whole sweep. let r = s.wz.join_k_path_into(&OPS, k, NO_PRUNE); - let ret = - if s.wz.focus_node_is_empty() { "?".to_string() } else { show_bool(r).to_string() }; + let ret = show_bool(r).to_string(); s.emit("join_k_path_into", &ret); } } diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index bf5676dc..fe622103 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -436,14 +436,13 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do -- subtrie in pathmap 0.3.1; see `Zip.joinKPathInto`. if k == 0 then some (emit s "join_k_path_into" skipK0, d) else - -- The bool is another `AbstractNodeRef` leak: an empty node still - -- comes back as `Some(...)` from `into_option()` for some - -- representations, so `true` gets reported for a collapse that - -- produced nothing. Compared only when something survived. - -- See FINDINGS.md #8. + -- The bool used to be masked to `?` when the collapse left an + -- empty focus, on the theory that it was another + -- `AbstractNodeRef` leak (FINDINGS.md #8). Unmasked it tracks + -- the spec: over 56M inputs, with the mask firing on about two + -- calls in three, no `join_k_path_into` line differs. let (r, z) := s.wz.joinKPathInto ops k noPrune - some (emit { s with wz := z } "join_k_path_into" - (if z.focusNodeIsEmpty then "?" else showBool r), d) + some (emit { s with wz := z } "join_k_path_into" (showBool r), d) | 43 => do let (p, d) ← d.path -- The empty prefix was skipped while `make_parents_in(b"", node)` -- discarded the node instead of doing nothing (FINDINGS.md #4). diff --git a/lean/README.md b/lean/README.md index f5eb4756..35a85a4b 100644 --- a/lean/README.md +++ b/lean/README.md @@ -359,20 +359,22 @@ the harness's order. `k_path_internal` carries iteration state and `pathmap`'s own debug assertions flag calling it cold. -Some return values are compared as `?` when the focus has no descendants -(`remove_branches`, `join_map_into`, `take_map`, and `join_k_path_into` on its -result): they report on whether an empty node happens to be materialised at the -focus, which is representation state rather than trie state. The *effects* are -still compared in full. Each was re-tested by unmasking it alone and re-running -the sweeps; `remove_branches`, `join_map_into` and `take_map` diverge on roughly -one input in eight the moment the mask comes off, which is what keeps them. - -`restrict` was on that list and is not any more. Unmasked, its status matched -the specification over the full 48M-input crate-mode sweep (it is `skip:act` in -ACT mode, so ACT does not exercise it), and the mask fires on about seven of -every eight calls, so that is 30M-odd compared statuses rather than a branch -that never runs. That is evidence the leak does not reach `restrict`, not a -proof that it cannot. +Three return values are compared as `?` when the focus has no descendants +(`remove_branches`, `join_map_into`, `take_map`): they report on whether an +empty node happens to be materialised at the focus, which is representation +state rather than trie state. The *effects* are still compared in full. Each +was re-tested by unmasking it alone and re-running the sweeps, and each diverges +on roughly one input in eight the moment the mask comes off, which is what keeps +it. + +`restrict` and `join_k_path_into` were on that list and are not any more. +Unmasked, both match the specification over the whole sweep — 20M inputs at +maxlen 120, 20M at 300, 8M at 600, and for `join_k_path_into` 8M each at +300/600 in ACT mode as well; `restrict` is `skip:act`, so ACT says nothing +about it. Neither is a branch that never runs: tracing 1500 inputs, the mask +fired on 824 of 947 `restrict` calls and 678 of 1003 `join_k_path_into` calls. +That is evidence the materialisation leak does not reach these two, not a proof +that it cannot; each was lifted in its own commit so it can be put back alone. ## The blind-zipper contract From 33bde3abfb648211e2bd41a40d1e1cdc88facafc Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 05:58:41 +0000 Subject: [PATCH 25/73] Drop the skip:quarantined vocabulary, which nothing declines by any more `skip:quarantined` meant "this op is disabled outright", and the only op it ever applied to was 54, `graft_child_maps`. That was un-quarantined when its bug was fixed, and the token went with it -- but the constant stayed defined in all three op tables, the reason stayed listed in all three doc blocks, and the README still described a rule the fuzzer cannot fire. No call site remains on any side: `skipQuarantined` in `Fuzz.lean`, `SKIP_QUARANTINED` in `harness.rs` and the `#[allow(dead_code)]` one in `reference/fuzz.rs` are each referenced exactly once, by their own definition. The Rust one had to be marked dead to compile at all, which is the compiler saying the same thing. A skip vocabulary is a contract between three transcriptions, so a token in it that nothing can emit is not harmless: it invites the next reader to keep a suppression alive by analogy with one that is already gone. Removed from all three, and from the README's table. No behaviour changes -- the token appears in no trace before or after -- so the sweeps are untouched by construction. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/harness.rs | 2 -- differential/src/reference/fuzz.rs | 6 ------ lean/PathMapModel/Fuzz.lean | 2 -- lean/README.md | 2 +- 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/differential/src/harness.rs b/differential/src/harness.rs index 97b5706f..f6dcd554 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -150,14 +150,12 @@ pub fn fingerprint + ZipperAbso /// behaviour is a function of node materialisation rather than trie state. /// * `skip:off-root-prune` — a prune on a write zipper not rooted at the map /// root, where the depth pruned is a function of internal node layout. -/// * `skip:quarantined` — the op is disabled outright (op 54). /// /// Each is recorded in lean/FINDINGS.md and commented at its site. pub const SKIP_ACT: &str = "skip:act"; pub const SKIP_K0: &str = "skip:k0"; pub const SKIP_EMPTY_FOCUS: &str = "skip:empty-focus"; pub const SKIP_OFF_ROOT_PRUNE: &str = "skip:off-root-prune"; -pub const SKIP_QUARANTINED: &str = "skip:quarantined"; /// Does the focus have no descendants at all? /// diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs index f43c7cd6..d7a21444 100644 --- a/differential/src/reference/fuzz.rs +++ b/differential/src/reference/fuzz.rs @@ -60,12 +60,6 @@ const SKIP_EMPTY_FOCUS: &str = "skip:empty-focus"; /// A prune on a write zipper not rooted at the map root, where the depth pruned /// is a function of internal node layout. const SKIP_OFF_ROOT_PRUNE: &str = "skip:off-root-prune"; -/// The op is disabled outright. Nothing uses this today: op 54 -/// (`graft_child_maps`) was quarantined when the archive was taken and has since -/// been let back in. Kept because `Fuzz.lean` and `harness.rs` both still -/// define it, and the vocabulary is the contract. -#[allow(dead_code)] -const SKIP_QUARANTINED: &str = "skip:quarantined"; // --------------------------------------------------------------------------- // Decoder — `Fuzz.Dec` diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index fe622103..c5094223 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -65,7 +65,6 @@ agree exactly or every input with a skip diverges. is a function of node materialisation rather than trie state. * `skip:off-root-prune` — a prune on a write zipper not rooted at the map root, where the depth pruned is a function of internal node layout. -* `skip:quarantined` — the op is disabled outright (op 54). Each is recorded in FINDINGS.md and commented at its site. -/ @@ -73,7 +72,6 @@ def skipAct : String := "skip:act" def skipK0 : String := "skip:k0" def skipEmptyFocus : String := "skip:empty-focus" def skipOffRootPrune : String := "skip:off-root-prune" -def skipQuarantined : String := "skip:quarantined" /-! ## Rendering -/ diff --git a/lean/README.md b/lean/README.md index 35a85a4b..33b5caed 100644 --- a/lean/README.md +++ b/lean/README.md @@ -332,7 +332,6 @@ skips diverges: | `skip:k0` | `meet_k_path_into(0)`, `join_k_path_into(0)`, `descend_first_k_path(0)` / `to_next_k_path(0)` — degenerate; the first two should be the identity and destroy the subtrie, the last reports success without moving, forever. | | `skip:empty-focus` | `meet_k_path_into` with no children (it does not terminate), and `restricting` when either side has nothing below its focus (the two branches differ in *effect*, not just in the reported bool). | | `skip:off-root-prune` | `prune_path` / `prune_ascend`, and the `prune` flag on every other operation, for a write zipper not rooted at the map root — the depth pruned is a function of internal node layout, so there is nothing to specify. | -| `skip:quarantined` | `graft_child_maps` (op 54), disabled outright: it is broken three ways (FINDINGS.md #15) and the node representations it leaves behind degrade the `AlgebraicStatus` that *later* operations report. | | `skip:act` | ACT mode only — the read source cannot be a merge source (`ZipperInfallibleSubtries` is not implemented for it) or does not implement the trait the op needs. | #### Suppressions that have been lifted @@ -347,6 +346,7 @@ back unchanged, since a fixed operation changes no state. |---|---| | `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root (FINDINGS.md #3). The override now guards on `at_root()`: it returns `None` and leaves `origin_path()` alone, for a read zipper and a write zipper alike, whether or not the root exists. A/B over 56M inputs (maxlen 120/300/600, crate and ACT) reproduced the baseline divergence report byte for byte, with no `ESCAPED-ROOT` anywhere. | | `skip:empty-path` | `insert_prefix("")` (FINDINGS.md #4). Fixed upstream, with the regression test `write_zipper_insert_prefix_empty_is_identity`. Directly: over seven trie shapes (empty, root value only, one line, branchy, wide, dangling, grafted), at the map root and at a non-root focus, `insert_prefix(&[])` leaves the trie untouched and returns exactly what the model's `focusNodeIsEmpty` rule predicts. Same 56M-input A/B, again byte for byte identical. | +| `skip:quarantined` | `graft_child_maps` (op 54), the only op it ever named, was un-quarantined when its bug was fixed. The token outlived it as a definition with no call site on any of the three sides — the Rust one needed `#[allow(dead_code)]` to compile. Removed rather than kept: a vocabulary entry nothing can emit invites the next reader to keep a live suppression alive by analogy with it. | Naming these turned up an ordering bug the bare token had hidden: for `restricting` the model tested ACT mode first and the harness tested the empty From 6c065a4e5e580c185f3f503616e84caee4aadbc3 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 06:01:28 +0000 Subject: [PATCH 26/73] Correct the input counts the lifted-suppression notes claim The four commits that lifted a suppression each A/B'd it over five sweeps -- 20M inputs at maxlen 120, 20M at 300 and 8M at 600 in crate mode, plus 8M each at 300 and 600 in ACT mode. That is 64M, and three of the notes say 56M: I added the five figures up wrong and the mistake was copied from the first commit message into the next two and into the README. The sweeps themselves are unchanged; only the arithmetic describing them was wrong, and it understated rather than overstated. `restrict`'s figure of 48M was already right -- it is `skip:act`, so the two ACT sweeps say nothing about it and its evidence is the crate-mode 48M alone. While here, the note about how much the two lifted masks actually exercised is made concrete rather than impressionistic: at the measured hit rates (824 of 947 `restrict` calls and 678 of 1003 `join_k_path_into` calls over 1500 traced inputs) the sweeps compared on the order of 26M and 29M returns that used to be reported as `?`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- lean/PathMapModel/Fuzz.lean | 2 +- lean/README.md | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index c5094223..09265559 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -437,7 +437,7 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do -- The bool used to be masked to `?` when the collapse left an -- empty focus, on the theory that it was another -- `AbstractNodeRef` leak (FINDINGS.md #8). Unmasked it tracks - -- the spec: over 56M inputs, with the mask firing on about two + -- the spec: over 64M inputs, with the mask firing on about two -- calls in three, no `join_k_path_into` line differs. let (r, z) := s.wz.joinKPathInto ops k noPrune some (emit { s with wz := z } "join_k_path_into" (showBool r), d) diff --git a/lean/README.md b/lean/README.md index 33b5caed..e93231b5 100644 --- a/lean/README.md +++ b/lean/README.md @@ -344,8 +344,8 @@ back unchanged, since a fixed operation changes no state. | token | why it is gone | |---|---| -| `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root (FINDINGS.md #3). The override now guards on `at_root()`: it returns `None` and leaves `origin_path()` alone, for a read zipper and a write zipper alike, whether or not the root exists. A/B over 56M inputs (maxlen 120/300/600, crate and ACT) reproduced the baseline divergence report byte for byte, with no `ESCAPED-ROOT` anywhere. | -| `skip:empty-path` | `insert_prefix("")` (FINDINGS.md #4). Fixed upstream, with the regression test `write_zipper_insert_prefix_empty_is_identity`. Directly: over seven trie shapes (empty, root value only, one line, branchy, wide, dangling, grafted), at the map root and at a non-root focus, `insert_prefix(&[])` leaves the trie untouched and returns exactly what the model's `focusNodeIsEmpty` rule predicts. Same 56M-input A/B, again byte for byte identical. | +| `skip:at-root` | `to_next_sibling_byte` / `to_prev_sibling_byte` at the zipper root (FINDINGS.md #3). The override now guards on `at_root()`: it returns `None` and leaves `origin_path()` alone, for a read zipper and a write zipper alike, whether or not the root exists. A/B over 64M inputs (20M at maxlen 120, 20M at 300 and 8M at 600 in crate mode, 8M each at 300/600 in ACT mode) reproduced the baseline divergence report byte for byte, with no `ESCAPED-ROOT` anywhere. | +| `skip:empty-path` | `insert_prefix("")` (FINDINGS.md #4). Fixed upstream, with the regression test `write_zipper_insert_prefix_empty_is_identity`. Directly: over seven trie shapes (empty, root value only, one line, branchy, wide, dangling, grafted), at the map root and at a non-root focus, `insert_prefix(&[])` leaves the trie untouched and returns exactly what the model's `focusNodeIsEmpty` rule predicts. Same 64M-input A/B, again byte for byte identical. | | `skip:quarantined` | `graft_child_maps` (op 54), the only op it ever named, was un-quarantined when its bug was fixed. The token outlived it as a definition with no call site on any of the three sides — the Rust one needed `#[allow(dead_code)]` to compile. Removed rather than kept: a vocabulary entry nothing can emit invites the next reader to keep a live suppression alive by analogy with it. | Naming these turned up an ordering bug the bare token had hidden: for @@ -372,9 +372,11 @@ Unmasked, both match the specification over the whole sweep — 20M inputs at maxlen 120, 20M at 300, 8M at 600, and for `join_k_path_into` 8M each at 300/600 in ACT mode as well; `restrict` is `skip:act`, so ACT says nothing about it. Neither is a branch that never runs: tracing 1500 inputs, the mask -fired on 824 of 947 `restrict` calls and 678 of 1003 `join_k_path_into` calls. -That is evidence the materialisation leak does not reach these two, not a proof -that it cannot; each was lifted in its own commit so it can be put back alone. +fired on 824 of 947 `restrict` calls and 678 of 1003 `join_k_path_into` calls, +which puts the sweeps at roughly 26M and 29M formerly-masked returns actually +compared. That is evidence the materialisation leak does not reach these two, +not a proof that it cannot; each was lifted in its own commit so it can be put +back alone. ## The blind-zipper contract From 7842407352f01e376ed9baa20d79ffc61decdc38 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 14:25:55 +0000 Subject: [PATCH 27/73] Emit graft_child_maps in generated repros instead of a quarantine comment `emit_repro` still wrote `// graft_child_maps: quarantined by the harness (FINDINGS #15)` for op 54 after the harness had started running it, so a repro for any input that reaches op 54 silently did not reproduce the trace -- the drift the module header warns about. The emitted code now does what `do_graft_child_maps` does: build one map per masked child of the source and graft them. Checked by compiling the generated repros and comparing the maps they print against the trace's MAP0/MAP1. Of 600 random inputs (seed 7, maxlen 300), 236 reach a live graft_child_maps. Before: 220 reproduce, 16 do not. After: 233 reproduce. The other 3 are a separate drift, still present: the repro calls `prune_path`/`prune_ascend` off the map root and `restricting` on an empty focus, where the harness skips them. With those guarded the same way as well, all 236 reproduce. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/repro.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/differential/src/repro.rs b/differential/src/repro.rs index 6d2c06f4..c59d6dd7 100644 --- a/differential/src/repro.rs +++ b/differential/src/repro.rs @@ -208,8 +208,11 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 52 => "rz.to_next_val(); // to_next_get_val".to_string(), 53 => { let n = g!(d.modn(4)); let m = g!(d.path_n(n)); let ru = g!(d.boolean()); format!("wz.graft_masked_branches(&rz, ByteMask::from_iter({}.iter().copied()), {ru});", rs_mask(&m)) } - 54 => { let n = g!(d.modn(4)); let _m = g!(d.path_n(n)); let _ru = g!(d.boolean()); - "// graft_child_maps: quarantined by the harness (FINDINGS #15)".to_string() } + // As `do_graft_child_maps`: fed the source's own child subtries under the mask. + 54 => { let n = g!(d.modn(4)); let m = g!(d.path_n(n)); let ru = g!(d.boolean()); + format!("{{ let m = ByteMask::from_iter({}.iter().copied()); \ + let maps: Vec> = m.iter().map(|b| {{ let mut c = rz.clone(); c.descend_to_byte(b); c.make_map() }}).collect(); \ + wz.graft_child_maps(m, maps, {ru}); }}", rs_mask(&m)) } 55 => { let p = g!(d.path(6)); format!("{{ let mut b = map1.read_zipper_at_path({}); b.descend_to({}); wz.meet_2(&rz, &b); }}", rs_bytes(&r1), rs_bytes(&p)) } From 1daf3ff13826ffabe3de9136e139b3beb31a1ee8 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 14:28:19 +0000 Subject: [PATCH 28/73] Emit insert_prefix with an empty prefix in generated repros The harness has compared `insert_prefix("")` since the skip was lifted (FINDINGS #4, fixed upstream), but `emit_repro` still wrote a "skipped by the harness" comment in its place. It emits the call now, spelled `&[0u8; 0]` because `insert_prefix` is generic over the prefix and a bare `&[]` has no element type to infer. The empty prefix is the identity, so the old comment changed no final maps; this is about the repro saying what the harness does. Checked by compiling the repros for 600 random inputs (seed 7), which contain 33 empty-prefix calls: they build, and the match count against the trace's MAP0/MAP1 is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/repro.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differential/src/repro.rs b/differential/src/repro.rs index c59d6dd7..840eeb68 100644 --- a/differential/src/repro.rs +++ b/differential/src/repro.rs @@ -190,7 +190,8 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { if k == 0 { "// join_k_path_into(0): skipped by the harness".to_string() } else { format!("wz.join_k_path_into({k}, false);") } } 43 => { let p = g!(d.path(6)); - if p.is_empty() { "// insert_prefix(\"\"): skipped by the harness".to_string() } + // `insert_prefix` is generic over the prefix, so a bare `&[]` has no element type. + if p.is_empty() { "wz.insert_prefix(&[0u8; 0]);".to_string() } else { format!("wz.insert_prefix({});", rs_bytes(&p)) } } 44 => { let n = g!(d.modn(6)); format!("wz.remove_prefix({n});") } 45 => { let _pr = g!(d.boolean()); From 03626d02de1d34b5791e7049cb390a220eaa7d2f Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 15:00:35 +0000 Subject: [PATCH 29/73] Drop a reached dangling slot from the dense subtract against a list or tiny node Root cause: `ByteNode::psubtract_abstract` (src/dense_byte_node.rs), used when a DenseByteNode/CellByteNode is subtracted by a LineListNode or TinyRefNode. For a key byte the other node reaches, a slot of `self` holding only a dangling path -- an empty onward link, or a CoFree with neither link nor value, as left behind by meet_2/restrict/graft_child_maps coming to nothing -- produces an empty CoFree that is (rightly) not carried into the new node. But `is_identity` was left set, so the op reported `Identity(SELF_IDENT)`, `subtract_into` kept the original node, and the dangling path survived where the spec (`PathMap.sub`: a location `b` reaches survives only on the way to a surviving value) drops it. Now dropping a slot always clears `is_identity`. A shadowed dangling link (value + empty link at one key) still keeps its CoFree via the value and stays an identity. This accounts for 1245 of the 1250 corpus inputs of the class (all with subtract_into as the first differing op), plus the 3 unclassified inputs. In-process campaigns, classified against Lean, before -> after: seed 101 maxlen 120 10M: meet_keeps_dangling 42 -> 0, status_imprecise 18 -> 17, sibling_after_iteration 7 -> 7, value_bias 1 -> 1 seed 102 maxlen 300 20M: meet_keeps_dangling 677 -> 2, status_imprecise 196 -> 157, sibling_after_iteration 41 -> 41, new 1 -> 0 seed 103 maxlen 600 5M: meet_keeps_dangling 531 -> 1, status_imprecise 103 -> 73, sibling_after_iteration 27 -> 27, value_bias 2 -> 2, new 2 -> 0 --act seed 104 maxlen 300 5M: 3 value_bias -> 3 value_bias --act seed 105 maxlen 600 3M: 2 value_bias -> 2 value_bias Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 9 +++++- src/write_zipper.rs | 68 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index b9d24628..79c3dcc7 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -552,10 +552,17 @@ impl> ByteNode } } - //If we ended up with a value or a link in the CF, insert it into a new node + //If we ended up with a value or a link in the CF, insert it into a new node. + // Otherwise the location is gone from the result -- including when `self` held + // only a dangling path here (an empty link, or a CoFree with neither), which the + // other node reaches and so does not survive the subtraction. Either way the + // result is no longer `self`; leaving `is_identity` set here reported `Identity` + // for a node that had just lost a branch, and the caller kept the dangling path. if new_cf.has_rec() || new_cf.has_val() { new_node.mask.set_bit(key_byte); new_node.values.push(new_cf); + } else { + is_identity = false; } } else { new_node.mask.set_bit(key_byte); diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 410d6e1a..0b3e293a 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -3897,6 +3897,74 @@ mod tests { assert_eq!(m.path_exists_at(&[2u8]), false); } + /// Every existing location in `map` -- dangling paths included -- with its value, in + /// depth-first order + fn all_locations(map: &PathMap) -> Vec<(Vec, Option)> { + let mut rz = map.read_zipper(); + let mut locations = vec![]; + loop { + locations.push((rz.path().to_vec(), rz.val().cloned())); + if !rz.to_next_step() { break } + } + locations + } + + /// A DenseByteNode subtracted by a node of another type (list or tiny) must drop a dangling + /// path the source reaches, and must not report `Identity` for having done so. Covers both + /// shapes a dangling slot takes in a dense node: an empty onward link, and a CoFree holding + /// neither a link nor a value. + #[test] + fn write_zipper_subtract_into_dense_drops_reached_dangling_path() { + // Empty onward link at [2]; the source (a LineListNode) reaches [2] + let mut dst = PathMap::::new(); + for b in [1u8, 3, 4] { dst.set_val_at(&[b], 1); } + dst.create_path(&[2u8]); + let mut src = PathMap::::new(); + src.set_val_at(&[2u8, 0, 1], 246); + let mut wz = dst.write_zipper(); + assert_eq!(wz.subtract_into(&src.read_zipper(), false), AlgebraicStatus::Element); + drop(wz); + assert_eq!(all_locations(&dst), vec![(vec![], None), (vec![1], Some(1)), (vec![3], Some(1)), (vec![4], Some(1))]); + + // A dangling [2] left by a meet that came to nothing, next to a value-and-link at [0]; the + // source is a TinyRefNode reaching [2] + let mut dst = PathMap::::new(); + dst.set_val_at(&[2u8, 0, 0, 0, 0], 0); + dst.set_val_at(&[0u8, 0, 0, 0, 0], 0); + dst.set_val_at(&[0u8], 0); + let mut srcs = PathMap::::new(); + srcs.set_val_at(&[1u8, 2, 0], 0); + let mut wz = dst.write_zipper(); + wz.descend_to_byte(2); + let ra = srcs.read_zipper_at_path(&[1u8]); + let rb = srcs.read_zipper_at_path(&[1u8, 0]); + assert_eq!(wz.meet_2(&ra, &rb), AlgebraicStatus::None); + wz.reset(); + assert_eq!({ let mut probe = wz.fork_read_zipper(); probe.descend_to(&[2u8]); probe.path_exists() }, true, "the meet should leave [2] dangling"); + assert_eq!(wz.subtract_into(&ra, false), AlgebraicStatus::Element); + drop(wz); + assert_eq!(all_locations(&dst), vec![ + (vec![], None), (vec![0], Some(0)), (vec![0, 0], None), (vec![0, 0, 0], None), + (vec![0, 0, 0, 0], None), (vec![0, 0, 0, 0, 0], Some(0)), + ]); + + // `graft_child_maps` of an empty map leaves a dangling [0] beside [1, 0] + let mut dst = PathMap::::new(); + dst.set_val_at(&[], 0); + dst.set_val_at(&[0u8], 0); + dst.set_val_at(&[0u8, 0], 0); + dst.set_val_at(&[1u8, 0], 0); + let mut src = PathMap::::new(); + src.set_val_at(&[2u8, 3, 0, 0], 0); + let mut wz = dst.write_zipper(); + wz.graft_child_maps(ByteMask::from_iter([0u8]), vec![PathMap::new()], false); + assert_eq!({ let mut probe = wz.fork_read_zipper(); probe.descend_to(&[0u8]); probe.path_exists() }, true, "the graft should leave [0] dangling"); + let rz = src.read_zipper_at_path(&[2u8, 3]); + assert_eq!(wz.subtract_into(&rz, false), AlgebraicStatus::Element); + drop(wz); + assert_eq!(all_locations(&dst), vec![(vec![], Some(0)), (vec![1], None), (vec![1, 0], Some(0))]); + } + /// Tests whether the [WriteZipper::subtract_into] operation will do the right thing with the root value #[test] fn write_zipper_subtract_into_test1() { From 618f8ce9ead6950bdf2eb11f137ef30b2ee3d3e3 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 15:06:10 +0000 Subject: [PATCH 30/73] Don't claim COUNTER_IDENT for a meet key that runs into an empty onward link Root cause: `pmeet_generic_internal` (src/trie_node.rs). When a lookup in `other` for one of `self`'s keys consumes part of the key and lands on an onward link, the rest of the key is looked up in that child. If the child is the empty node -- a dangling path in `other` -- nothing is found and `node_key_overlap` on the empty node is 0, so the key was answered `COUNTER_IDENT`: "the result equals `other` here". It does not: the meet drops the dangling path. The typical shape is a LineListNode meeting a DenseByteNode destination with a dangling child (the dense side enumerates the list's payloads with `swapped`), where the dangling byte's sibling is an Arc shared by both operands, so every other key also reports an identity and the whole meet came back `Identity`, keeping the dangling child. An empty `other_node` can only be reached through such a link, so it now never claims `COUNTER_IDENT`. These are the last 3 of the 1250 corpus inputs (meet_into first differing). In-process campaigns, classified against Lean, before this commit -> after (original base in brackets): seed 101 maxlen 120 10M: meet_keeps_dangling 0 -> 0 [42], status_imprecise 17 -> 17 [18], sibling_after_iteration 7 -> 7, value_bias 1 -> 1 seed 102 maxlen 300 20M: meet_keeps_dangling 2 -> 0 [677], status_imprecise 157 -> 157 [196], sibling_after_iteration 41 -> 41, new 0 -> 0 [1] seed 103 maxlen 600 5M: meet_keeps_dangling 1 -> 0 [531], status_imprecise 73 -> 73 [103], sibling_after_iteration 27 -> 27, value_bias 2 -> 2, new 0 -> 0 [2] --act seed 104 maxlen 300 5M: 3 value_bias, unchanged --act seed 105 maxlen 600 3M: 2 value_bias, unchanged Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/trie_node.rs | 6 +++++- src/write_zipper.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/trie_node.rs b/src/trie_node.rs index 984bc8ee..cc1f2019 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -778,7 +778,11 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: // prefix with the key -- typically a dangling path, which the lookup above reports as // "nothing found, everything covered" -- is not carried into the result, so claiming // `COUNTER_IDENT` would hand the caller `other` with the dangling path still in it. - let nothing_here = if other_node.node_key_overlap(keys[idx].0) == 0 { + // + //The same holds when `other_node` is itself empty: we only get here by following an + // onward link from a parent, so an empty `other_node` is a dangling path in `other` + // running along this key, and it does not survive the meet either. + let nothing_here = if !other_node.node_is_empty() && other_node.node_key_overlap(keys[idx].0) == 0 { FatAlgebraicResult::new(COUNTER_IDENT, None) } else { FatAlgebraicResult::none() diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 0b3e293a..480f57cd 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -3965,6 +3965,46 @@ mod tests { assert_eq!(all_locations(&dst), vec![(vec![], Some(0)), (vec![1], None), (vec![1, 0], Some(0))]); } + /// `pmeet_generic` must not claim `COUNTER_IDENT` for a key whose lookup in `other` runs into an + /// empty onward link: that is a dangling path in `other`, and the meet does not keep it. Here the + /// DenseByteNode destination is met by enumerating the source LineListNode's payloads, and the + /// claimed identity used to hand back the destination with its dangling path intact. + #[test] + fn write_zipper_meet_into_drops_dangling_link_reached_through_lookup() { + // dst = { [0] dangling, [1, 1, 0] = 0 } (the [1] branch shared with src); src = { [0, 0, 0] = 0, [1, 1, 0] = 0 } + let mut dst = PathMap::::new(); + let mut src = PathMap::::new(); + src.set_val_at(&[1u8, 1, 0], 0); + src.set_val_at(&[0u8, 0, 0], 0); + let mut wz = dst.write_zipper(); + let mut rz = src.read_zipper(); + wz.join_into(&rz); + rz.descend_to_byte(1); + wz.graft_masked_branches(&rz, ByteMask::from_iter([3u8, 2, 0]), false); + rz.reset(); + assert_eq!({ let mut probe = wz.fork_read_zipper(); probe.descend_to(&[0u8]); probe.path_exists() }, true, "the graft should leave [0] dangling"); + assert_eq!(wz.meet_into(&rz, false), AlgebraicStatus::Element); + drop(wz); + assert_eq!(all_locations(&dst), vec![(vec![], None), (vec![1], None), (vec![1, 1], None), (vec![1, 1, 0], Some(0))]); + + // A dangling path one level deeper: dst = { [0, 0] dangling, [1, 0] = 0 } + let mut dst = PathMap::::new(); + let mut src = PathMap::::new(); + src.set_val_at(&[2u8, 1, 0], 0); + src.set_val_at(&[2u8, 0, 0, 0, 0], 0); + src.set_val_at(&[], 0); + let mut wz = dst.write_zipper(); + let rz = src.read_zipper_at_path(&[2u8]); + wz.graft(&rz); + wz.descend_to(&[0u8, 0]); + wz.remove_unmasked_branches(ByteMask::new(), false); + wz.reset(); + assert_eq!({ let mut probe = wz.fork_read_zipper(); probe.descend_to(&[0u8, 0]); probe.path_exists() }, true, "the removal should leave [0, 0] dangling"); + assert_eq!(wz.meet_into(&rz, false), AlgebraicStatus::Element); + drop(wz); + assert_eq!(all_locations(&dst), vec![(vec![], None), (vec![1], None), (vec![1, 0], Some(0))]); + } + /// Tests whether the [WriteZipper::subtract_into] operation will do the right thing with the root value #[test] fn write_zipper_subtract_into_test1() { From a7bc9509456c7168ef3ca2cc766fa6f431f54648 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 16:03:40 +0000 Subject: [PATCH 31/73] Keep dangling paths in meet results, as master does Reverts 7171655 ("Drop dangling paths from meet results") and 97ae27f ("Don't claim COUNTER_IDENT for a meet key that runs into an empty onward link"). Both made the crate follow the Lean model's rule that dangling paths never survive a meet. That rule is the model's own: on master `EmptyNode::pmeet_dyn` answers `Identity`, and master's tests assert it outright ("Should have had its value removed, but the path should remain"), which 7171655 had rewritten. Master's meet intersects locations -- a dangling path survives when the other side has that path too -- and prunes nothing. The drop was also not even consistent: a meet with a clone kept the dangling path through the pointer-equality shortcut while a meet with an independent copy dropped it. With the revert both keep it and report Identity, as on master. 053a186 (subtract dropping a dangling path the source reaches) stays: on master that already happens for list/list and dense/dense subtracts, and only the dense-against-list case kept the path. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 9 ++- src/empty_node.rs | 10 +-- src/lib.rs | 37 +++------- src/trie_node.rs | 47 ++++--------- src/write_zipper.rs | 156 ++--------------------------------------- 5 files changed, 39 insertions(+), 220 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 79c3dcc7..e3515547 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -2024,9 +2024,12 @@ impl, Other rec_status.merge(val_status, true, true) } fn pmeet(&self, other: &OtherCf) -> AlgebraicResult { - //A dangling cofree (a path leading to no value) survives no meet - if (!self.has_rec() && !self.has_val()) || (!other.has_rec() && !other.has_val()) { - return AlgebraicResult::None + //If one or the other cofree is dangling, it's an identity result for the dangling cofree + let mut identity_flag = 0; + if !self.has_rec() && !self.has_val() {identity_flag = SELF_IDENT;} + if !other.has_rec() && !other.has_val() {identity_flag |= COUNTER_IDENT;} + if identity_flag > 0 { + return AlgebraicResult::Identity(identity_flag) } //Otherwise actually work with what the cofrees contain diff --git a/src/empty_node.rs b/src/empty_node.rs index df567331..ce336dd1 100644 --- a/src/empty_node.rs +++ b/src/empty_node.rs @@ -135,10 +135,12 @@ impl TrieNode for EmptyNode { fn drop_head_dyn(&mut self, _byte_cnt: usize) -> Option> where V: Lattice { None } - fn pmeet_dyn(&self, _other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { - //A dangling path leads to no value, so it survives no meet. (Reporting it as an - // identity kept the empty node -- and its path -- in every meet result.) - AlgebraicResult::None + fn pmeet_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { + if other.node_is_empty() { + AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT) + } else { + AlgebraicResult::Identity(SELF_IDENT) + } } fn psubtract_dyn(&self, _other: TaggedNodeRef) -> AlgebraicResult> where V: DistributiveLattice { AlgebraicResult::None diff --git a/src/lib.rs b/src/lib.rs index f5801701..6d38dc03 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -514,8 +514,6 @@ mod tests { assert_eq!(map.val_at(&[0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]), None); } - /// A meet keeps a location only if it leads to a surviving value, so dangling paths on either - /// side never survive it -- whatever the branching factor of the nodes involved. #[test] fn map_meet_dangling_branching_factor_test1() { // Left contains a path without a split @@ -528,17 +526,16 @@ mod tests { right.set_val_at([7u8, 2u8, 0u8], 20); right.create_path([7u8, 3u8]); - // A dangling path meets a value to nothing: the whole intersection is empty let intersection = left.meet(&right); - assert!(intersection.is_empty()); - assert_eq!(intersection.path_exists_at([7u8, 1u8, 0u8]), false); + assert_eq!(intersection.path_exists_at([7u8, 1u8, 0u8]), true); //Should have had its value removed, but the path should remain + assert_eq!(intersection.val_at([7u8, 1u8, 0u8]), None); assert_eq!(intersection.path_exists_at([7u8, 2u8, 0u8]), false); assert_eq!(intersection.path_exists_at([7u8, 3u8]), false); //Make sure the result is the same with the opposite operand order let intersection = right.meet(&left); - assert!(intersection.is_empty()); - assert_eq!(intersection.path_exists_at([7u8, 1u8, 0u8]), false); + assert_eq!(intersection.path_exists_at([7u8, 1u8, 0u8]), true); + assert_eq!(intersection.val_at([7u8, 1u8, 0u8]), None); assert_eq!(intersection.path_exists_at([7u8, 2u8, 0u8]), false); assert_eq!(intersection.path_exists_at([7u8, 3u8]), false); @@ -551,43 +548,29 @@ mod tests { right.create_path([7u8, 3u8]); let intersection = left.meet(&right); - assert!(intersection.is_empty()); - assert_eq!(intersection.path_exists_at([7u8, 1u8]), false); + assert_eq!(intersection.path_exists_at([7u8, 1u8]), true); //Should have had its value removed, but the path should remain + assert_eq!(intersection.val_at([7u8, 1u8]), None); assert_eq!(intersection.path_exists_at([7u8, 2u8]), false); assert_eq!(intersection.path_exists_at([7u8, 3u8]), false); //Make sure the result is the same with the opposite operand order let intersection = right.meet(&left); - assert!(intersection.is_empty()); - assert_eq!(intersection.path_exists_at([7u8, 1u8]), false); + assert_eq!(intersection.path_exists_at([7u8, 1u8]), true); + assert_eq!(intersection.val_at([7u8, 1u8]), None); assert_eq!(intersection.path_exists_at([7u8, 2u8]), false); assert_eq!(intersection.path_exists_at([7u8, 3u8]), false); - - // TEST 3. A dangling path next to a surviving value is dropped; the value survives - let mut left: PathMap = PathMap::new(); - left.set_val_at([7u8, 1u8], 10); - left.create_path([7u8, 2u8]); - let mut right: PathMap = PathMap::new(); - right.set_val_at([7u8, 1u8], 10); - right.set_val_at([7u8, 2u8], 20); - for intersection in [left.meet(&right), right.meet(&left)] { - assert_eq!(intersection.val_count(), 1); - assert_eq!(intersection.path_exists_at([7u8, 1u8]), true); - assert_eq!(intersection.path_exists_at([7u8, 2u8]), false); - } } #[test] fn map_meet_dangling_branching_factor_test2() { - //Test 1: Path subsets. Two dangling paths, one a prefix of the other, meet to nothing + //Test 1: Path subsets let mut left: PathMap<()> = PathMap::new(); left.create_path(b"OneTwo"); let mut right: PathMap<()> = PathMap::new(); right.create_path(b"OneTwoThree"); let intersection = left.meet(&right); - assert!(intersection.is_empty()); - assert_eq!(intersection.path_exists_at(b"OneTwo"), false); + assert_eq!(intersection.path_exists_at(b"OneTwo"), true); assert_eq!(intersection.path_exists_at(b"OneTwoThree"), false); assert_eq!(intersection.path_exists_at(b"OneTwoT"), false); } diff --git a/src/trie_node.rs b/src/trie_node.rs index cc1f2019..70bb82b3 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -773,20 +773,6 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: } else { pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results, swapped); - //`other` holds no payload at this key, so the result has nothing here. That equals - // `other` at this key only if `other` has no path along it at all. A path that shares a - // prefix with the key -- typically a dangling path, which the lookup above reports as - // "nothing found, everything covered" -- is not carried into the result, so claiming - // `COUNTER_IDENT` would hand the caller `other` with the dangling path still in it. - // - //The same holds when `other_node` is itself empty: we only get here by following an - // onward link from a parent, so an empty `other_node` is a dangling path in `other` - // running along this key, and it does not survive the meet either. - let nothing_here = if !other_node.node_is_empty() && other_node.node_key_overlap(keys[idx].0) == 0 { - FatAlgebraicResult::new(COUNTER_IDENT, None) - } else { - FatAlgebraicResult::none() - }; let result = match &self_payloads[idx].1 { PayloadRef::Child(self_link) => { match other_node.get_node_at_key(keys[idx].0).into_option() { @@ -799,14 +785,20 @@ pub(crate) fn pmeet_generic_internal<'trie, const MAX_PAYLOAD_CNT: usize, V, A: FatAlgebraicResult::from_binary_op_result(result, self_link, &other_onward_node) .map(|child| ValOrChild::Child(child)) }, - //Nothing in `other` below this key -- whether or not `self` is dangling here. - // A meet keeps only locations that lead to a surviving value, so a dangling - // `self` path meeting a value in `other` also yields nothing. - None => nothing_here, + None => { + //Check to see if we have a dangling path, because a dangling path meet with a value should result in a path, but no value + if self_link.is_empty() && other_node.node_get_val(keys[idx].0).is_some() { + FatAlgebraicResult::new(SELF_IDENT, Some(ValOrChild::Child(TrieNodeODRc::new_empty()))) + } else { + FatAlgebraicResult::new(COUNTER_IDENT, None) + } + } } }, - //If self_payload is a val and we didn't get a corresponding val, then this result is None - PayloadRef::Val(_self_val) => nothing_here, + PayloadRef::Val(_self_val) => { + //If self_payload is a val and we didn't get a corresponding val, then this result is None + FatAlgebraicResult::new(COUNTER_IDENT, None) + }, _ => unreachable!() }; results[idx] = result; @@ -1546,11 +1538,6 @@ mod tagged_node_ref { } pub fn pmeet_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { - //An empty node (dangling path) meets to nothing, even with itself: the sentinel is - // shared, so this must come before the identity shortcut - if self.node_is_empty() || other.node_is_empty() { - return AlgebraicResult::None; - } if self.shared_node_id() == other.shared_node_id() { return AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT); } @@ -2213,11 +2200,6 @@ mod tagged_node_ref { } pub fn pmeet_dyn(&self, other: TaggedNodeRef) -> AlgebraicResult> where V: Lattice { - //An empty node (dangling path) meets to nothing, even with itself: the sentinel is - // shared, so this must come before the identity shortcut - if self.node_is_empty() || other.node_is_empty() { - return AlgebraicResult::None; - } if self.ptr == other.ptr { return AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT); } @@ -3322,10 +3304,7 @@ impl TrieNodeODRc { } #[inline] pub fn pmeet(&self, other: &Self) -> AlgebraicResult { - if self.is_empty() || other.is_empty() { - //A dangling path survives no meet; see `EmptyNode::pmeet_dyn` - AlgebraicResult::None - } else if self.ptr_eq(other) { + if self.ptr_eq(other) { AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT) } else { self.as_tagged().pmeet_dyn(other.as_tagged()) diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 480f57cd..260be318 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -3766,10 +3766,7 @@ mod tests { assert_eq!(btm.path_exists_at(&[1, 255, 0]), true); assert_eq!(btm.path_exists_at(&[0, 255, 0]), true); - // Test 3: meet from a higher level with all dangling paths and prune=true. A location - // survives a meet only if it leads to a surviving value, so dangling paths never survive - // one -- even where both sides hold the same dangling path. The result is empty, and with - // `prune = true` the focus path itself goes. + // Test 3: meet from a higher level with all dangling paths and prune=true let mut btm2: PathMap<()> = PathMap::new(); btm2.create_path(&[0, 255, 0]); btm2.create_path(&[0, 255, 1]); @@ -3780,121 +3777,16 @@ mod tests { let mut wz = zh2.write_zipper_at_exclusive_path(&[0]).unwrap(); let rz = zh2.read_zipper_at_path(&[1]).unwrap(); let alg_result = wz.meet_into(&rz, true); - assert_eq!(alg_result, AlgebraicStatus::None); - zh2.cleanup_write_zipper(wz); + assert_eq!(alg_result, AlgebraicStatus::Element); + drop(wz); drop(rz); drop(zh2); // Verify the meet operation did what it should have assert_eq!(btm2.path_exists_at(&[1, 255, 0]), true); - assert_eq!(btm2.path_exists_at(&[0, 255, 0]), false); + assert_eq!(btm2.path_exists_at(&[0, 255, 0]), true); assert_eq!(btm2.path_exists_at(&[0, 200, 5]), false); assert_eq!(btm2.path_exists_at(&[0, 255, 1]), false); - assert_eq!(btm2.path_exists_at(&[0]), false); - } - - /// A meet keeps a location only if it leads to a surviving value, so a dangling path on either - /// side is dropped -- not kept as an identity. Covers the two-slot LineListNode and the - /// DenseByteNode CoFree cases, the shared empty sentinel meeting itself, `meet_k_path_into` - /// and `PathMap::meet`. - #[test] - fn write_zipper_meet_into_drops_dangling_paths() { - // LineListNode: dst = { [2] = 0 } plus a dangling [1]; src = { [1] = 5, [2] = 0 } - let mut dst = PathMap::::new(); - dst.set_val_at(&[2u8], 0); - dst.create_path(&[1u8]); - let mut src = PathMap::::new(); - src.set_val_at(&[1u8], 5); - src.set_val_at(&[2u8], 0); - let mut wz = dst.write_zipper(); - assert_eq!(wz.meet_into(&src.read_zipper(), false), AlgebraicStatus::Element); - assert_eq!(wz.child_count(), 1); - drop(wz); - assert_eq!(dst.path_exists_at(&[1u8]), false); - assert_eq!(dst.get_val_at(&[2u8]), Some(&0)); - - // Only a dangling child left: the whole result is empty - let mut dst = PathMap::::new(); - dst.create_path(&[1u8]); - let mut wz = dst.write_zipper(); - assert_eq!(wz.meet_into(&src.read_zipper(), false), AlgebraicStatus::None); - assert_eq!(wz.child_count(), 0); - drop(wz); - assert_eq!(dst.path_exists_at(&[1u8]), false); - - // The same dangling path on both sides is not an identity either - let mut dst = PathMap::::new(); - dst.create_path(&[1u8]); - let mut both = PathMap::::new(); - both.create_path(&[1u8]); - let mut wz = dst.write_zipper(); - assert_eq!(wz.meet_into(&both.read_zipper(), false), AlgebraicStatus::None); - assert_eq!(wz.child_count(), 0); - drop(wz); - - // DenseByteNode: four children, one of them dangling - let mut dst = PathMap::::new(); - for b in [0u8, 2, 3] { dst.set_val_at(&[b], 0); } - dst.create_path(&[1u8]); - let mut src = PathMap::::new(); - for b in [0u8, 1, 2, 3] { src.set_val_at(&[b], 0); } - let mut wz = dst.write_zipper(); - assert_eq!(wz.meet_into(&src.read_zipper(), false), AlgebraicStatus::Element); - assert_eq!(wz.child_count(), 3); - drop(wz); - assert_eq!(dst.path_exists_at(&[1u8]), false); - assert_eq!(dst.val_count(), 3); - - // DenseByteNode destination holding only a dangling cofree at [2] and a subtree at [3], met - // with a LineListNode source keyed [2, 2] and [3]: the lookup for [2, 2] finds nothing and - // used to report the result as identical to the destination, dangling [2] included. - let mut dst = PathMap::::new(); - for b in [2u8, 4, 5] { dst.set_val_at(&[b], 1); } - dst.set_val_at(&[3u8, 0], 9); - dst.remove_val_at(&[4u8], true); - dst.remove_val_at(&[5u8], true); - let mut src = PathMap::::new(); - src.set_val_at(&[2u8, 2], 204); - src.set_val_at(&[3u8, 0], 9); - let nothing = PathMap::::new(); - let mut nothing_rz = nothing.read_zipper(); - nothing_rz.descend_to(&[1u8]); - let mut wz = dst.write_zipper(); - wz.descend_to(&[2u8]); - wz.graft(¬hing_rz); - wz.ascend(1); - assert_eq!(wz.path_exists(), true); - let mut probe = wz.fork_read_zipper(); - probe.descend_to(&[2u8]); - assert_eq!(probe.path_exists(), true, "the graft of nothing should leave [2] dangling"); - drop(probe); - assert_eq!(wz.meet_into(&src.read_zipper(), false), AlgebraicStatus::Element); - assert_eq!(wz.child_count(), 1); - drop(wz); - assert_eq!(dst.path_exists_at(&[2u8]), false); - assert_eq!(dst.get_val_at(&[3u8, 0]), Some(&9)); - - // meet_k_path_into: the k-paths [0] -> { [2] = 0 } and [1] -> { [2, 0] = 0 } meet to nothing, - // since [2] is dangling on the second. Used to leave a dangling [2] behind and report true. - let mut map = PathMap::::new(); - map.set_val_at(&[0u8, 2], 0); - map.set_val_at(&[1u8, 2, 0], 0); - let mut wz = map.write_zipper(); - assert_eq!(wz.meet_k_path_into(1, false), false); - assert_eq!(wz.child_count(), 0); - drop(wz); - assert_eq!(map.val_count(), 0); - - // PathMap::meet - let mut a = PathMap::::new(); - a.set_val_at(&[1u8], 0); - a.create_path(&[2u8]); - let mut b = PathMap::::new(); - b.set_val_at(&[1u8], 0); - b.set_val_at(&[2u8], 0); - let m = a.meet(&b); - assert_eq!(m.val_count(), 1); - assert_eq!(m.path_exists_at(&[2u8]), false); } /// Every existing location in `map` -- dangling paths included -- with its value, in @@ -3965,46 +3857,6 @@ mod tests { assert_eq!(all_locations(&dst), vec![(vec![], Some(0)), (vec![1], None), (vec![1, 0], Some(0))]); } - /// `pmeet_generic` must not claim `COUNTER_IDENT` for a key whose lookup in `other` runs into an - /// empty onward link: that is a dangling path in `other`, and the meet does not keep it. Here the - /// DenseByteNode destination is met by enumerating the source LineListNode's payloads, and the - /// claimed identity used to hand back the destination with its dangling path intact. - #[test] - fn write_zipper_meet_into_drops_dangling_link_reached_through_lookup() { - // dst = { [0] dangling, [1, 1, 0] = 0 } (the [1] branch shared with src); src = { [0, 0, 0] = 0, [1, 1, 0] = 0 } - let mut dst = PathMap::::new(); - let mut src = PathMap::::new(); - src.set_val_at(&[1u8, 1, 0], 0); - src.set_val_at(&[0u8, 0, 0], 0); - let mut wz = dst.write_zipper(); - let mut rz = src.read_zipper(); - wz.join_into(&rz); - rz.descend_to_byte(1); - wz.graft_masked_branches(&rz, ByteMask::from_iter([3u8, 2, 0]), false); - rz.reset(); - assert_eq!({ let mut probe = wz.fork_read_zipper(); probe.descend_to(&[0u8]); probe.path_exists() }, true, "the graft should leave [0] dangling"); - assert_eq!(wz.meet_into(&rz, false), AlgebraicStatus::Element); - drop(wz); - assert_eq!(all_locations(&dst), vec![(vec![], None), (vec![1], None), (vec![1, 1], None), (vec![1, 1, 0], Some(0))]); - - // A dangling path one level deeper: dst = { [0, 0] dangling, [1, 0] = 0 } - let mut dst = PathMap::::new(); - let mut src = PathMap::::new(); - src.set_val_at(&[2u8, 1, 0], 0); - src.set_val_at(&[2u8, 0, 0, 0, 0], 0); - src.set_val_at(&[], 0); - let mut wz = dst.write_zipper(); - let rz = src.read_zipper_at_path(&[2u8]); - wz.graft(&rz); - wz.descend_to(&[0u8, 0]); - wz.remove_unmasked_branches(ByteMask::new(), false); - wz.reset(); - assert_eq!({ let mut probe = wz.fork_read_zipper(); probe.descend_to(&[0u8, 0]); probe.path_exists() }, true, "the removal should leave [0, 0] dangling"); - assert_eq!(wz.meet_into(&rz, false), AlgebraicStatus::Element); - drop(wz); - assert_eq!(all_locations(&dst), vec![(vec![], None), (vec![1], None), (vec![1, 0], Some(0))]); - } - /// Tests whether the [WriteZipper::subtract_into] operation will do the right thing with the root value #[test] fn write_zipper_subtract_into_test1() { From 75c869a2cbec5c1e2214026be2ec0ed483e18a54 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 16:37:45 +0000 Subject: [PATCH 32/73] Specify meet as an intersection of locations, and prune as dropping dangling paths The model's meet kept only the locations leading to a surviving value, so no dangling path survived a meet. Upstream intends otherwise: master's `map_meet_dangling_branching_factor_test1` keeps a dangling path met against a value ("value removed, but the path should remain") and checks both operand orders. The rule is now: prune = false a path exists in the result exactly when it exists in both operands; a value exists exactly where both hold one, as the meet of the two. meet(a, a) = a, dangling paths included. prune = true the same values, and only the locations leading to one of them: every dangling path is dropped, including one both operands had. In both, the focus -- the root of the met subtrie -- is never removed. `meet_2` and `PathMap::meet`, which take no `prune`, are the first form. So meet({[0,0]:-}, {[0,1]:-}) is {[0]:-}, and {} with prune. Encoded as `PathMap.meet` / `PathMap.meetPruned` (with `dropDangling`) in the Lean model and transcribed into the Rust reference model; `meet_into` and `meet_k_path_into` choose between them on `prune` and no longer prune above the focus. Meet's `prune` has an exact meaning now, so ops 38 and 46 use the decoded flag instead of forcing false, in Fuzz.lean, harness.rs, the reference op table and the repro generator. The Spec laws gain `meetIdem`, `meetPrunedIdem` and `meetCommOnPaths`, guarded in Check.lean and check.rs; the doc that meet is "not idempotent on locations" is gone. The Lean and Rust models agree on 1,300,000 crate-mode and 500,000 ACT-mode inputs (maxlen 120/300/600). The crate does not yet follow the rule. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/harness.rs | 10 +++++--- differential/src/reference/check.rs | 3 +++ differential/src/reference/fuzz.rs | 10 +++++--- differential/src/reference/laws.rs | 20 ++++++++++++--- differential/src/reference/pathmap.rs | 23 ++++++++++++++++-- differential/src/reference/write.rs | 34 +++++++++++++++----------- differential/src/repro.rs | 6 ++--- lean/PathMapModel/Check.lean | 3 +++ lean/PathMapModel/Fuzz.lean | 10 +++++--- lean/PathMapModel/PathMap.lean | 20 ++++++++++++--- lean/PathMapModel/Spec.lean | 16 ++++++++++-- lean/PathMapModel/Write.lean | 35 +++++++++++++-------------- 12 files changed, 133 insertions(+), 57 deletions(-) diff --git a/differential/src/harness.rs b/differential/src/harness.rs index f6dcd554..c34e9f9b 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -732,8 +732,10 @@ pub fn run_ops( ("join_map_into", s) } 38 => { - let _pr = get!(d.boolean()); // decoded for stream alignment; see `no_prune` - ("meet_into", show_status_opt((*rz).do_meet_into(&mut wz, no_prune))) + // Unlike the other operations, meet's `prune` has an exact meaning + // (see `Zip.meetInto`), so the decoded flag is used. + let pr = get!(d.boolean()); + ("meet_into", show_status_opt((*rz).do_meet_into(&mut wz, pr))) } 39 => { let _pr = get!(d.boolean()); // decoded for stream alignment; see `no_prune` @@ -801,7 +803,7 @@ pub fn run_ops( } 46 => { let k = get!(d.modn(4)); - let _pr = get!(d.boolean()); // decoded for stream alignment; see `no_prune` + let pr = get!(d.boolean()); // meet's `prune` is used; see op 38 // `meet_k_path_into` spins forever when the focus has no // children, and escapes the focus subtree when k == 0. // See `Zip.meetKPathUnspecified`. @@ -812,7 +814,7 @@ pub fn run_ops( } else { ( "meet_k_path_into", - show_bool(wz.meet_k_path_into(k, no_prune)).to_string(), + show_bool(wz.meet_k_path_into(k, pr)).to_string(), ) } } diff --git a/differential/src/reference/check.rs b/differential/src/reference/check.rs index 161bb4fd..c819eb08 100644 --- a/differential/src/reference/check.rs +++ b/differential/src/reference/check.rs @@ -259,10 +259,13 @@ mod tests { for a in &fs { assert!(join_idem(&OPS, a)); assert!(meet_idem_on_vals(&OPS, a)); + assert!(meet_idem(&OPS, a)); + assert!(meet_pruned_idem(&OPS, a)); assert!(sub_self_empty_vals(&OPS, a)); assert!(restrict_self(&OPS, a)); for b in &fs { assert!(join_comm_on_paths(&OPS, a, b)); + assert!(meet_comm_on_paths(&OPS, a, b)); for c in &fs { assert!(join_assoc(&OPS, a, b, c)); } diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs index d7a21444..796633d3 100644 --- a/differential/src/reference/fuzz.rs +++ b/differential/src/reference/fuzz.rs @@ -508,11 +508,13 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { } } 38 => { - let _pr = d.boolean()?; + let pr = d.boolean()?; if s.act { s.emit("meet_into", SKIP_ACT); } else { - let st = s.wz.meet_into(&OPS, &s.rz, NO_PRUNE); + // Unlike the other operations, meet's `prune` has an exact meaning + // (see `Zip::meet_into`), so the decoded flag is used. + let st = s.wz.meet_into(&OPS, &s.rz, pr); s.emit("meet_into", &show_status(st)); } } @@ -604,7 +606,7 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { } 46 => { let k = d.modn(4)?; - let _pr = d.boolean()?; + let pr = d.boolean()?; // `meet_k_path_into` is not implementable for these arguments; see // `Zip::meet_k_path_unspecified`, whose two disjuncts are split out // here so the skip names which one fired. The crate side matches. @@ -613,7 +615,7 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { } else if s.wz.focus_node_is_empty() { s.emit("meet_k_path_into", SKIP_EMPTY_FOCUS); } else { - let r = s.wz.meet_k_path_into(&OPS, k, NO_PRUNE); + let r = s.wz.meet_k_path_into(&OPS, k, pr); s.emit("meet_k_path_into", show_bool(r)); } } diff --git a/differential/src/reference/laws.rs b/differential/src/reference/laws.rs index 197b2f61..32e25b9f 100644 --- a/differential/src/reference/laws.rs +++ b/differential/src/reference/laws.rs @@ -234,14 +234,28 @@ pub fn join_assoc( l.beq_t(ops, &r) } -/// `meet` is idempotent *on values*. It is not idempotent on locations: a -/// meet discards dangling paths, so `meet a a` keeps only the value-bearing -/// skeleton. +/// `meet` is idempotent *on values*. pub fn meet_idem_on_vals(ops: &impl ValOps, a: &PathMap) -> bool { PathMap::meet(ops, a, a).vals().map(|(k, _)| k.clone()).collect::>() == a.vals().map(|(k, _)| k.clone()).collect::>() } +/// `meet` is idempotent outright, dangling paths included: a location survives +/// exactly when both operands have it. +pub fn meet_idem(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::meet(ops, a, a).beq_t(ops, a) +} + +/// `meet_pruned(a, a)` is `a` with its dangling paths dropped. +pub fn meet_pruned_idem(ops: &impl ValOps, a: &PathMap) -> bool { + PathMap::meet_pruned(ops, a, a).beq_t(ops, &a.drop_dangling()) +} + +/// `meet` is commutative on locations. +pub fn meet_comm_on_paths(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> bool { + PathMap::meet(ops, a, b).paths().collect::>() == PathMap::meet(ops, b, a).paths().collect::>() +} + /// Subtracting a map from itself leaves no values. pub fn sub_self_empty_vals(ops: &impl ValOps, a: &PathMap) -> bool { PathMap::sub(ops, a, a).vals().next().is_none() diff --git a/differential/src/reference/pathmap.rs b/differential/src/reference/pathmap.rs index 9b3c5789..c3bd2428 100644 --- a/differential/src/reference/pathmap.rs +++ b/differential/src/reference/pathmap.rs @@ -458,8 +458,13 @@ impl PathMap { PathMap::mk(vals, a.paths().chain(b.paths()).cloned()) } - /// Meet (intersection). A location survives only if it lies on the way to a - /// surviving value, so dangling paths never survive a meet. + /// Meet (intersection). A location exists in the result exactly when it + /// exists in both operands, dangling or not, and a value exists exactly where + /// both operands hold one, as the meet of the two. So meet is commutative, + /// associative and idempotent on locations, and a meet with an equal trie + /// changes nothing. + /// + /// This is the meet with `prune = false`; see [`PathMap::meet_pruned`]. pub fn meet(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { let vals: Vec<(Vec, V)> = a .vals() @@ -467,9 +472,23 @@ impl PathMap { Self::meet_val(ops, Some(av), b.val_at(k)).map(|v| (k.clone(), v)) }) .collect(); + PathMap::mk(vals, a.paths().filter(|q| b.path_exists(q)).cloned()) + } + + /// The trie with every dangling path removed: only the root and the locations + /// on the way to a value remain. + pub fn drop_dangling(&self) -> PathMap { + let vals: Vec<(Vec, V)> = self.vals().map(|(k, v)| (k.clone(), v.clone())).collect(); PathMap::mk(vals, std::iter::empty()) } + /// The meet with `prune = true`: the same values as [`PathMap::meet`], and only + /// the locations on the way to one of them, so no dangling path survives -- + /// including one both operands had. + pub fn meet_pruned(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { + Self::meet(ops, a, b).drop_dangling() + } + /// Subtract. /// /// Two rules interact here. Where `b` has no node at all, `a`'s subtree is diff --git a/differential/src/reference/write.rs b/differential/src/reference/write.rs index 0bd5564b..92a55c47 100644 --- a/differential/src/reference/write.rs +++ b/differential/src/reference/write.rs @@ -476,9 +476,12 @@ impl Zip { /// `ZipperWriting::meet_into`: intersect the focus's submap with the /// source's. /// - /// The value step runs first and can prune the focus out from under the node - /// step. A meet drops every dangling path, since a location only survives if - /// it leads to a surviving value. + /// Below the focus the result is [`PathMap::meet`] with `prune = false` -- + /// every location both sides have survives, dangling ones included -- and + /// [`PathMap::meet_pruned`] with `prune = true`, which keeps only the locations + /// leading to a surviving value. The focus itself is never removed: pruning + /// stops at it, so a focus left without a value or anything below it is still + /// there. pub fn meet_into(&mut self, ops: &impl ValOps, src: &Zip, prune: bool) -> AlgStatus { let (val_status, val_was_none) = match (self.val().cloned(), src.val().cloned()) { (Some(sv), Some(ov)) => { @@ -489,14 +492,14 @@ impl Zip { self.set_val(v); } None => { - self.remove_val(prune); + self.remove_val(false); } } (st, false) } (None, Some(_)) => (AlgStatus::None, true), (Some(_), None) => { - self.remove_val(prune); + self.remove_val(false); (AlgStatus::None, false) } (None, None) => (AlgStatus::None, true), @@ -506,17 +509,16 @@ impl Zip { if self_b.is_empty_map() { return AlgStatus::merge(AlgStatus::None, val_status, true, val_was_none); } + let f = self.focus(); if src_b.is_empty_map() { - let f = self.focus(); self.trie.remove_below(&f); - if prune { - self.trie.prune_path(0, &f); - } return AlgStatus::merge(AlgStatus::None, val_status, false, val_was_none); } - let r = PathMap::meet(ops, &self_b, &src_b); + let r = if prune { PathMap::meet_pruned(ops, &self_b, &src_b) } else { PathMap::meet(ops, &self_b, &src_b) }; let st = Self::node_status(ops, &self_b, &r); - self.write_node(st, &r, prune); + if st != AlgStatus::Identity { + self.trie.graft_below(&f, &r); + } AlgStatus::merge(st, val_status, false, val_was_none) } @@ -561,7 +563,8 @@ impl Zip { } /// `ZipperWriting::meet_2`: meet two *source* submaps and write the result at - /// the focus. + /// the focus. There is no `prune` argument, so this is [`PathMap::meet`]: + /// every location both sources have survives, dangling ones included. /// /// Two things separate this from [`Zip::meet_into`]. It does not consult /// what is already at the focus, so — as the implementation notes — it never @@ -690,13 +693,16 @@ impl Zip { Some(a) => PathMap::meet(ops, &a, &m), }); } - match result { + // `prune` drops every dangling path from the meet, including one that a + // single k-path's subtrie brings along unmet. The focus itself is never + // removed. + match result.map(|m| if prune { m.drop_dangling() } else { m }) { Some(m) if !m.is_empty_map() => { self.graft_map(&m); true } _ => { - self.remove_branches(prune); + self.remove_branches(false); false } } diff --git a/differential/src/repro.rs b/differential/src/repro.rs index 840eeb68..d442bb4b 100644 --- a/differential/src/repro.rs +++ b/differential/src/repro.rs @@ -182,7 +182,7 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { format!("wz.graft_src_at(&rz, {});", rs_bytes(&p)) } 36 => "wz.join_into(&rz);".to_string(), 37 => "wz.join_map_into(rz.make_map());".to_string(), - 38 => { let _pr = g!(d.boolean()); "wz.meet_into(&rz, false);".to_string() } + 38 => { let pr = g!(d.boolean()); format!("wz.meet_into(&rz, {pr});") } 39 => { let _pr = g!(d.boolean()); "wz.subtract_into(&rz, false);".to_string() } 40 => "wz.restrict(&rz);".to_string(), 41 => "wz.restricting(&rz);".to_string(), @@ -196,8 +196,8 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 44 => { let n = g!(d.modn(6)); format!("wz.remove_prefix({n});") } 45 => { let _pr = g!(d.boolean()); "if let Some(m) = wz.take_map(false) { wz.graft_map(m); }".to_string() } - 46 => { let k = g!(d.modn(4)); let _pr = g!(d.boolean()); - format!("if {k} != 0 && wz.child_count() != 0 {{ wz.meet_k_path_into({k}, false); }}") } + 46 => { let k = g!(d.modn(4)); let pr = g!(d.boolean()); + format!("if {k} != 0 && wz.child_count() != 0 {{ wz.meet_k_path_into({k}, {pr}); }}") } 47 => { let t = g!(d.modn(2)); format!("{{ let mut obs = Vec::new(); {}.descend_until_observed(&mut obs); }}", z!(t)) } 48 => { let v = g!(d.u8()) as u64; diff --git a/lean/PathMapModel/Check.lean b/lean/PathMapModel/Check.lean index ea252df0..076107d2 100644 --- a/lean/PathMapModel/Check.lean +++ b/lean/PathMapModel/Check.lean @@ -147,6 +147,9 @@ def dropT1Result : T := ((zipAt dropT1 [0x31,0x32,0x33,0x3a] []).joinKPathInto o #guard fixtures.all (joinIdem ops) #guard fixtures.all (fun a => fixtures.all (fun b => fixtures.all (joinAssoc ops a b))) #guard fixtures.all (meetIdemOnVals ops) +#guard fixtures.all (meetIdem ops) +#guard fixtures.all (meetPrunedIdem ops) +#guard fixtures.all (fun a => fixtures.all (meetCommOnPaths ops a)) #guard fixtures.all (subSelfEmptyVals ops) #guard fixtures.all (restrictSelf ops) diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index 09265559..6fb87f4c 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -396,10 +396,12 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do let (st, z) := s.wz.joinMapInto ops s.rz.makeMap some (emit { s with wz := z } "join_map_into" (if leaky then "?" else toString st), d) - | 38 => do let (_pr, d) ← d.bool + | 38 => do let (pr, d) ← d.bool if s.act then some (emit s "meet_into" skipAct, d) else - let (st, z) := s.wz.meetInto ops s.rz noPrune + -- Unlike the other operations, meet's `prune` has an exact meaning + -- (see `Zip.meetInto`), so the decoded flag is used. + let (st, z) := s.wz.meetInto ops s.rz pr some (emit { s with wz := z } "meet_into" (toString st), d) | 39 => do let (_pr, d) ← d.bool if s.act then some (emit s "subtract_into" skipAct, d) @@ -461,7 +463,7 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do match m with | some mm => some (emit { s with wz := z.graftMap mm } "take_map_restore" "1", d) | none => some (emit { s with wz := z } "take_map_restore" "0", d) - | 46 => do let (k, d) ← d.mod 4; let (_pr, d) ← d.bool + | 46 => do let (k, d) ← d.mod 4; let (pr, d) ← d.bool -- `meet_k_path_into` is not implementable for these arguments; see -- `Zip.meetKPathUnspecified`, whose two disjuncts are split out here -- so the skip names which one fired. The Rust side matches. @@ -469,7 +471,7 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do else if s.wz.focusNodeIsEmpty then some (emit s "meet_k_path_into" skipEmptyFocus, d) else - let (r, z) := s.wz.meetKPathInto ops k noPrune + let (r, z) := s.wz.meetKPathInto ops k pr some (emit { s with wz := z } "meet_k_path_into" (showBool r), d) | 47 => do let (t, d) ← d.mod 2 -- The blind-zipper addition: `descend_until` reporting the bytes it diff --git a/lean/PathMapModel/PathMap.lean b/lean/PathMapModel/PathMap.lean index 2054dfd9..0e2bcd6a 100644 --- a/lean/PathMapModel/PathMap.lean +++ b/lean/PathMapModel/PathMap.lean @@ -338,11 +338,25 @@ def join (a b : PathMap V) : PathMap V := mk' (keys.filterMap fun k => (joinVal ops (a.valAt k) (b.valAt k)).map (k, ·)) (a.paths ++ b.paths) -/-- Meet (intersection). A location survives only if it lies on the way to a -surviving value, so dangling paths never survive a meet. -/ +/-- Meet (intersection). A location exists in the result exactly when it exists +in both operands, dangling or not, and a value exists exactly where both operands +hold one, as the meet of the two. So meet is commutative, associative and +idempotent on locations, and a meet with an equal trie changes nothing. + +This is the meet with `prune = false`; see `meetPruned`. -/ def meet (a b : PathMap V) : PathMap V := let keys := Path.sortDedup (a.vals.map (·.1)) - mk' (keys.filterMap fun k => (meetVal ops (a.valAt k) (b.valAt k)).map (k, ·)) [] + mk' (keys.filterMap fun k => (meetVal ops (a.valAt k) (b.valAt k)).map (k, ·)) + (a.paths.filter b.pathExists) + +/-- The trie with every dangling path removed: only the root and the locations on +the way to a value remain. -/ +def dropDangling (t : PathMap V) : PathMap V := mk' t.vals [] + +/-- The meet with `prune = true`: the same values as `meet`, and only the +locations on the way to one of them, so no dangling path survives -- including +one both operands had. -/ +def meetPruned (a b : PathMap V) : PathMap V := (meet ops a b).dropDangling /-- Subtract. diff --git a/lean/PathMapModel/Spec.lean b/lean/PathMapModel/Spec.lean index 44158a55..c930f056 100644 --- a/lean/PathMapModel/Spec.lean +++ b/lean/PathMapModel/Spec.lean @@ -314,11 +314,23 @@ def joinIdem (ops : ValOps V) (a : PathMap V) : Bool := def joinAssoc (ops : ValOps V) (a b c : PathMap V) : Bool := PathMap.beqT ops (PathMap.join ops (PathMap.join ops a b) c) (PathMap.join ops a (PathMap.join ops b c)) -/-- `meet` is idempotent *on values*. It is not idempotent on locations: a meet -discards dangling paths, so `meet a a` keeps only the value-bearing skeleton. -/ +/-- `meet` is idempotent *on values*. -/ def meetIdemOnVals (ops : ValOps V) (a : PathMap V) : Bool := (PathMap.meet ops a a).vals.map (·.1) == a.vals.map (·.1) +/-- `meet` is idempotent outright, dangling paths included: a location survives +exactly when both operands have it. -/ +def meetIdem (ops : ValOps V) (a : PathMap V) : Bool := + PathMap.beqT ops (PathMap.meet ops a a) a + +/-- `meetPruned a a` is `a` with its dangling paths dropped. -/ +def meetPrunedIdem (ops : ValOps V) (a : PathMap V) : Bool := + PathMap.beqT ops (PathMap.meetPruned ops a a) a.dropDangling + +/-- `meet` is commutative on locations. -/ +def meetCommOnPaths (ops : ValOps V) (a b : PathMap V) : Bool := + (PathMap.meet ops a b).paths == (PathMap.meet ops b a).paths + /-- Subtracting a map from itself leaves no values. -/ def subSelfEmptyVals (ops : ValOps V) (a : PathMap V) : Bool := (PathMap.sub ops a a).vals.isEmpty diff --git a/lean/PathMapModel/Write.lean b/lean/PathMapModel/Write.lean index 6ee3a84e..39020d08 100644 --- a/lean/PathMapModel/Write.lean +++ b/lean/PathMapModel/Write.lean @@ -328,9 +328,11 @@ def joinIntoTake (src : Zip V) (prune : Bool) : AlgStatus × Zip V × Zip V := /-- `ZipperWriting::meet_into`: intersect the focus's subtrie with the source's. -The value step runs first and can prune the focus out from under the node step. -A meet drops every dangling path, since a location only survives if it leads to -a surviving value. -/ +Below the focus the result is `PathMap.meet` with `prune = false` -- every +location both sides have survives, dangling ones included -- and +`PathMap.meetPruned` with `prune = true`, which keeps only the locations leading +to a surviving value. The focus itself is never removed: pruning stops at it, so +a focus left without a value or anything below it is still there. -/ def meetInto (src : Zip V) (prune : Bool) : AlgStatus × Zip V := let (valStatus, valWasNone, z1) := match z.val, src.val with @@ -339,26 +341,20 @@ def meetInto (src : Zip V) (prune : Bool) : AlgStatus × Zip V := (AlgStatus.ofValRes r, false, match r.resolve sv ov with | some v => (z.setVal v).2 - | none => (z.removeVal prune).2) + | none => (z.removeVal false).2) | none, some _ => (AlgStatus.none, true, z) - | some _, none => (AlgStatus.none, false, (z.removeVal prune).2) + | some _, none => (AlgStatus.none, false, (z.removeVal false).2) | none, none => (AlgStatus.none, true, z) let selfB := z1.focusNode let srcB := src.focusNode if selfB.isEmptyMap then (AlgStatus.merge .none valStatus true valWasNone, z1) else if srcB.isEmptyMap then - let z2 := z1.withTrie (z1.trie.removeBelow z1.focus) - let z3 := if prune then (z2.prunePath).2 else z2 - (AlgStatus.merge .none valStatus false valWasNone, z3) + (AlgStatus.merge .none valStatus false valWasNone, z1.withTrie (z1.trie.removeBelow z1.focus)) else - let r := PathMap.meet ops selfB srcB + let r := if prune then PathMap.meetPruned ops selfB srcB else PathMap.meet ops selfB srcB let st := nodeStatus ops selfB r - let z2 := - if st == .identity then z1 - else - let zg := z1.withTrie (z1.trie.graftBelow z1.focus r) - if st == .none && prune then (zg.prunePath).2 else zg + let z2 := if st == .identity then z1 else z1.withTrie (z1.trie.graftBelow z1.focus r) (AlgStatus.merge st valStatus false valWasNone, z2) /-- `ZipperWriting::subtract_into`: remove the source's subtrie from the focus's. @@ -396,7 +392,8 @@ def subtractInto (src : Zip V) (prune : Bool) : AlgStatus × Zip V := (AlgStatus.merge st valStatus false valWasNone, z2) /-- `ZipperWriting::meet_2`: meet two *source* subtries and write the result at -the focus. +the focus. There is no `prune` argument, so this is `PathMap.meet`: every location +both sources have survives, dangling ones included. Two things separate this from `meet_into`. It does not consult what is already at the focus, so — as the implementation notes — it never reports `Identity`, @@ -486,9 +483,11 @@ def meetKPathInto (k : Nat) (prune : Bool) : Bool × Zip V := match acc with | none => some m | some a => some (PathMap.meet ops a m)) none - match result with - | some m => if m.isEmptyMap then (false, (z.removeBranches prune).2) else (true, z.graftMap m) - | none => (false, (z.removeBranches prune).2) + -- `prune` drops every dangling path from the meet, including one that a single + -- k-path's subtrie brings along unmet. The focus itself is never removed. + match result.map fun m => if prune then m.dropDangling else m with + | some m => if m.isEmptyMap then (false, (z.removeBranches false).2) else (true, z.graftMap m) + | none => (false, (z.removeBranches false).2) end Zip end PathMapModel From e01474166888922227f01fffd9343034b83c25b2 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 16:42:34 +0000 Subject: [PATCH 33/73] Compare meet with prune = false only; prune = true may skip shared nodes `prune = true` is relaxed: a meet may skip a node shared with the source rather than walk it for dangling paths, so a dangling path inside a shared node may survive. The result lies between `meetPruned` and `meet` -- the same values, and locations between the two -- but is no longer exact, so ops 38 and 46 go back to forcing `prune = false` in Fuzz.lean, harness.rs, the reference op table and the repro generator. The model keeps `meetPruned` as the most pruned result, documented as the lower bound. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/harness.rs | 12 ++++++------ differential/src/reference/fuzz.rs | 12 ++++++------ differential/src/reference/pathmap.rs | 10 +++++++--- differential/src/reference/write.rs | 7 ++++--- differential/src/repro.rs | 6 +++--- lean/PathMapModel/Fuzz.lean | 12 ++++++------ lean/PathMapModel/PathMap.lean | 11 ++++++++--- lean/PathMapModel/Write.lean | 10 ++++++---- 8 files changed, 46 insertions(+), 34 deletions(-) diff --git a/differential/src/harness.rs b/differential/src/harness.rs index c34e9f9b..ce3b0361 100644 --- a/differential/src/harness.rs +++ b/differential/src/harness.rs @@ -732,10 +732,10 @@ pub fn run_ops( ("join_map_into", s) } 38 => { - // Unlike the other operations, meet's `prune` has an exact meaning - // (see `Zip.meetInto`), so the decoded flag is used. - let pr = get!(d.boolean()); - ("meet_into", show_status_opt((*rz).do_meet_into(&mut wz, pr))) + // `prune = true` is best-effort (nodes shared with the source may be + // left unpruned; see `Zip.meetInto`), so only `prune = false` is compared. + let _pr = get!(d.boolean()); + ("meet_into", show_status_opt((*rz).do_meet_into(&mut wz, no_prune))) } 39 => { let _pr = get!(d.boolean()); // decoded for stream alignment; see `no_prune` @@ -803,7 +803,7 @@ pub fn run_ops( } 46 => { let k = get!(d.modn(4)); - let pr = get!(d.boolean()); // meet's `prune` is used; see op 38 + let _pr = get!(d.boolean()); // decoded for stream alignment; see op 38 // `meet_k_path_into` spins forever when the focus has no // children, and escapes the focus subtree when k == 0. // See `Zip.meetKPathUnspecified`. @@ -814,7 +814,7 @@ pub fn run_ops( } else { ( "meet_k_path_into", - show_bool(wz.meet_k_path_into(k, pr)).to_string(), + show_bool(wz.meet_k_path_into(k, no_prune)).to_string(), ) } } diff --git a/differential/src/reference/fuzz.rs b/differential/src/reference/fuzz.rs index 796633d3..ab1c257f 100644 --- a/differential/src/reference/fuzz.rs +++ b/differential/src/reference/fuzz.rs @@ -508,13 +508,13 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { } } 38 => { - let pr = d.boolean()?; + let _pr = d.boolean()?; if s.act { s.emit("meet_into", SKIP_ACT); } else { - // Unlike the other operations, meet's `prune` has an exact meaning - // (see `Zip::meet_into`), so the decoded flag is used. - let st = s.wz.meet_into(&OPS, &s.rz, pr); + // `prune = true` is best-effort (nodes shared with the source may be + // left unpruned; see `Zip::meet_into`), so only `prune = false` is compared. + let st = s.wz.meet_into(&OPS, &s.rz, NO_PRUNE); s.emit("meet_into", &show_status(st)); } } @@ -606,7 +606,7 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { } 46 => { let k = d.modn(4)?; - let pr = d.boolean()?; + let _pr = d.boolean()?; // `meet_k_path_into` is not implementable for these arguments; see // `Zip::meet_k_path_unspecified`, whose two disjuncts are split out // here so the skip names which one fired. The crate side matches. @@ -615,7 +615,7 @@ fn step(s: &mut St, d: &mut Dec) -> Option<()> { } else if s.wz.focus_node_is_empty() { s.emit("meet_k_path_into", SKIP_EMPTY_FOCUS); } else { - let r = s.wz.meet_k_path_into(&OPS, k, pr); + let r = s.wz.meet_k_path_into(&OPS, k, NO_PRUNE); s.emit("meet_k_path_into", show_bool(r)); } } diff --git a/differential/src/reference/pathmap.rs b/differential/src/reference/pathmap.rs index c3bd2428..ffda0f9c 100644 --- a/differential/src/reference/pathmap.rs +++ b/differential/src/reference/pathmap.rs @@ -482,9 +482,13 @@ impl PathMap { PathMap::mk(vals, std::iter::empty()) } - /// The meet with `prune = true`: the same values as [`PathMap::meet`], and only - /// the locations on the way to one of them, so no dangling path survives -- - /// including one both operands had. + /// The meet with `prune = true`, at its most pruned: the same values as + /// [`PathMap::meet`], and only the locations on the way to one of them. + /// + /// `pathmap` may stop short of this: it can skip a node shared with the source + /// rather than walk it, so a dangling path inside a shared node may survive. + /// The result always lies between `meet_pruned` and `meet`, and is not + /// compared by the fuzzer. pub fn meet_pruned(ops: &impl ValOps, a: &PathMap, b: &PathMap) -> PathMap { Self::meet(ops, a, b).drop_dangling() } diff --git a/differential/src/reference/write.rs b/differential/src/reference/write.rs index 92a55c47..f351ec19 100644 --- a/differential/src/reference/write.rs +++ b/differential/src/reference/write.rs @@ -477,9 +477,10 @@ impl Zip { /// source's. /// /// Below the focus the result is [`PathMap::meet`] with `prune = false` -- - /// every location both sides have survives, dangling ones included -- and - /// [`PathMap::meet_pruned`] with `prune = true`, which keeps only the locations - /// leading to a surviving value. The focus itself is never removed: pruning + /// every location both sides have survives, dangling ones included. With + /// `prune = true` the model gives [`PathMap::meet_pruned`]; `pathmap` may leave + /// dangling paths inside nodes shared with the source, so that case is + /// best-effort and not compared. The focus itself is never removed: pruning /// stops at it, so a focus left without a value or anything below it is still /// there. pub fn meet_into(&mut self, ops: &impl ValOps, src: &Zip, prune: bool) -> AlgStatus { diff --git a/differential/src/repro.rs b/differential/src/repro.rs index d442bb4b..840eeb68 100644 --- a/differential/src/repro.rs +++ b/differential/src/repro.rs @@ -182,7 +182,7 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { format!("wz.graft_src_at(&rz, {});", rs_bytes(&p)) } 36 => "wz.join_into(&rz);".to_string(), 37 => "wz.join_map_into(rz.make_map());".to_string(), - 38 => { let pr = g!(d.boolean()); format!("wz.meet_into(&rz, {pr});") } + 38 => { let _pr = g!(d.boolean()); "wz.meet_into(&rz, false);".to_string() } 39 => { let _pr = g!(d.boolean()); "wz.subtract_into(&rz, false);".to_string() } 40 => "wz.restrict(&rz);".to_string(), 41 => "wz.restricting(&rz);".to_string(), @@ -196,8 +196,8 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 44 => { let n = g!(d.modn(6)); format!("wz.remove_prefix({n});") } 45 => { let _pr = g!(d.boolean()); "if let Some(m) = wz.take_map(false) { wz.graft_map(m); }".to_string() } - 46 => { let k = g!(d.modn(4)); let pr = g!(d.boolean()); - format!("if {k} != 0 && wz.child_count() != 0 {{ wz.meet_k_path_into({k}, {pr}); }}") } + 46 => { let k = g!(d.modn(4)); let _pr = g!(d.boolean()); + format!("if {k} != 0 && wz.child_count() != 0 {{ wz.meet_k_path_into({k}, false); }}") } 47 => { let t = g!(d.modn(2)); format!("{{ let mut obs = Vec::new(); {}.descend_until_observed(&mut obs); }}", z!(t)) } 48 => { let v = g!(d.u8()) as u64; diff --git a/lean/PathMapModel/Fuzz.lean b/lean/PathMapModel/Fuzz.lean index 6fb87f4c..ab86db81 100644 --- a/lean/PathMapModel/Fuzz.lean +++ b/lean/PathMapModel/Fuzz.lean @@ -396,12 +396,12 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do let (st, z) := s.wz.joinMapInto ops s.rz.makeMap some (emit { s with wz := z } "join_map_into" (if leaky then "?" else toString st), d) - | 38 => do let (pr, d) ← d.bool + | 38 => do let (_pr, d) ← d.bool if s.act then some (emit s "meet_into" skipAct, d) else - -- Unlike the other operations, meet's `prune` has an exact meaning - -- (see `Zip.meetInto`), so the decoded flag is used. - let (st, z) := s.wz.meetInto ops s.rz pr + -- `prune = true` is best-effort (nodes shared with the source may be + -- left unpruned; see `Zip.meetInto`), so only `prune = false` is compared. + let (st, z) := s.wz.meetInto ops s.rz noPrune some (emit { s with wz := z } "meet_into" (toString st), d) | 39 => do let (_pr, d) ← d.bool if s.act then some (emit s "subtract_into" skipAct, d) @@ -463,7 +463,7 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do match m with | some mm => some (emit { s with wz := z.graftMap mm } "take_map_restore" "1", d) | none => some (emit { s with wz := z } "take_map_restore" "0", d) - | 46 => do let (k, d) ← d.mod 4; let (pr, d) ← d.bool + | 46 => do let (k, d) ← d.mod 4; let (_pr, d) ← d.bool -- `meet_k_path_into` is not implementable for these arguments; see -- `Zip.meetKPathUnspecified`, whose two disjuncts are split out here -- so the skip names which one fired. The Rust side matches. @@ -471,7 +471,7 @@ def step (s : St) (d : Dec) : Option (St × Dec) := do else if s.wz.focusNodeIsEmpty then some (emit s "meet_k_path_into" skipEmptyFocus, d) else - let (r, z) := s.wz.meetKPathInto ops k pr + let (r, z) := s.wz.meetKPathInto ops k noPrune some (emit { s with wz := z } "meet_k_path_into" (showBool r), d) | 47 => do let (t, d) ← d.mod 2 -- The blind-zipper addition: `descend_until` reporting the bytes it diff --git a/lean/PathMapModel/PathMap.lean b/lean/PathMapModel/PathMap.lean index 0e2bcd6a..81fa43b4 100644 --- a/lean/PathMapModel/PathMap.lean +++ b/lean/PathMapModel/PathMap.lean @@ -353,9 +353,14 @@ def meet (a b : PathMap V) : PathMap V := the way to a value remain. -/ def dropDangling (t : PathMap V) : PathMap V := mk' t.vals [] -/-- The meet with `prune = true`: the same values as `meet`, and only the -locations on the way to one of them, so no dangling path survives -- including -one both operands had. -/ +/-- The meet with `prune = true`, at its most pruned: the same values as `meet`, +and only the locations on the way to one of them. + +`pathmap` may stop short of this: it can skip a node shared with the source +rather than walk it, so a dangling path inside a shared node may survive. The +result always lies between `meetPruned` and `meet` -- same values, locations a +subset of `meet`'s and a superset of `meetPruned`'s -- and is not compared by +the fuzzer. -/ def meetPruned (a b : PathMap V) : PathMap V := (meet ops a b).dropDangling /-- Subtract. diff --git a/lean/PathMapModel/Write.lean b/lean/PathMapModel/Write.lean index 39020d08..f086652a 100644 --- a/lean/PathMapModel/Write.lean +++ b/lean/PathMapModel/Write.lean @@ -329,10 +329,12 @@ def joinIntoTake (src : Zip V) (prune : Bool) : AlgStatus × Zip V × Zip V := /-- `ZipperWriting::meet_into`: intersect the focus's subtrie with the source's. Below the focus the result is `PathMap.meet` with `prune = false` -- every -location both sides have survives, dangling ones included -- and -`PathMap.meetPruned` with `prune = true`, which keeps only the locations leading -to a surviving value. The focus itself is never removed: pruning stops at it, so -a focus left without a value or anything below it is still there. -/ +location both sides have survives, dangling ones included. With `prune = true` +the model gives `PathMap.meetPruned`, which keeps only the locations leading to a +surviving value; `pathmap` may leave dangling paths inside nodes shared with the +source, so that case is best-effort and not compared. The focus itself is never +removed: pruning stops at it, so a focus left without a value or anything below +it is still there. -/ def meetInto (src : Zip V) (prune : Bool) : AlgStatus × Zip V := let (valStatus, valWasNone, z1) := match z.val, src.val with From ada5458ec01522233a5a3aea1ea8f2d5aa48f648 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 16:44:13 +0000 Subject: [PATCH 34/73] Test the meet rule for dangling paths (fails until the crate follows it) `write_zipper_meet_into_dangling_paths` states the rule the models now encode: with `prune = false` a path exists in the result exactly when it exists in both operands and a value exactly where both hold one, so meet({[0,0]:-}, {[0,1]:-}) is {[0]:-} and a meet with an equal trie -- a clone or an independent copy -- reports Identity and changes nothing; with `prune = true` every dangling path is dropped except that nodes shared with the source may be skipped; the focus is never removed; and the result does not depend on the node types of the two sides. It fails today, first on meet({[0,0]:-}, {[0,1]:-}), which the crate answers with {} as master does. Committed failing on request, ahead of the crate fix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/write_zipper.rs | 115 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 260be318..4adfdd6d 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -3857,6 +3857,121 @@ mod tests { assert_eq!(all_locations(&dst), vec![(vec![], Some(0)), (vec![1], None), (vec![1, 0], Some(0))]); } + /// The meet rule, dangling paths included. + /// + /// With `prune = false` a path exists in the result exactly when it exists in both operands, and + /// a value exists exactly where both hold one, as the meet of the two. So a meet with an equal + /// trie -- a clone that shares its nodes, or an independent copy -- changes nothing and reports + /// `Identity`, whatever node types the two sides use. + /// + /// With `prune = true` the values are the same, but only the locations on the way to a value + /// survive: every dangling path is dropped, including one both operands had -- except that a node + /// shared with the source may be skipped rather than walked, so a dangling path inside it may + /// survive. The focus itself is never removed. + #[test] + fn write_zipper_meet_into_dangling_paths() { + type Build = fn() -> PathMap; + type Locations = Vec<(Vec, Option)>; + + // meet({[0,0]:-}, {[0,1]:-}) -> {[0]:-}, and with prune nothing but the root + let a = || { let mut m = PathMap::::new(); m.create_path(&[0u8, 0]); m }; + let b = || { let mut m = PathMap::::new(); m.create_path(&[0u8, 1]); m }; + assert_eq!(all_locations(&a().meet(&b())), vec![(vec![], None), (vec![0], None)]); + let mut d = a(); + assert_eq!(d.write_zipper().meet_into(&b().read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(all_locations(&d), vec![(vec![], None), (vec![0], None)]); + let mut d = a(); + assert_eq!(d.write_zipper().meet_into(&b().read_zipper(), true), AlgebraicStatus::None); + assert_eq!(all_locations(&d), vec![(vec![], None)]); + + // A dangling path met against a value keeps the path without the value, in both orders + let left = || { let mut m = PathMap::::new(); m.create_path([7u8, 1, 0]); m }; + let right = || { let mut m = PathMap::::new(); m.set_val_at([7u8, 1, 0], 10); m.set_val_at([7u8, 2, 0], 20); m.create_path([7u8, 3]); m }; + let expected: Locations = vec![(vec![], None), (vec![7], None), (vec![7, 1], None), (vec![7, 1, 0], None)]; + assert_eq!(all_locations(&left().meet(&right())), expected); + assert_eq!(all_locations(&right().meet(&left())), expected); + + // Meeting an equal trie: unchanged without prune, dangling paths dropped with it + let equal_cases: [(&str, Build, Locations); 3] = [ + ("create_path", || { let mut m = PathMap::new(); m.create_path(&[1u8]); m }, vec![(vec![], None)]), + ("value removed", || { let mut m = PathMap::new(); m.set_val_at(&[1u8], 5); m.remove_val_at(&[1u8], false); m }, vec![(vec![], None)]), + ("dangling beside values", || { + let mut m = PathMap::new(); + m.set_val_at(&[1u8, 2], 5); + m.set_val_at(&[1u8, 4, 4, 4], 5); + m.set_val_at(&[9u8], 5); + m.write_zipper_at_path(&[1u8, 4]).remove_branches(false); + m + }, vec![(vec![], None), (vec![1], None), (vec![1, 2], Some(5)), (vec![9], Some(5))]), + ]; + for (name, build, pruned) in equal_cases { + let expected = all_locations(&build()); + let mut shared = build(); + let clone = shared.clone(); + assert_eq!(shared.write_zipper().meet_into(&clone.read_zipper(), false), AlgebraicStatus::Identity, "{name}: meet with a clone"); + assert_eq!(all_locations(&shared), expected, "{name}: meet with a clone"); + let mut unshared = build(); + assert_eq!(unshared.write_zipper().meet_into(&build().read_zipper(), false), AlgebraicStatus::Identity, "{name}: meet with a copy"); + assert_eq!(all_locations(&unshared), expected, "{name}: meet with a copy"); + assert_eq!(all_locations(&build().meet(&build())), expected, "{name}: PathMap::meet"); + + // With a clone every node is shared, so pruning may skip all of it: the result lies + // between fully pruned and not pruned at all, with the values unchanged + let mut shared = build(); + let clone = shared.clone(); + shared.write_zipper().meet_into(&clone.read_zipper(), true); + let got = all_locations(&shared); + assert!(pruned.iter().all(|l| got.contains(l)) && got.iter().all(|l| expected.contains(l)), + "{name}: pruned meet with a clone: {got:?} is not between {pruned:?} and {expected:?}"); + let mut unshared = build(); + unshared.write_zipper().meet_into(&build().read_zipper(), true); + assert_eq!(all_locations(&unshared), pruned, "{name}: pruned meet with a copy"); + } + + // Pruning never removes the focus, below the root or at a zipper's root + let dst = || { let mut m = PathMap::::new(); m.set_val_at(&[9u8], 9); m.create_path(&[5u8, 0]); m }; + let src = || { let mut m = PathMap::::new(); m.create_path(&[5u8, 1]); m }; + let mut d = dst(); + let s = src(); + { let mut wz = d.write_zipper(); wz.descend_to(&[5u8]); assert_eq!(wz.meet_into(&s.read_zipper_at_path(&[5u8]), true), AlgebraicStatus::None); } + assert_eq!(all_locations(&d), vec![(vec![], None), (vec![5], None), (vec![9], Some(9))]); + let mut d = dst(); + { let mut wz = d.write_zipper_at_path(&[5u8]); assert_eq!(wz.meet_into(&s.read_zipper_at_path(&[5u8]), true), AlgebraicStatus::None); } + assert_eq!(all_locations(&d), vec![(vec![], None), (vec![5], None), (vec![9], Some(9))]); + + // A dangling [2] in the destination, against sources of different node types + let dense_dst: Build = || { let mut d = PathMap::new(); for b in [1u8, 3, 4] { d.set_val_at(&[b], 1); } d.create_path(&[2u8]); d }; + let list_dst: Build = || { let mut d = PathMap::new(); d.set_val_at(&[1u8], 1); d.create_path(&[2u8]); d }; + let list_src_below_2: Build = || { let mut s = PathMap::new(); s.set_val_at(&[1u8], 1); s.set_val_at(&[2u8, 0, 1], 246); s }; + let dense_src_below_2: Build = || { let mut s = PathMap::new(); for b in [1u8, 3, 4] { s.set_val_at(&[b], 1); } s.set_val_at(&[2u8, 0, 1], 246); s }; + let dense_src_without_2: Build = || { let mut s = PathMap::new(); for b in [1u8, 3, 4] { s.set_val_at(&[b], 1); } s }; + let dense_all: Locations = vec![(vec![], None), (vec![1], Some(1)), (vec![2], None), (vec![3], Some(1)), (vec![4], Some(1))]; + let cases: [(&str, Build, Build, AlgebraicStatus, Locations); 6] = [ + ("list dst, list src with [2]", list_dst, list_src_below_2, AlgebraicStatus::Identity, + vec![(vec![], None), (vec![1], Some(1)), (vec![2], None)]), + ("list dst, list src dangling at [2]", list_dst, list_dst, AlgebraicStatus::Identity, + vec![(vec![], None), (vec![1], Some(1)), (vec![2], None)]), + ("dense dst, dense src with [2]", dense_dst, dense_src_below_2, AlgebraicStatus::Identity, dense_all.clone()), + ("dense dst, dense src dangling at [2]", dense_dst, dense_dst, AlgebraicStatus::Identity, dense_all), + ("dense dst, list src with [2]", dense_dst, list_src_below_2, AlgebraicStatus::Element, + vec![(vec![], None), (vec![1], Some(1)), (vec![2], None)]), + ("dense dst, src without [2]", dense_dst, dense_src_without_2, AlgebraicStatus::Element, + vec![(vec![], None), (vec![1], Some(1)), (vec![3], Some(1)), (vec![4], Some(1))]), + ]; + let mut failures = vec![]; + for (name, dst, src, status, expected) in cases { + let mut d = dst(); + let s = src(); + let st = d.write_zipper().meet_into(&s.read_zipper(), false); + let got = all_locations(&d); + let whole = all_locations(&dst().meet(&src())); + if st != status || got != expected || whole != expected { + failures.push(format!("{name}: meet_into {st:?} {got:?}, PathMap::meet {whole:?}; expected {status:?} {expected:?}")); + } + } + assert!(failures.is_empty(), "\n{}", failures.join("\n")); + } + /// Tests whether the [WriteZipper::subtract_into] operation will do the right thing with the root value #[test] fn write_zipper_subtract_into_test1() { From 6002fb9d25cc143f8fc6ea1d8d7d06d6d01102e4 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 16:51:36 +0000 Subject: [PATCH 35/73] Restore the node stack after prune_path without ascending `prune_path_internal(false)` does not move the zipper, but to find where pruning stops it walks the node stack up, popping `focus_stack` and `prefix_idx`, and it left them there. When the focus sat in a child node -- after a `graft` at the zipper's root, for instance -- the stack then described an ancestor while `prefix_buf` still named the focus, so the next write through the focus went to the wrong node. `get_val_or_set_mut` caught it by panicking on the value it had just set (`src/write_zipper.rs:1429`); other writes went astray silently. It now walks the stack back down towards the focus with `descend_to_internal`, as far as nodes still exist. Present on master (f477a91) as well, through `prune_path` directly and through `meet_into(.., true)`, which calls it. Found by fuzzing meets with the prune flag: map1 {[1,0,0,0,0]:0}, a write zipper at [0,0] grafts the read zipper at [1], descends one byte, meets with prune, then writes a value. Regression test `write_zipper_write_after_prune_path_below_a_graft` checks the resulting trie for both routes and fails without the fix. Fuzzing with the prune flag passed through to the crate: without the fix each of three 300k runs (maxlen 120/300/600) stops on this panic; with it all three complete with no panic. The suites pass apart from `write_zipper_meet_into_dangling_paths`, which is committed failing ahead of the meet fix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/write_zipper.rs | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 4adfdd6d..2163c1a2 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -2651,6 +2651,12 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC if should_ascend { self.key.prefix_buf.truncate(temp_path.len()); + } else if ascended { + //The loop above walked the node stack up to where pruning stopped, but the zipper has not + // moved. Walk it back down towards the focus, as far as nodes still exist, or the stack + // and `prefix_idx` describe an ancestor while the path still names the focus, and the next + // write through the focus lands in the wrong node + self.descend_to_internal(); } pruned_bytes @@ -3857,6 +3863,52 @@ mod tests { assert_eq!(all_locations(&dst), vec![(vec![], Some(0)), (vec![1], None), (vec![1, 0], Some(0))]); } + /// `prune_path` does not move the zipper, but it used to leave the node stack where its upward + /// walk stopped. When the focus sat in a child node -- as it does after a `graft` at the zipper's + /// root -- the stack then described an ancestor while the path still named the focus, and the next + /// write through the focus went to the wrong node: `get_val_or_set_mut` panicked on the value it + /// had just set. `meet_into(.., true)` reached this through its own `prune_path`. + #[test] + fn write_zipper_write_after_prune_path_below_a_graft() { + let build = || { + let mut m0 = PathMap::::new(); + let mut m1 = PathMap::::new(); + m1.set_val_at(&[1u8, 0, 0, 0, 0], 7); + m0.create_path(&[0u8, 0]); + m1.create_path(&[1u8]); + (m0, m1) + }; + + // prune_path directly + let (mut m0, m1) = build(); + { + let mut wz = m0.write_zipper_at_path(&[0u8, 0]); + let rz = m1.read_zipper_at_path(&[1u8]); + wz.graft(&rz); + wz.descend_last_byte(); + wz.remove_branches(false); + wz.prune_path(); + assert_eq!(wz.path(), &[0u8]); + assert_eq!(*wz.get_val_or_set_mut_with(|| 3), 3); + assert_eq!(wz.val(), Some(&3)); + } + assert_eq!(all_locations(&m0), vec![(vec![], None), (vec![0], None), (vec![0, 0], None), (vec![0, 0, 0], Some(3))]); + + // through meet_into with prune + let (mut m0, m1) = build(); + { + let mut wz = m0.write_zipper_at_path(&[0u8, 0]); + let rz = m1.read_zipper_at_path(&[1u8]); + wz.graft(&rz); + wz.descend_last_byte(); + wz.meet_into(&rz, true); + assert_eq!(wz.path(), &[0u8]); + assert_eq!(*wz.get_val_or_set_mut_with(|| 3), 3); + assert_eq!(wz.val(), Some(&3)); + } + assert_eq!(m0.get_val_at(&[0u8, 0, 0]), Some(&3)); + } + /// The meet rule, dangling paths included. /// /// With `prune = false` a path exists in the result exactly when it exists in both operands, and From 76621956020436433a68ea1daaf50fb63b23f434 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 17:29:19 +0000 Subject: [PATCH 36/73] Keep a byte both dense nodes have when its contents meet to nothing Root cause: `CoFree::pmeet` (src/dense_byte_node.rs) returned `None` when neither the onward nodes nor the values of two co-frees met to anything, and `ByteNode::pmeet` then cleared the byte from the result mask. Under the meet rule a location exists in the result exactly when it exists in both operands, so the byte must survive as a dangling co-free. A co-free whose onward link is an empty node is dangling too, and was treated as holding a node, which made the identity masks wrong for it. The slim-pointer dispatch of `pmeet_dyn` also answered `None` for an empty node, where `EmptyNode::pmeet_dyn` answers an identity of the empty side. Fix: two co-frees always meet to at least the bare location; a dangling side (no value, no non-empty onward node) is an identity; otherwise each part is met and the identity mask is taken part by part, a part that meets to nothing being equal to the side that had nothing there. `map_meet_after_join_test` asserted master's rule (a meet of maps with no common value is empty); the maps share the first bytes of their paths, so the result now holds those as dangling paths and no value. Crate vs reference model, prune = false, 1M inputs, first differing op (before -> after): seed 101 / maxlen 120: meet_into 2985 -> 2671, meet_2 1181 -> 1156, meet_k_path_into 642 -> 644 seed 102 / maxlen 300: meet_into 7606 -> 6960, meet_2 3484 -> 3427, meet_k_path_into 1449 -> 1453 seed 103 / maxlen 600: meet_into 13755 -> 12777, meet_2 6838 -> 6728, meet_k_path_into 2503 -> 2504 Most remaining meet divergences go through list nodes (next commit). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 58 ++++++++++++++++++++++++++++++++++-------- src/lib.rs | 5 +++- src/trie_node.rs | 2 +- src/write_zipper.rs | 50 ++++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 12 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index e3515547..972c18c3 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -2024,18 +2024,56 @@ impl, Other rec_status.merge(val_status, true, true) } fn pmeet(&self, other: &OtherCf) -> AlgebraicResult { - //If one or the other cofree is dangling, it's an identity result for the dangling cofree - let mut identity_flag = 0; - if !self.has_rec() && !self.has_val() {identity_flag = SELF_IDENT;} - if !other.has_rec() && !other.has_val() {identity_flag |= COUNTER_IDENT;} - if identity_flag > 0 { - return AlgebraicResult::Identity(identity_flag) + //Both co-frees exist, so the location they stand at exists in both operands, and it survives + // the meet even when nothing at or below it does: the result is never `None`. An onward link + // to an empty node carries nothing, so it counts as no link at all. + let self_rec = self.rec().filter(|node| !node.as_tagged().node_is_empty()); + let other_rec = other.rec().filter(|node| !node.as_tagged().node_is_empty()); + let self_dangling = self_rec.is_none() && !self.has_val(); + let other_dangling = other_rec.is_none() && !other.has_val(); + if self_dangling || other_dangling { + //The meet is the bare location, which is exactly what a dangling side holds + let mut mask = 0; + if self_dangling { mask |= SELF_IDENT; } + if other_dangling { mask |= COUNTER_IDENT; } + return AlgebraicResult::Identity(mask) + } + + let rec = match (self_rec, other_rec) { + (Some(l), Some(r)) => l.pmeet(r), + _ => AlgebraicResult::None, + }; + let val = self.val().pmeet(&other.val()); + + //A part that meets to nothing equals the side that had nothing there + let (rec_self, rec_counter) = match &rec { + AlgebraicResult::Identity(mask) => (mask & SELF_IDENT > 0, mask & COUNTER_IDENT > 0), + AlgebraicResult::None => (self_rec.is_none(), other_rec.is_none()), + AlgebraicResult::Element(_) => (false, false), + }; + let (val_self, val_counter) = match &val { + AlgebraicResult::Identity(mask) => (mask & SELF_IDENT > 0, mask & COUNTER_IDENT > 0), + AlgebraicResult::None => (!self.has_val(), !other.has_val()), + AlgebraicResult::Element(_) => (false, false), + }; + let mut mask = 0; + if rec_self && val_self { mask |= SELF_IDENT; } + if rec_counter && val_counter { mask |= COUNTER_IDENT; } + if mask > 0 { + return AlgebraicResult::Identity(mask) } - //Otherwise actually work with what the cofrees contain - let rec = self.rec().pmeet(&other.rec()); - let val = self.val().pmeet(&other.val()); - self.combine_algebraic_results(other, rec, val) + let new_rec = match rec { + AlgebraicResult::Element(node) => Some(node), + AlgebraicResult::Identity(mask) => if mask & SELF_IDENT > 0 { self_rec.cloned() } else { other_rec.cloned() }, + AlgebraicResult::None => None, + }; + let new_val = match val { + AlgebraicResult::Element(val) => val, + AlgebraicResult::Identity(mask) => if mask & SELF_IDENT > 0 { self.val().cloned() } else { other.val().cloned() }, + AlgebraicResult::None => None, + }; + AlgebraicResult::Element(Self::new(new_rec, new_val)) } //GOAT, HeteroLattice will totally disappear when we do the policy refactor // fn join_all(_xs: &[&Self]) -> Self where Self: Sized { diff --git a/src/lib.rs b/src/lib.rs index 6d38dc03..25d6c2f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -593,8 +593,11 @@ mod tests { assert_eq!(met, l); } + //`l` and `r` hold no value in common, but they share the first bytes of their paths, and + // those locations survive the meet as dangling paths let met = met.meet(&r); - assert!(met.is_empty()); + assert_eq!(met.val_count(), 0); + assert!(!met.is_empty()); } #[test] diff --git a/src/trie_node.rs b/src/trie_node.rs index 70bb82b3..b87a99da 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -2205,7 +2205,7 @@ mod tagged_node_ref { } let (ptr, tag) = self.ptr.get_raw_parts(); match tag { - EMPTY_NODE_TAG => AlgebraicResult::None, + EMPTY_NODE_TAG => crate::empty_node::EmptyNode.pmeet_dyn(other), DENSE_BYTE_NODE_TAG => unsafe{ &*ptr.cast::>() }.pmeet_dyn(other), LINE_LIST_NODE_TAG => unsafe{ &*ptr.cast::>() }.pmeet_dyn(other), CELL_BYTE_NODE_TAG => unsafe{ &*ptr.cast::>() }.pmeet_dyn(other), diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 2163c1a2..d6786d33 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -4024,6 +4024,56 @@ mod tests { assert!(failures.is_empty(), "\n{}", failures.join("\n")); } + /// The tag of the root node of `map` + fn root_tag(map: &PathMap) -> usize { + map.root().unwrap().as_tagged().tag() + } + + /// Two dense nodes whose co-frees at a byte meet to nothing still share the byte, so it survives + /// the meet as a dangling path. A dangling byte -- a co-free with neither a value nor an onward + /// node, or one whose onward node is empty -- is what such a meet leaves, so it meets anything at + /// that byte as an identity of the dangling side. + #[test] + fn write_zipper_meet_into_dense_keeps_bytes_that_meet_to_nothing() { + type Build = fn() -> PathMap; + type Locations = Vec<(Vec, Option)>; + const BYTES: [u8; 4] = [1, 2, 3, 4]; + let values_at_0: Build = || { let mut m = PathMap::::new(); for b in BYTES { m.set_val_at(&[b, 0], 1); } m }; + let values_at_1: Build = || { let mut m = PathMap::::new(); for b in BYTES { m.set_val_at(&[b, 1], 2); } m }; + let dangling: Build = || { let mut m = PathMap::::new(); for b in BYTES { m.create_path(&[b]); } m }; + let emptied: Build = || { + let mut m = PathMap::::new(); + for b in BYTES { m.set_val_at(&[b, 0], 1); } + for b in BYTES { m.write_zipper_at_path(&[b]).remove_branches(false); } + m + }; + for build in [values_at_0, values_at_1, dangling, emptied] { + assert_eq!(root_tag(&build()), DENSE_BYTE_NODE_TAG); + } + let bare: Locations = core::iter::once((vec![], None)).chain(BYTES.iter().map(|b| (vec![*b], None))).collect(); + assert_eq!(all_locations(&dangling()), bare); + assert_eq!(all_locations(&emptied()), bare); + + // The values below each byte meet to nothing; the bytes stay + assert_eq!(all_locations(&values_at_0().meet(&values_at_1())), bare); + let mut dst = values_at_0(); + assert_eq!(dst.write_zipper().meet_into(&values_at_1().read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(all_locations(&dst), bare); + + // A dangling side is an identity for the meet, whichever form the dangling byte takes + for dangling_side in [dangling, emptied] { + for other in [values_at_0, values_at_1, dangling, emptied] { + let mut dst = dangling_side(); + assert_eq!(dst.write_zipper().meet_into(&other().read_zipper(), false), AlgebraicStatus::Identity); + assert_eq!(all_locations(&dst), bare); + let mut dst = other(); + let expected = if all_locations(&dst) == bare { AlgebraicStatus::Identity } else { AlgebraicStatus::Element }; + assert_eq!(dst.write_zipper().meet_into(&dangling_side().read_zipper(), false), expected); + assert_eq!(all_locations(&dst), bare); + } + } + } + /// Tests whether the [WriteZipper::subtract_into] operation will do the right thing with the root value #[test] fn write_zipper_subtract_into_test1() { From 96088cf20d2ea782c4e3aab1461bd952a5c5114a Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 17:31:40 +0000 Subject: [PATCH 37/73] Never remove the focus in meet_into, and make prune drop dangling paths below it Root cause: `WriteZipperCore::meet_into` (src/write_zipper.rs) called `prune_path()` whenever the node meet left nothing below the focus and `prune` was set, and removed the focus value with `remove_val(prune)`; both can remove the focus and its ancestors, which the meet rule forbids. `meet_k_path_into` did the same through `remove_branches(prune)`. And `prune = true` did not drop dangling paths: it only differed from `prune = false` in removing the focus. Fix: the value step removes with `prune = false`, and nothing left below the focus grafts `None` without pruning, so only what is below the focus and the focus value change. With `prune`, the meet result is passed through `node_drop_dangling` (src/trie_node.rs, with `drop_dangling` for dense and list nodes), which rebuilds the result keeping only the locations on the way to a value, and skips a node pointer-equal to the source's node at the same location (the relaxation the rule allows), so meeting a clone stays cheap. `meet_k_path_into` drops the dangling paths of the met map the same way before grafting it, and never prunes the focus. `write_zipper_meet_into_test4`, test 3, asserted master's behaviour: a pruned meet of dangling paths kept the one both sides had and reported `Element`. Under the rule `prune` drops it too, leaving nothing below the focus (`None`) and the focus itself in place. Added test 4 for the same meet without prune. `prune = true` is not fuzzed. prune = false divergences are unchanged by this commit (seed 101/120: 4543, 102/300: 11963, 103/600: 22167 at 1M inputs). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 50 ++++++++++ src/line_list_node.rs | 43 +++++++++ src/trie_node.rs | 56 +++++++++++ src/write_zipper.rs | 207 ++++++++++++++++++++++++++++++++--------- 4 files changed, 311 insertions(+), 45 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 972c18c3..a4e0561e 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -2537,6 +2537,56 @@ impl> ByteNode where Self: TrieNodeDowncast { + /// See [node_drop_dangling] + pub(crate) fn drop_dangling(&self, src: Option>) -> DropDangling { + let mut new_node: Option = None; + for (idx, byte) in self.mask.iter().enumerate() { + let cf = unsafe{ self.values.get_unchecked(idx) }; + let rec = match cf.rec() { + Some(child) => node_drop_dangling(child, meet_src_child(src, &[byte])), + None => DropDangling::Empty, + }; + let unchanged = match (&rec, cf.rec()) { + (DropDangling::Unchanged, _) => true, + (DropDangling::Empty, None) => cf.has_val(), + _ => false, + }; + if unchanged && new_node.is_none() { + continue + } + let new_node = new_node.get_or_insert_with(|| { + let mut node = Self::with_capacity_in(self.values.len(), self.alloc.clone()); + for (prev_idx, prev_byte) in self.mask.iter().enumerate().take(idx) { + node.set_cf(prev_byte, unsafe{ self.values.get_unchecked(prev_idx) }.rec().cloned(), unsafe{ self.values.get_unchecked(prev_idx) }.val().cloned()); + } + node + }); + let new_rec = match rec { + DropDangling::Unchanged => cf.rec().cloned(), + DropDangling::Empty => None, + DropDangling::New(node) => Some(node), + }; + if new_rec.is_some() || cf.has_val() { + new_node.set_cf(byte, new_rec, cf.val().cloned()); + } + } + match new_node { + None => DropDangling::Unchanged, + Some(node) if node.values.len() == 0 => DropDangling::Empty, + Some(node) => DropDangling::New(TrieNodeODRc::new_in(node, self.alloc.clone())), + } + } + fn set_cf(&mut self, byte: u8, rec: Option>, val: Option) { + if let Some(rec) = rec { + self.set_child(byte, rec); + } + if let Some(val) = val { + self.set_val(byte, val); + } + } +} + impl> ByteNode { fn prestrict>(&self, other: &ByteNode) -> AlgebraicResult where Self: Sized { // Iterate the overlap mask directly. Slot indexes are recovered with diff --git a/src/line_list_node.rs b/src/line_list_node.rs index 2ab29c91..b3eb6ffc 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -3040,6 +3040,49 @@ impl LineListNode { }) } + /// See [node_drop_dangling] + pub(crate) fn drop_dangling(&self, src: Option>) -> DropDangling { + let (key0, key1) = self.get_both_keys(); + let slot_result = |key: &[u8], is_child: bool, child: fn(&Self) -> &TrieNodeODRc| { + if is_child { + node_drop_dangling(child(self), meet_src_child(src, key)) + } else { + DropDangling::Unchanged + } + }; + let result0 = if self.is_used::<0>() { + slot_result(key0, self.is_child_ptr::<0>(), |node| unsafe{ node.child_in_slot::<0>() }) + } else { + DropDangling::Empty + }; + let result1 = if self.is_used::<1>() { + slot_result(key1, self.is_child_ptr::<1>(), |node| unsafe{ node.child_in_slot::<1>() }) + } else { + DropDangling::Empty + }; + let is_unchanged = |result: &DropDangling, used: bool| matches!(result, DropDangling::Unchanged) || (!used && matches!(result, DropDangling::Empty)); + if is_unchanged(&result0, self.is_used::<0>()) && is_unchanged(&result1, self.is_used::<1>()) { + return DropDangling::Unchanged + } + let payload = |result: DropDangling, slot: usize| match result { + DropDangling::Unchanged => if slot == 0 { self.clone_payload::<0>() } else { self.clone_payload::<1>() }, + DropDangling::Empty => None, + DropDangling::New(node) => Some(ValOrChild::Child(node)), + }; + let mut new_node = Self::new_in(self.alloc.clone()); + match (payload(result0, 0), payload(result1, 1)) { + (Some(payload0), Some(payload1)) => { + unsafe{ new_node.set_payload_owned::<0>(key0, payload0); } + unsafe{ new_node.set_payload_owned::<1>(key1, payload1); } + }, + (Some(payload), None) => unsafe{ new_node.set_payload_owned::<0>(key0, payload); }, + (None, Some(payload)) => unsafe{ new_node.set_payload_owned::<0>(key1, payload); }, + (None, None) => return DropDangling::Empty, + } + debug_assert!(validate_node(&new_node)); + DropDangling::New(TrieNodeODRc::new_in(new_node, self.alloc.clone())) + } + /// Part of the implementation of methods the remove subtries from a node fn remove_subtries(&mut self, remove_0: bool, remove_1: bool, key0_starts_with: bool, prune: bool, key_len: usize) { //NOTE: the order here is important because removing slot_0 first might shift the diff --git a/src/trie_node.rs b/src/trie_node.rs index b87a99da..16575816 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -688,6 +688,62 @@ pub(crate) fn pmeet_generic(mut node: TaggedNodeRef<'n, V, A>, mut key: &'k [u8]) -> (TaggedNodeRef<'n, V, A>, &'k [u8]) { + debug_assert!(key.len() > 0); + while let Some((consumed, child)) = node.node_get_child(key) { + if consumed >= key.len() { + break + } + node = child.as_tagged(); + key = &key[consumed..]; + } + (node, key) +} + +/// The onward node exactly at `key` in `src`, if there is one +#[inline] +pub(crate) fn meet_src_child<'a, V: Clone + Send + Sync, A: Allocator>(src: Option>, key: &[u8]) -> Option> { + let (node, rest) = meet_locate_key(src?, key); + match node.node_get_child(rest) { + Some((consumed, child)) if consumed == rest.len() => Some(child.as_tagged()), + _ => None + } +} + +/// The outcome of [node_drop_dangling] +pub(crate) enum DropDangling { + /// The node is kept as it is + Unchanged, + /// No value is left below the node's root + Empty, + /// The node with its dangling paths dropped + New(TrieNodeODRc), +} + +/// Drops the dangling paths below the root of `node`, keeping only the locations on the way to a +/// value. `src` is the node standing at the same location in the source of a pruned meet: a node +/// shared with it is skipped rather than walked, so a dangling path inside a shared node survives. +pub(crate) fn node_drop_dangling(node: &TrieNodeODRc, src: Option>) -> DropDangling { + let tagged = node.as_tagged(); + if tagged.node_is_empty() { + return DropDangling::Empty + } + if let Some(src) = src { + if tagged.shared_node_id() == src.shared_node_id() { + return DropDangling::Unchanged + } + } + match tagged.tag() { + DENSE_BYTE_NODE_TAG => unsafe{ tagged.as_dense_unchecked() }.drop_dangling(src), + LINE_LIST_NODE_TAG => unsafe{ tagged.as_list_unchecked() }.drop_dangling(src), + CELL_BYTE_NODE_TAG => unsafe{ tagged.as_cell_unchecked() }.drop_dangling(src), + _ => unreachable!() + } +} + pub(crate) fn node_count_branches_recursive(node: TaggedNodeRef, key: &[u8]) -> usize { if key.len() == 0 { return node.count_branches(b""); diff --git a/src/write_zipper.rs b/src/write_zipper.rs index d6786d33..8fc415f6 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -1973,8 +1973,21 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } else { PathMap::new_in(self.alloc.clone()) }; + //`prune` drops the dangling paths from the meet; the focus itself is never removed + let temp_map = if prune { + let alloc = temp_map.alloc.clone(); + let (root, root_val) = temp_map.into_root(); + let root = root.and_then(|root| match node_drop_dangling(&root, None) { + DropDangling::Unchanged => Some(root), + DropDangling::Empty => None, + DropDangling::New(new_root) => Some(new_root), + }); + PathMap::new_with_root_in(root, root_val, alloc) + } else { + temp_map + }; if temp_map.is_empty() { - self.remove_branches(prune); + self.remove_branches(false); false } else { self.graft_map(temp_map); @@ -2046,6 +2059,8 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC } /// See [ZipperWriting::meet_into] pub fn meet_into>(&mut self, read_zipper: &Z, prune: bool) -> AlgebraicStatus where V: Lattice { + //The focus is never removed, with or without `prune`: only what is below it can change, + // along with the focus value let src_root_val = read_zipper.val(); #[cfg(not(feature = "graft_root_vals"))] let _ = src_root_val; @@ -2054,60 +2069,77 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC (Some(self_val), Some(src_val)) => { let new_status = match self_val.pmeet(src_val) { AlgebraicResult::Element(new_val) => {self.set_val(new_val); AlgebraicStatus::Element }, - AlgebraicResult::None => {self.remove_val(prune); AlgebraicStatus::None }, + AlgebraicResult::None => {self.remove_val(false); AlgebraicStatus::None }, AlgebraicResult::Identity(_) => { AlgebraicStatus::Identity } }; (new_status, false) }, (None, Some(_)) => { (AlgebraicStatus::None, true) }, - (Some(_), None) => { self.remove_val(prune); (AlgebraicStatus::None, false) }, + (Some(_), None) => { self.remove_val(false); (AlgebraicStatus::None, false) }, (None, None) => { (AlgebraicStatus::None, true) }, }; - let node_was_none; - let node_status = match self.get_focus().try_as_tagged() { - Some(self_node) => { - if !self_node.node_is_empty() { - node_was_none = false; - let src = read_zipper.get_focus(); - if src.is_none() { - self.graft_internal(None); + let self_focus = self.get_focus(); + let node_was_none = match self_focus.try_as_tagged() { + Some(self_node) => self_node.node_is_empty(), + None => true + }; + let node_status = if node_was_none { + AlgebraicStatus::None + } else { + let src = read_zipper.get_focus(); + let result = match src.try_as_tagged() { + Some(src_node) => self_focus.as_tagged().pmeet_dyn(src_node), + None => AlgebraicResult::None, + }; + //With `prune`, only the locations on the way to a value are kept. A node shared with the + // source may be left as it is. + let drop_dangling = |node: TrieNodeODRc, src: Option>| -> Option> { + match node_drop_dangling(&node, src) { + DropDangling::Unchanged => Some(node), + DropDangling::Empty => None, + DropDangling::New(new_node) => Some(new_node), + } + }; + let (unchanged, new_node) = match result { + AlgebraicResult::Element(intersection) => { + (false, if prune { drop_dangling(intersection, src.try_as_tagged()) } else { Some(intersection) }) + }, + AlgebraicResult::None => (false, None), + AlgebraicResult::Identity(mask) => { + if mask & SELF_IDENT > 0 { if prune { - self.prune_path(); + let self_rc = self_focus.into_option().unwrap(); + match node_drop_dangling(&self_rc, src.try_as_tagged()) { + DropDangling::Unchanged => (true, None), + DropDangling::Empty => (false, None), + DropDangling::New(new_node) => (false, Some(new_node)), + } + } else { + (true, None) } - AlgebraicStatus::None } else { - match self_node.pmeet_dyn(src.as_tagged()) { - AlgebraicResult::Element(intersection) => { - self.graft_internal(Some(intersection)); - AlgebraicStatus::Element - }, - AlgebraicResult::None => { - self.graft_internal(None); - if prune { - self.prune_path(); - } - AlgebraicStatus::None - }, - AlgebraicResult::Identity(mask) => { - if mask & SELF_IDENT > 0 { - AlgebraicStatus::Identity - } else { - debug_assert_eq!(mask, COUNTER_IDENT); //It's gotta be self or other - self.graft_internal(Some(src.into_option().unwrap())); - AlgebraicStatus::Element - } - }, - } + debug_assert_eq!(mask, COUNTER_IDENT); //It's gotta be self or other + //The source's own node is shared with the source, so pruning may skip it + let src_is_shared = matches!(src.0, AbstractNodeRef::BorrowedRc(_)); + let src_rc = src.into_option().unwrap(); + (false, if prune && !src_is_shared { drop_dangling(src_rc, None) } else { Some(src_rc) }) + } + }, + }; + if unchanged { + AlgebraicStatus::Identity + } else { + match new_node { + Some(new_node) => { + self.graft_internal(Some(new_node)); + AlgebraicStatus::Element + }, + None => { + self.graft_internal(None); + AlgebraicStatus::None } - } else { - node_was_none = true; - AlgebraicStatus::None } - }, - None => { - node_was_none = true; - AlgebraicStatus::None } }; @@ -2116,6 +2148,7 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC #[cfg(feature = "graft_root_vals")] return node_status.merge(val_status, node_was_none, val_was_none) } + /// See [WriteZipper::meet_2] pub fn meet_2, ZB: ZipperInfallibleSubtries>(&mut self, rz_a: &ZA, rz_b: &ZB) -> AlgebraicStatus where V: Lattice { let a_focus = rz_a.get_focus(); @@ -3772,7 +3805,9 @@ mod tests { assert_eq!(btm.path_exists_at(&[1, 255, 0]), true); assert_eq!(btm.path_exists_at(&[0, 255, 0]), true); - // Test 3: meet from a higher level with all dangling paths and prune=true + // Test 3: meet from a higher level with all dangling paths and prune=true. With prune, only + // the locations on the way to a value survive, so the dangling path both sides have goes + // too, leaving nothing below the focus; the focus itself stays. let mut btm2: PathMap<()> = PathMap::new(); btm2.create_path(&[0, 255, 0]); btm2.create_path(&[0, 255, 1]); @@ -3783,16 +3818,35 @@ mod tests { let mut wz = zh2.write_zipper_at_exclusive_path(&[0]).unwrap(); let rz = zh2.read_zipper_at_path(&[1]).unwrap(); let alg_result = wz.meet_into(&rz, true); - assert_eq!(alg_result, AlgebraicStatus::Element); + assert_eq!(alg_result, AlgebraicStatus::None); drop(wz); drop(rz); drop(zh2); // Verify the meet operation did what it should have assert_eq!(btm2.path_exists_at(&[1, 255, 0]), true); - assert_eq!(btm2.path_exists_at(&[0, 255, 0]), true); + assert_eq!(btm2.path_exists_at(&[0]), true); + assert_eq!(btm2.path_exists_at(&[0, 255]), false); assert_eq!(btm2.path_exists_at(&[0, 200, 5]), false); assert_eq!(btm2.path_exists_at(&[0, 255, 1]), false); + + // Test 4: the same without prune keeps the path both sides have, and only that + let mut btm3: PathMap<()> = PathMap::new(); + btm3.create_path(&[0, 255, 0]); + btm3.create_path(&[0, 255, 1]); + btm3.create_path(&[0, 200, 5]); + btm3.create_path(&[1, 255, 0]); + let zh3 = btm3.zipper_head(); + + let mut wz = zh3.write_zipper_at_exclusive_path(&[0]).unwrap(); + let rz = zh3.read_zipper_at_path(&[1]).unwrap(); + assert_eq!(wz.meet_into(&rz, false), AlgebraicStatus::Element); + drop(wz); + drop(rz); + drop(zh3); + assert_eq!(btm3.path_exists_at(&[0, 255, 0]), true); + assert_eq!(btm3.path_exists_at(&[0, 200]), false); + assert_eq!(btm3.path_exists_at(&[0, 255, 1]), false); } /// Every existing location in `map` -- dangling paths included -- with its value, in @@ -4074,6 +4128,69 @@ mod tests { } } + /// `meet_into` never removes its focus: not when the focus value goes because the source has + /// none, and not when nothing is left below it, with or without `prune`. `prune` drops only the + /// dangling paths below the focus. + #[test] + fn write_zipper_meet_into_keeps_focus() { + type Locations = Vec<(Vec, Option)>; + let dst = || { let mut m = PathMap::::new(); m.set_val_at(&[5u8], 1); m.set_val_at(&[5u8, 0], 2); m.set_val_at(&[9u8], 9); m }; + let src = || { let mut m = PathMap::::new(); m.set_val_at(&[5u8, 1], 3); m }; + let focus_left: Locations = vec![(vec![], None), (vec![5], None), (vec![9], Some(9))]; + for prune in [false, true] { + let mut d = dst(); + let s = src(); + assert_eq!(d.write_zipper_at_path(&[5u8]).meet_into(&s.read_zipper_at_path(&[5u8]), prune), AlgebraicStatus::None, "prune = {prune}"); + assert_eq!(all_locations(&d), focus_left, "prune = {prune}"); + } + + // The source's focus value is gone, but a dangling path both sides have survives unless pruned + let dst = || { let mut m = PathMap::::new(); m.set_val_at(&[5u8], 1); m.create_path(&[5u8, 0, 0]); m.set_val_at(&[9u8], 9); m }; + let src = || { let mut m = PathMap::::new(); m.create_path(&[5u8, 0, 0]); m }; + let mut d = dst(); + assert_eq!(d.write_zipper_at_path(&[5u8]).meet_into(&src().read_zipper_at_path(&[5u8]), false), AlgebraicStatus::Element); + assert_eq!(all_locations(&d), vec![(vec![], None), (vec![5], None), (vec![5, 0], None), (vec![5, 0, 0], None), (vec![9], Some(9))]); + let mut d = dst(); + assert_eq!(d.write_zipper_at_path(&[5u8]).meet_into(&src().read_zipper_at_path(&[5u8]), true), AlgebraicStatus::None); + assert_eq!(all_locations(&d), focus_left); + } + + /// `meet_k_path_into` meets the subtries `k` bytes below the focus, dangling paths included, and + /// with `prune` drops the dangling paths from the result. The focus stays either way. + #[test] + fn write_zipper_meet_k_path_into_dangling_paths() { + let build = || { + let mut m = PathMap::::new(); + m.set_val_at(&[8u8], 8); + m.set_val_at(&[9u8, 1, 2], 5); + m.create_path(&[9u8, 1, 3]); + m.set_val_at(&[9u8, 2, 2], 6); + m.create_path(&[9u8, 2, 3]); + m + }; + let mut m = build(); + assert_eq!(m.write_zipper_at_path(&[9u8]).meet_k_path_into(1, false), true); + assert_eq!(all_locations(&m), vec![(vec![], None), (vec![8], Some(8)), (vec![9], None), (vec![9, 2], Some(5)), (vec![9, 3], None)]); + let mut m = build(); + assert_eq!(m.write_zipper_at_path(&[9u8]).meet_k_path_into(1, true), true); + assert_eq!(all_locations(&m), vec![(vec![], None), (vec![8], Some(8)), (vec![9], None), (vec![9, 2], Some(5))]); + + // Only a dangling path in common: kept without prune, and with prune nothing is left below + let build = || { + let mut m = PathMap::::new(); + m.set_val_at(&[8u8], 8); + m.create_path(&[9u8, 1, 3]); + m.create_path(&[9u8, 2, 3]); + m + }; + let mut m = build(); + assert_eq!(m.write_zipper_at_path(&[9u8]).meet_k_path_into(1, false), true); + assert_eq!(all_locations(&m), vec![(vec![], None), (vec![8], Some(8)), (vec![9], None), (vec![9, 3], None)]); + let mut m = build(); + assert_eq!(m.write_zipper_at_path(&[9u8]).meet_k_path_into(1, true), false); + assert_eq!(all_locations(&m), vec![(vec![], None), (vec![8], Some(8)), (vec![9], None)]); + } + /// Tests whether the [WriteZipper::subtract_into] operation will do the right thing with the root value #[test] fn write_zipper_subtract_into_test1() { From 15a392d479b3c09ee42fd31fe34507f60a43cdc0 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 17:37:01 +0000 Subject: [PATCH 38/73] Meet list nodes slot by slot, keeping the part of each key both sides have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: the list-node meet (`pmeet_generic` / `pmeet_generic_internal` in src/trie_node.rs, used by `LineListNode::pmeet_dyn_oriented` for list nodes and, swapped, for dense nodes met against list nodes) produced a result entry only where the other node had a payload at a slot's full key, or a value where the slot had an empty link. So, against the rule that a location exists in the result exactly when it exists in both operands: - a slot whose key the other side only has a prefix of was dropped, losing that prefix: meet({[0,0]}, {[0,1]}) gave {} instead of {[0]}; - a slot whose onward nodes met to nothing was dropped instead of left as a dangling path at its key; - a value slot met against a location without a value was dropped; - and because a dense node met against a list node is computed by the list node, the dense side's result depended on the list node's walk, so a dense destination met with a list source dropped a dangling path other layouts keep, and its `SELF_IDENT` relied on `COUNTER_IDENT` from an exhaustiveness test (`node_get_payloads`) that did not account for dangling paths. Fix: `meet_list_slot` locates each slot's key in the other node through its onward links, and contributes the slot's payload, a met value or onward node, or a dangling path along the deepest prefix of the key the other side has. Dangling contributions made redundant by the other slot are dropped so the node stays valid. `SELF_IDENT` is decided per slot and is exact. `COUNTER_IDENT` is only claimed when the list node computes a meet for a dense node (`swapped`), where it becomes the dense node's `SELF_IDENT`; it holds when each slot's contribution covers the other side along and below the key and `meet_other_within_slots` finds nothing else in the other node (no other branch off the keys, no value on them except under a value slot). `pmeet_generic*` and the `node_get_payloads` methods it was the only user of are removed, with the helpers that became unused. (The identity-mask fixes to `node_get_payloads` on branch worktree-agent-a769033384de53103, 9639511 and f1b1460, are not needed and were not cherry-picked.) Crate vs reference model, prune = false, first differing op in meet_into / meet_2 / meet_k_path_into (base 3c868e1 at 1M -> this commit): seed 101 / maxlen 120: 2985 / 1181 / 642 -> 0 / 0 / 0 (1M and 5M) seed 102 / maxlen 300: 7606 / 3484 / 1449 -> 0 / 0 / 0 (1M and 5M) seed 103 / maxlen 600: 13755 / 6838 / 2503 -> 0 / 0 / 0 (1M and 5M) Fresh seeds 9001/120, 9002/300, 9003/600: 2986 / 1137 / 617, 7461 / 3481 / 1440, 13893 / 6867 / 2582 at 1M -> 0 / 0 / 0 at 5M. ACT mode, 1M: seed 501/300 1698 meet_k_path_into divergences -> 0 in total; seed 502/600 3065 -> 0 (one `dump` value-bias divergence remains, as on base). The residue is to_prev_sibling_byte, subtract_into, and join_k_path_into value bias (MAP0), none downstream of a meet. meet benches (median, before -> after): superdense_meet 32000 252.9 -> 216.6 µs, sparse_meet 1600 76.87 -> 63.02 µs, binary_meet 1600 373.8 -> 354.7 µs; no bench slower by more than ~6%. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 96 --------- src/empty_node.rs | 3 - src/line_list_node.rs | 192 +++++++++--------- src/ring.rs | 30 --- src/tiny_node.rs | 27 --- src/trie_node.rs | 431 +++++++++++++++++------------------------ src/write_zipper.rs | 79 ++++++++ 7 files changed, 347 insertions(+), 511 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index a4e0561e..798fa530 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -840,102 +840,6 @@ impl> TrieNode let cf = self.get_mut(key[0]).unwrap(); *cf.rec_mut().unwrap() = new_node; } - fn node_get_payloads<'node, 'res>(&'node self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool { - //DISCUSSION: This function appears overly complicated primarily because it needs to track - // whether or not a both the val and the rec each cofree are requested, but we don't have a bitmask - // in advance that records vals and rec links separately. Since we don't want nested loops, we leverage - // the fact that a rec must be requested before a val, to stash the val for the next trip through the - // loop. The loop body therefore is an annoying state-machine. But at least it's not that much code. - - //Becomes true if only half of `(Some, Some)` CoFree is requested, without requesting the other half - // This flag never gets unset once it gets set - let mut unrequested_cofree_half = false; - //Temporary state that bridges across multiple requests into a `(Some, Some)` CoFree, by holding the - // val until it's requested, leveraging the fact that values are requested after rec links - let mut stashed_val: Option<&V> = None; - //Tracks whether the current CoFree's val has been taken. So, `last_byte` toggles to `Some` and stays - // at `Some` until we move onto a different CF, while `stashed_val` toggles to `Some`, and toggles back - // as soon as the value is requested. - let mut last_byte: Option = None; - //Tracks which CoFrees have yet to be requested from the node - let mut requested_mask = ByteMask::from(self.mask); - - debug_assert!(results.len() >= keys.len()); - for ((key, expect_val), (result_key_len, payload_ref)) in keys.into_iter().zip(results.iter_mut()) { - if key.len() > 0 { - let byte = key[0]; - - //Check to see if we had a Val from the CoFree that we aren't going to request - match &last_byte { - Some(prev_byte) => { - if byte != *prev_byte { - if stashed_val.is_some() { - unrequested_cofree_half = true; - } - stashed_val = None; - last_byte = None; - } - }, - None => {} - } - - //Check to see if this trip through the loop is the request for the stashed val - match stashed_val { - Some(val) => { - if key.len() == 1 && *expect_val { - *result_key_len = 1; - *payload_ref = PayloadRef::Val(val); - stashed_val = None; - continue; - } - }, - None => {} - } - - requested_mask.clear_bit(byte); - match self.get(byte) { - Some(cf) => { - // An exact value-only request does not enumerate an onward link stored in the - // same CoFree. The preceding stashed-value fast path means this branch is - // reached only when that link was not requested separately. - if key.len() == 1 && *expect_val && cf.has_rec() { - unrequested_cofree_half = true; - } - - //A key longer than 1 byte or an explicit request for a rec link can be answered with a Child - if key.len() > 1 || !*expect_val { - match cf.rec() { - Some(rec) => { - *result_key_len = 1; - *payload_ref = PayloadRef::Child(rec); - }, - None => {} - } - } - match cf.val() { - Some(val) => { - //Answer an explicit request for this val, or stash the val for - if key.len() == 1 && *expect_val { - debug_assert!(stashed_val.is_none()); - *result_key_len = 1; - *payload_ref = PayloadRef::Val(val); - } else { - if last_byte.is_none() { - stashed_val = Some(val); - last_byte = Some(byte); - } - } - }, - None => {} - } - }, - None => {} - } - } - } - - !unrequested_cofree_half && stashed_val.is_none() && requested_mask.is_empty_mask() - } fn node_contains_val(&self, key: &[u8]) -> bool { if key.len() == 1 { match self.get(key[0]) { diff --git a/src/empty_node.rs b/src/empty_node.rs index ce336dd1..0cf79d06 100644 --- a/src/empty_node.rs +++ b/src/empty_node.rs @@ -26,9 +26,6 @@ impl TrieNode for EmptyNode { fn node_replace_child(&mut self, _key: &[u8], _new_node: TrieNodeODRc) { unreachable!() //Should not be called unless it's known that the node being replaced exists } - fn node_get_payloads<'node, 'res>(&'node self, _keys: &[(&[u8], bool)], _results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool { - true - } fn node_contains_val(&self, _key: &[u8]) -> bool { false } diff --git a/src/line_list_node.rs b/src/line_list_node.rs index b3eb6ffc..5dc47fe5 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -1804,52 +1804,6 @@ impl TrieNode for LineListNode debug_assert!(consumed_bytes == key.len()); *child_node = new_node; } - fn node_get_payloads<'node, 'res>(&'node self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool { - let mut slot_0_requested = !self.is_used::<0>(); - let mut slot_1_requested = !self.is_used::<1>(); - let (node_key_0, node_key_1) = self.get_both_keys(); - - debug_assert!(results.len() >= keys.len()); - for ((key, expect_val), (result_key_len, payload_ref)) in keys.into_iter().zip(results.iter_mut()) { - if self.is_used::<0>() { - if starts_with(key, node_key_0) { - let node_key_len = node_key_0.len(); - if self.is_child_ptr::<0>() { - if !*expect_val || node_key_len < key.len() { - slot_0_requested = true; - *result_key_len = node_key_len; - *payload_ref = PayloadRef::Child(unsafe{ &*self.val_or_child0.child }); - } - } else { - if *expect_val && node_key_len == key.len() { - slot_0_requested = true; - *result_key_len = node_key_len; - *payload_ref = PayloadRef::Val(unsafe{ &**self.val_or_child0.val }); - } - } - } - } - if self.is_used::<1>() { - if starts_with(key, node_key_1) { - let node_key_len = node_key_1.len(); - if self.is_child_ptr::<1>() { - if !*expect_val || node_key_len < key.len() { - slot_1_requested = true; - *result_key_len = node_key_len; - *payload_ref = PayloadRef::Child(unsafe{ &*self.val_or_child1.child }); - } - } else { - if *expect_val && node_key_len == key.len() { - slot_1_requested = true; - *result_key_len = node_key_len; - *payload_ref = PayloadRef::Val(unsafe{ &**self.val_or_child1.val }); - } - } - } - } - } - slot_0_requested && slot_1_requested - } fn node_contains_val(&self, key: &[u8]) -> bool { self.contains_val(key) } @@ -2976,68 +2930,106 @@ impl TrieNode for LineListNode impl LineListNode { /// The body of [TrieNode::pmeet_dyn]. `swapped` means `self` is really the *right* operand of - /// the meet and `other` the left one; see [pmeet_generic]. A node type that cannot enumerate - /// its own payloads cheaply (a `ByteNode`) meets a list node by calling this with `swapped = - /// true` and inverting the identity mask of the result. + /// the meet and `other` the left one. A node type that cannot enumerate its own payloads cheaply + /// (a `ByteNode`) meets a list node by calling this with `swapped = true` and inverting the + /// identity mask of the result. + /// + /// The meet is left-biased (a `Lattice` impl resolves a collision as `left.pmeet(right)`), so + /// with `swapped` every value and every recursive node meet is computed as `other op self` and + /// its identity mask re-expressed relative to `self`. + /// + /// A location survives the meet exactly when both operands have it, dangling or not. So each + /// slot contributes the deepest prefix of its key that `other` also has, even when nothing at or + /// below that prefix survives. + /// + /// `SELF_IDENT` in the result is exact. `COUNTER_IDENT` is only ever claimed with `swapped`, + /// where the caller inverts it into its own `SELF_IDENT`, and there it is exact too. pub(crate) fn pmeet_dyn_oriented(&self, other: TaggedNodeRef, swapped: bool) -> AlgebraicResult> where V: Lattice { debug_assert!(validate_node(self)); - let mut self_payloads_buf: [(&[u8], PayloadRef); 2] = [(&[], PayloadRef::None); 2]; - - //A shadowed dangling slot carries nothing (see `slot_is_shadowed_dangling`), and a dangling - // path survives no meet, so it can only ever answer `None` and drag the identity mask to - // zero -- making the node report `Element` for a meet result that holds exactly what - // `self` holds. Leaving it out of the meet altogether gives the same result trie with an - // identity mask that tells the truth. - let skipped = if self.slot_is_shadowed_dangling(0) { - Some(0) - } else if self.slot_is_shadowed_dangling(1) { - Some(1) - } else { - None + //A shadowed dangling slot carries nothing (see `slot_is_shadowed_dangling`): the location it + // names is already carried by its sibling. Leaving it out of the meet gives the same trie. + let (use0, use1) = match self.used_slot_count() { + 0 => return AlgebraicResult::None, + 1 => (true, false), + _ => { + if self.slot_is_shadowed_dangling(0) { + (false, true) + } else if self.slot_is_shadowed_dangling(1) { + (true, false) + } else { + (true, true) + } + } }; + let (key0, key1) = self.get_both_keys(); - let self_slot_count = self.used_slot_count(); - let self_payloads = match (self_slot_count, skipped) { - (0, _) => return AlgebraicResult::None, - (_, Some(0)) => { - let key = unsafe{ self.key_unchecked::<1>() }; - let payload = unsafe{ self.payload_in_slot::<1>() }; - self_payloads_buf[0] = (key, payload); - &self_payloads_buf[..1] - }, - (1, _) | (_, Some(1)) => { - let key = unsafe{ self.key_unchecked::<0>() }; - let payload = unsafe{ self.payload_in_slot::<0>() }; - self_payloads_buf[0] = (key, payload); - &self_payloads_buf[..1] - }, - (2, None) => { - let (key0, key1) = self.get_both_keys(); - let payload0 = unsafe{ self.payload_in_slot::<0>() }; - let payload1 = unsafe{ self.payload_in_slot::<1>() }; - self_payloads_buf[0] = (key0, payload0); - self_payloads_buf[1] = (key1, payload1); - &self_payloads_buf[..2] - }, - _ => unsafe{ unreachable_unchecked() } + let out0 = if use0 { + meet_list_slot(key0, unsafe{ self.payload_in_slot::<0>() }, other, swapped) + } else { + (true, true, SlotMeet::Skipped) + }; + let out1 = if use1 { + meet_list_slot(key1, unsafe{ self.payload_in_slot::<1>() }, other, swapped) + } else { + (true, true, SlotMeet::Skipped) }; + if out0.2.is_nothing() && out1.2.is_nothing() { + return AlgebraicResult::None + } - pmeet_generic::<2, V, A, _>(self_payloads, other, swapped, |payloads| { - debug_assert_eq!(payloads.len(), self_payloads.len()); - //With a slot skipped, the single result belongs to the slot that stayed in, and the - // skipped one is dropped -- which is what the meet would have done with it anyway. - let (slot0_payload, slot1_payload) = match skipped { - Some(0) => (None, payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into())), - Some(_) => (payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into()), None), - None => ( - payloads.get_mut(0).and_then(|p| core::mem::take(p)).map(|p| p.into()), - payloads.get_mut(1).and_then(|p| core::mem::take(p)).map(|p| p.into()), - ), + let mut mask = 0; + if out0.0 && out1.0 { + mask |= SELF_IDENT; + } + if swapped && out0.1 && out1.1 { + let slots = [ + if use0 { Some((key0, self.is_child_ptr::<0>(), out0.2.reach(key0.len()))) } else { None }, + if use1 { Some((key1, self.is_child_ptr::<1>(), out1.2.reach(key1.len()))) } else { None }, + ]; + if meet_other_within_slots(other, &slots) { + mask |= COUNTER_IDENT; + } + } + if mask > 0 { + return AlgebraicResult::Identity(mask) + } + + //Build the result from what each slot contributes + let mut items: [Option<(&[u8], ValOrChild)>; 2] = [ + out0.2.into_item(key0, || self.clone_payload::<0>().unwrap()), + out1.2.into_item(key1, || self.clone_payload::<1>().unwrap()), + ]; + //A dangling path is redundant where the other item's path runs through it, or stands at it + // with a value or an onward node. Dropping it keeps the node valid (no onward link in slot_0 + // that slot_1 extends, no two onward links under one key). + for i in 0..2 { + let j = 1 - i; + let redundant = match (&items[i], &items[j]) { + (Some((key_i, ValOrChild::Child(child_i))), Some((key_j, payload_j))) if child_i.as_tagged().node_is_empty() && key_j.starts_with(key_i) => { + let j_dangling = matches!(payload_j, ValOrChild::Child(child_j) if child_j.as_tagged().node_is_empty()); + key_j.len() > key_i.len() || !j_dangling || i > j + }, + _ => false }; - let new_node = self.clone_with_updated_payloads(slot0_payload, slot1_payload).unwrap(); - TrieNodeODRc::new_in(new_node, self.alloc.clone()) - }) + if redundant { + items[i] = None; + } + } + let mut new_node = Self::new_in(self.alloc.clone()); + let [item0, item1] = items; + match (item0, item1) { + (Some((key0, payload0)), Some((key1, payload1))) => { + unsafe{ new_node.set_payload_owned::<0>(key0, payload0); } + unsafe{ new_node.set_payload_owned::<1>(key1, payload1); } + }, + (Some((key, payload)), None) | (None, Some((key, payload))) => { + unsafe{ new_node.set_payload_owned::<0>(key, payload); } + }, + (None, None) => unreachable!() + } + debug_assert!(validate_node(&new_node)); + AlgebraicResult::Element(TrieNodeODRc::new_in(new_node, self.alloc.clone())) } /// See [node_drop_dangling] diff --git a/src/ring.rs b/src/ring.rs index cf94e210..0452fea1 100644 --- a/src/ring.rs +++ b/src/ring.rs @@ -425,36 +425,6 @@ impl FatAlgebraicResult { pub(crate) const fn new(identity_mask: u64, element: Option) -> Self { Self {identity_mask, element} } - /// Converts an [AlgebraicResult] into a `FatAlgebraicResult`, assuming the source `result` was the - /// output of a binary operation (two arguments). - #[inline] - pub(crate) fn from_binary_op_result(result: AlgebraicResult, a: &V, b: &V) -> Self - where V: Clone - { - match result { - AlgebraicResult::None => FatAlgebraicResult::none(), - AlgebraicResult::Element(v) => FatAlgebraicResult::element(v), - AlgebraicResult::Identity(mask) => { - debug_assert!(mask <= (SELF_IDENT | COUNTER_IDENT)); - if mask & SELF_IDENT > 0 { - FatAlgebraicResult::new(mask, Some(a.clone())) - } else { - debug_assert_eq!(mask, COUNTER_IDENT); - FatAlgebraicResult::new(mask, Some(b.clone())) - } - } - } - } - /// Maps a `FatAlgebraicResult` to `FatAlgebraicResult` by applying a function to a contained value - #[inline] - pub fn map(self, f: F) -> FatAlgebraicResult - where F: FnOnce(V) -> U, - { - FatAlgebraicResult:: { - identity_mask: self.identity_mask, - element: self.element.map(f) - } - } /// The result of an operation between non-none arguments that results in None #[inline(always)] pub(crate) const fn none() -> Self { diff --git a/src/tiny_node.rs b/src/tiny_node.rs index 6ece3c72..c76ad615 100644 --- a/src/tiny_node.rs +++ b/src/tiny_node.rs @@ -149,33 +149,6 @@ impl<'a, V: Clone + Send + Sync, A: Allocator> TrieNode for TinyRefNode<'a } fn node_get_child_mut(&mut self, _key: &[u8]) -> Option<(usize, &mut TrieNodeODRc)> { unreachable!() } fn node_replace_child(&mut self, _key: &[u8], _new_node: TrieNodeODRc) { unreachable!() } - fn node_get_payloads<'node, 'res>(&'node self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool { - if self.node_is_empty() { - return true - } - let mut requested_contained_item = false; // This node type only has one item - let self_key = self.key(); - debug_assert!(results.len() >= keys.len()); - for ((key, expect_val), (result_key_len, payload_ref)) in keys.into_iter().zip(results.into_iter()) { - if starts_with(key, self_key) { - let self_key_len = self_key.len(); - if self.is_child_ptr() { - if !*expect_val || self_key_len < key.len() { - requested_contained_item = true; - *result_key_len = self_key_len; - *payload_ref = PayloadRef::Child(unsafe{ &*self.payload.child }); - } - } else { - if *expect_val && self_key_len == key.len() { - requested_contained_item = true; - *result_key_len = self_key_len; - *payload_ref = PayloadRef::Val(unsafe{ &**self.payload.val }); - } - } - } - } - requested_contained_item - } fn node_contains_val(&self, key: &[u8]) -> bool { if self.is_used_val() { let node_key = self.key(); diff --git a/src/trie_node.rs b/src/trie_node.rs index 16575816..dd5f2f95 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -5,7 +5,6 @@ use core::ptr::NonNull; use std::collections::HashMap; use dyn_clone::*; use local_or_heap::LocalOrHeap; -use arrayvec::ArrayVec; use crate::utils::ByteMask; use crate::alloc::Allocator; @@ -72,35 +71,6 @@ pub(crate) trait TrieNode: TrieNodeDowncas /// cheaper, but it is adequate for the places that call it fn node_replace_child(&mut self, key: &[u8], new_node: TrieNodeODRc); - /// Retrieves multiple values or child links from the node, associated with elements from `keys`, - /// and places them into the respective element in `results` - /// - /// The `bool` in `keys` indicates whether a value is expected at the requested key. `true` will be - /// passed to indicate a **value**. (WARNING: This is different from the convention in some node types) - /// - /// If a node contains both an onward link and a value at the same key, the `bool` specifies which to - /// return; however, a node may be returned for a requested value, if the path to the node is a prefix - /// to the path to the requested value. This is because the value may live within a child node. On - /// the other hand, a value will only be returned if it is an exact match with the key provided. - /// - /// The `usize` in `results` functions the same way as the returned `usize` in [TrieNode::node_get_child], - /// to indicate the number of key bytes matched by the key contained within the node. - /// - /// The implementation may assume `keys` will be in sorted order, and `false` sorts before `true` if - /// both a value and a node at the same key are requested. - /// - /// Returns `true` if the requested `keys` completely enumerate the set of elements contained within - /// the node, or `false` if the node contains additional elements that were not requested - /// - /// If a result is not found for a given key, the implementation does not guarantee the corresponding - /// element in `results` will be set to [`PayloadRef::None`], therefore the caller should init `results` - /// to default values. - /// - /// Panics if `keys.len() > results.len()` - /// - /// NOTE: It perfectly fine for multiple keys to share a prefix, and sometimes that means multiple - /// results will be identical if the node represents only the prefix portion of the key. - fn node_get_payloads<'node, 'res>(&'node self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'node, V, A>)]) -> bool; /// Returns `true` if the node contains a value at the specified key, otherwise returns `false` /// @@ -526,32 +496,6 @@ impl Default for PayloadRef<'_, V, A> { } } -impl<'a, V: Clone + Send + Sync, A: Allocator> PayloadRef<'a, V, A> { - pub fn is_none(&self) -> bool { - match self { - Self::None => true, - _ => false - } - } - pub fn is_val(&self) -> bool { - match self { - Self::Val(_) => true, - _ => false - } - } - pub fn child(&self) -> &'a TrieNodeODRc { - match self { - Self::Child(child) => child, - _ => panic!() - } - } - pub fn val(&self) -> &'a V { - match self { - Self::Val(val) => val, - _ => panic!() - } - } -} #[derive(Clone)] pub(crate) enum ValOrChild { @@ -630,64 +574,6 @@ impl ValOrChildUnion { } } -/// An implementation of pmeet_dyn that should be correct for any two node types, Although it -/// certainly won't be optimally efficient. -/// -/// WARNING: just like [TrieNode::node_get_payloads], the keys in `self_payloads` must be in -/// sorted order. -// -//NOTE: I have confirmed that this function behaves no more conservatively than the function it replaced. -// In other words, I have confirmed that, in tests where the old function was behaving correctly, this -// function returns *identical* results. Furthermore those same tests are the ones where the 20% slowdown -// was observed. Therefore the the ~20% slowdown is simply the higher overheads of this generic function. -// -//The next port of call for optimization is probably to remove the recursion -// -/// `swapped` says which operand `self_payloads` came from. The meet is left-biased (a `Lattice` -/// impl resolves a collision as `left.pmeet(right)`), so when a caller enumerates the *right* -/// operand's payloads because that node type is the easier one to iterate, it passes `swapped = -/// true`: every value and every recursive node meet is then computed as `other op self` and the -/// identity masks are re-expressed relative to `self_payloads`. The caller still applies -/// `invert_identity()` to the final result to get back to its own orientation. Without this, a -/// dense-node-versus-list-node meet returned the list node's values regardless of which side it -/// was on. -pub(crate) fn pmeet_generic(self_payloads: &[(&[u8], PayloadRef)], other: TaggedNodeRef, swapped: bool, merge_f: MergeF) -> AlgebraicResult> - where - MergeF: FnOnce(&mut [Option>]) -> TrieNodeODRc, - V: Clone + Send + Sync + Lattice -{ - let mut request_keys = ArrayVec::<(&[u8], bool), MAX_PAYLOAD_CNT>::new(); - let mut element_results = ArrayVec::>, MAX_PAYLOAD_CNT>::new(); - let mut request_results = ArrayVec::<(usize, PayloadRef), MAX_PAYLOAD_CNT>::new(); - for (self_key, self_payload) in self_payloads.iter() { - debug_assert!(!self_payload.is_none()); - request_keys.push((self_key, self_payload.is_val())); - element_results.push(FatAlgebraicResult::none()); - request_results.push((0, PayloadRef::default())); - } - - let is_exhaustive = pmeet_generic_internal::(self_payloads, &mut request_keys[..], &mut request_results[..], &mut element_results[..], other, swapped); - let mut is_none = true; - let mut combined_mask = SELF_IDENT | COUNTER_IDENT; - let mut result_payloads = ArrayVec::>, MAX_PAYLOAD_CNT>::new(); - for result in element_results { - combined_mask &= result.identity_mask; - is_none = is_none && result.element.is_none(); - result_payloads.push(result.element); - } - - if is_none { - return AlgebraicResult::None - } - if !is_exhaustive { - combined_mask &= !COUNTER_IDENT; - } - if combined_mask > 0 { - return AlgebraicResult::Identity(combined_mask) - } - AlgebraicResult::Element(merge_f(&mut result_payloads[..])) -} - /// Where `key` lands in `node`: follows the onward links that cover a strict prefix of `key`, and /// returns the node that holds the rest of the key, together with that rest (never empty) #[inline] @@ -744,6 +630,182 @@ pub(crate) fn node_drop_dangling(node: &Tr } } +/// The number of branches below `key` in the trie rooted at `node` +fn meet_count_branches_at(node: TaggedNodeRef, key: &[u8]) -> usize { + if key.is_empty() { + return node.count_branches(&[]) + } + let (node, rest) = meet_locate_key(node, key); + match node.node_get_child(rest) { + Some((consumed, child)) if consumed == rest.len() => child.as_tagged().count_branches(&[]), + _ => node.count_branches(rest) + } +} + +/// What one slot of a list node contributes to a meet; see [meet_list_slot] +pub(crate) enum SlotMeet { + /// The slot was left out of the meet + Skipped, + /// Not even the first byte of the slot's key exists in the other operand + Nothing, + /// The slot's own payload, unchanged + Keep, + /// A value at the slot's key + Val(V), + /// An onward node at the slot's key + Child(TrieNodeODRc), + /// A dangling path along the first `n` bytes of the slot's key + Dangling(usize), +} + +impl SlotMeet { + #[inline] + pub(crate) fn is_nothing(&self) -> bool { + matches!(self, Self::Nothing | Self::Skipped) + } + /// How many bytes of the slot's key exist in the result + #[inline] + pub(crate) fn reach(&self, key_len: usize) -> usize { + match self { + Self::Skipped | Self::Nothing => 0, + Self::Dangling(n) => *n, + _ => key_len + } + } + /// The key and payload of the contribution; `keep` supplies the slot's own payload + #[inline] + pub(crate) fn into_item<'k, F: FnOnce() -> ValOrChild>(self, key: &'k [u8], keep: F) -> Option<(&'k [u8], ValOrChild)> { + match self { + Self::Skipped | Self::Nothing => None, + Self::Keep => Some((key, keep())), + Self::Val(val) => Some((key, ValOrChild::Val(val))), + Self::Child(node) => Some((key, ValOrChild::Child(node))), + Self::Dangling(n) => Some((&key[..n], ValOrChild::Child(TrieNodeODRc::new_empty()))), + } + } +} + +/// Meets one slot of a list node, `payload` at `key`, against `other`, following the rule that a +/// location survives exactly when both operands have it, and a value exactly when both hold one. +/// +/// Returns `(self_ident, counter_ok, contribution)`. `self_ident` is true exactly when the +/// contribution equals the slot. `counter_ok` is true when, along this slot's key and below it, +/// the contribution holds everything `other` holds there; [meet_other_within_slots] checks the rest. +/// `swapped` orients the meet as in [crate::line_list_node::LineListNode::pmeet_dyn_oriented]. +pub(crate) fn meet_list_slot(key: &[u8], payload: PayloadRef, other: TaggedNodeRef, swapped: bool) -> (bool, bool, SlotMeet) { + let (node, rest) = meet_locate_key(other, key); + let exact_child = match node.node_get_child(rest) { + Some((consumed, child)) if consumed == rest.len() => Some(child), + _ => None + }; + let other_val = node.node_get_val(rest); + if exact_child.is_none() && other_val.is_none() && !node.node_contains_partial_key(rest) { + //`other` stops partway along the key: the part both have is a dangling path + let reach = key.len() - rest.len() + node.node_key_overlap(rest); + return (false, true, if reach > 0 { SlotMeet::Dangling(reach) } else { SlotMeet::Nothing }) + } + + match payload { + PayloadRef::Val(self_val) => match other_val { + Some(other_val) => { + let result = if swapped { other_val.pmeet(self_val).invert_identity() } else { self_val.pmeet(other_val) }; + match result { + AlgebraicResult::Identity(mask) => if mask & SELF_IDENT > 0 { + (true, mask & COUNTER_IDENT > 0, SlotMeet::Keep) + } else { + (false, true, SlotMeet::Val(other_val.clone())) + }, + AlgebraicResult::Element(val) => (false, false, SlotMeet::Val(val)), + AlgebraicResult::None => (false, false, SlotMeet::Dangling(key.len())), + } + }, + None => (false, true, SlotMeet::Dangling(key.len())), + }, + PayloadRef::Child(self_child) => { + let onward = match exact_child { + Some(child) => AbstractNodeRef::BorrowedRc(child), + None => node.get_node_at_key(rest), + }; + let self_empty = self_child.as_tagged().node_is_empty(); + let other_empty = match onward.try_as_tagged() { + Some(below) => below.node_is_empty(), + None => true + }; + if self_empty || other_empty { + //Nothing below the key survives, which leaves the key itself + let contribution = if self_empty { SlotMeet::Keep } else { SlotMeet::Dangling(key.len()) }; + return (self_empty, other_empty, contribution) + } + let result = { + let other_below = onward.as_tagged(); + if swapped { + other_below.pmeet_dyn(self_child.as_tagged()).invert_identity() + } else { + self_child.as_tagged().pmeet_dyn(other_below) + } + }; + match result { + AlgebraicResult::Identity(mask) => if mask & SELF_IDENT > 0 { + (true, mask & COUNTER_IDENT > 0, SlotMeet::Keep) + } else { + (false, true, SlotMeet::Child(onward.into_option().unwrap())) + }, + AlgebraicResult::Element(node) => (false, false, SlotMeet::Child(node)), + AlgebraicResult::None => (false, false, SlotMeet::Dangling(key.len())), + } + }, + PayloadRef::None => unreachable!() + } +} + +/// Given the slots `(key, is_child, reach)` of a list node that have each been met against `other` +/// with `counter_ok` (see [meet_list_slot]), is all of `other` inside the result? That holds when +/// `other` branches nowhere off the paths the slots reach, and holds no value on them except where a +/// value slot stands. Below a slot's onward node, the node meet has already answered. +pub(crate) fn meet_other_within_slots(other: TaggedNodeRef, slots: &[Option<(&[u8], bool, usize)>; 2]) -> bool { + //The distinct bytes the slots continue with after `prefix` + let branches_expected = |prefix: &[u8]| -> usize { + let depth = prefix.len(); + let mut first = None; + let mut count = 0; + for (key, _, reach) in slots.iter().flatten() { + if *reach > depth && &key[..depth] == prefix { + let byte = key[depth]; + if first != Some(byte) { + count += 1; + first = Some(byte); + } + } + } + count + }; + if meet_count_branches_at(other, &[]) != branches_expected(&[]) { + return false + } + for (key, _, reach) in slots.iter().flatten() { + for depth in 1..=*reach { + let prefix = &key[..depth]; + //A value slot can share its key with an onward-node slot, which then answers for below + let below_is_met = slots.iter().flatten().any(|(slot_key, slot_is_child, slot_reach)| { + *slot_is_child && *slot_reach == depth && *slot_key == prefix + }); + if !below_is_met && meet_count_branches_at(other, prefix) != branches_expected(prefix) { + return false + } + let (node, rest) = meet_locate_key(other, prefix); + if node.node_get_val(rest).is_some() { + let covered = slots.iter().flatten().any(|(slot_key, slot_is_child, slot_reach)| { + !*slot_is_child && *slot_reach == depth && *slot_key == prefix + }); + if !covered { + return false + } + } + } + } + true +} + pub(crate) fn node_count_branches_recursive(node: TaggedNodeRef, key: &[u8]) -> usize { if key.len() == 0 { return node.count_branches(b""); @@ -761,128 +823,7 @@ pub(crate) fn node_count_branches_recursive(self_payloads: &[(&[u8], PayloadRef)], keys: &mut [(&[u8], bool)], request_results: &mut [(usize, PayloadRef<'trie, V, A>)], results: &mut [FatAlgebraicResult>], other_node: TaggedNodeRef<'trie, V, A>, swapped: bool) -> bool - where V: Clone + Send + Sync + Lattice -{ - //If is_exhaustive gets set to `false`, then the pmeet method cannot return a `COUNTER_IDENTITY` result - let mut is_exhaustive = true; - - //Get the payload results from the node - if !other_node.node_get_payloads(&keys[..], request_results) { - is_exhaustive = false; - } - - //Divide the results into groups based on the returned node. Because keys must be - // in sorted order, we can assume that query results returning the same node will - // be contiguous. - //NOTE: It's theoretically possible (although pretty unlikely) that a node will - // have multiple discontinuous internal paths leading to the same child node, however - // the TrieNodeODRc pointers will be different in that case, so this logic is still - // correct. - let mut cur_group: Option<(usize, &TrieNodeODRc)> = None; - for idx in 0..keys.len() { - let (consumed_bytes, payload) = core::mem::take(request_results.get_mut(idx).unwrap()); - if !payload.is_none() { - let is_val = keys[idx].1; - if consumed_bytes < keys[idx].0.len() { - keys[idx].0 = &keys[idx].0[consumed_bytes..]; - debug_assert!(!payload.is_val()); - let child = payload.child(); - - //Continue to grow range, or do the recursive call, depending on whether - // we have the same node as the previous time through the loop - if cur_group.is_some() { - if (cur_group.as_ref().unwrap().1 as *const TrieNodeODRc) != (child as *const TrieNodeODRc) { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results, swapped); - cur_group = Some((idx, child)); - } - } else { - cur_group = Some((idx, child)); - } - } else { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results, swapped); - - //We've arrived at a contained value or onward link that has a correspondence - // to one of the values or links in `self` - debug_assert_eq!(consumed_bytes, keys[idx].0.len()); - debug_assert_eq!(is_val, payload.is_val()); - let result = match &self_payloads[idx].1 { - PayloadRef::Child(self_link) => { - let other_link = payload.child(); - let result = if swapped { other_link.pmeet(self_link).invert_identity() } else { self_link.pmeet(other_link) }; - FatAlgebraicResult::from_binary_op_result(result, self_link, other_link) - .map(|child| ValOrChild::Child(child)) - }, - PayloadRef::Val(self_val) => { - let other_val = payload.val(); - let result = if swapped { other_val.pmeet(*self_val).invert_identity() } else { (*self_val).pmeet(other_val) }; - FatAlgebraicResult::from_binary_op_result(result, *self_val, other_val) - .map(|val| ValOrChild::Val(val)) - }, - _ => unreachable!() - }; - debug_assert!(results[idx].element.is_none()); - debug_assert_eq!(results[idx].identity_mask, 0); - results[idx] = result; - } - } else { - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, idx, self_payloads, keys, request_results, results, swapped); - - let result = match &self_payloads[idx].1 { - PayloadRef::Child(self_link) => { - match other_node.get_node_at_key(keys[idx].0).into_option() { - Some(other_onward_node) => { - let result = if swapped { - other_onward_node.as_tagged().pmeet_dyn(self_link.as_tagged()).invert_identity() - } else { - self_link.as_tagged().pmeet_dyn(other_onward_node.as_tagged()) - }; - FatAlgebraicResult::from_binary_op_result(result, self_link, &other_onward_node) - .map(|child| ValOrChild::Child(child)) - }, - None => { - //Check to see if we have a dangling path, because a dangling path meet with a value should result in a path, but no value - if self_link.is_empty() && other_node.node_get_val(keys[idx].0).is_some() { - FatAlgebraicResult::new(SELF_IDENT, Some(ValOrChild::Child(TrieNodeODRc::new_empty()))) - } else { - FatAlgebraicResult::new(COUNTER_IDENT, None) - } - } - } - }, - PayloadRef::Val(_self_val) => { - //If self_payload is a val and we didn't get a corresponding val, then this result is None - FatAlgebraicResult::new(COUNTER_IDENT, None) - }, - _ => unreachable!() - }; - results[idx] = result; - } - } - pmeet_generic_recursive_reset::(&mut cur_group, &mut is_exhaustive, keys.len(), self_payloads, keys, request_results, results, swapped); - is_exhaustive -} - -/// Effectively part of `pmeet_generic_internal`, but factored out separately because it's called in -/// several different places. Resets the `cur_group` state and does a recursive call of `pmeet_generic_internal` -#[inline] -fn pmeet_generic_recursive_reset<'trie, const MAX_PAYLOAD_CNT: usize, V, A: Allocator>(cur_group: &mut Option<(usize, &'trie TrieNodeODRc)>, is_exhaustive: &mut bool, idx: usize, self_payloads: &[(&[u8], PayloadRef)], keys: &mut [(&[u8], bool)], request_results: &mut [(usize, PayloadRef<'trie, V, A>)], results: &mut [FatAlgebraicResult>], swapped: bool) - where V: Clone + Send + Sync + Lattice -{ - match core::mem::take(cur_group) { - Some((group_start, next_node)) => { - let group_keys = &mut keys[group_start..idx]; - let group_results = &mut results[group_start..idx]; - let group_self_payloads = &self_payloads[group_start..idx]; - if !pmeet_generic_internal::(group_self_payloads, group_keys, request_results, group_results, next_node.as_tagged(), swapped) { - *is_exhaustive = false; - } - }, - None => {} - } -} /// An abstracted reference to the node at the zipper's focus, returned by [`crate::zipper::ZipperInfallibleSubtries::get_focus`] /// @@ -1334,15 +1275,6 @@ mod tagged_node_ref { } } - pub(crate) fn node_get_payloads<'res>(&self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'a, V, A>)]) -> bool { - match self { - Self::DenseByteNode(node) => node.node_get_payloads(keys, results), - Self::LineListNode(node) => node.node_get_payloads(keys, results), - Self::CellByteNode(node) => node.node_get_payloads(keys, results), - Self::TinyRefNode(node) => node.node_get_payloads(keys, results), - Self::EmptyNode => true, - } - } pub fn node_contains_val(&self, key: &[u8]) -> bool { match self { @@ -1981,17 +1913,6 @@ mod tagged_node_ref { _ => unsafe{ unreachable_unchecked() } } } - pub fn node_get_payloads<'res>(&self, keys: &[(&[u8], bool)], results: &'res mut [(usize, PayloadRef<'a, V, A>)]) -> bool { - let (ptr, tag) = self.ptr.get_raw_parts(); - match tag { - EMPTY_NODE_TAG => true, - DENSE_BYTE_NODE_TAG => unsafe{ &*ptr.cast::>() }.node_get_payloads(keys, results), - LINE_LIST_NODE_TAG => unsafe{ &*ptr.cast::>() }.node_get_payloads(keys, results), - CELL_BYTE_NODE_TAG => unsafe{ &*ptr.cast::>() }.node_get_payloads(keys, results), - TINY_REF_NODE_TAG => unsafe{ &*ptr.cast::>() }.node_get_payloads(keys, results), - _ => unsafe{ unreachable_unchecked() } - } - } pub fn node_contains_val(&self, key: &[u8]) -> bool { let (ptr, tag) = self.ptr.get_raw_parts(); match tag { diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 8fc415f6..1970faf6 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -4128,6 +4128,85 @@ mod tests { } } + /// A list node meets another node slot by slot, and each slot keeps the deepest prefix of its + /// key that the other side also has -- through onward links of any node type, and whether the + /// slot holds a value or an onward node. The result must not depend on which side is the list + /// node, so every pair is checked in both orders. + #[test] + fn write_zipper_meet_into_list_keeps_shared_key_prefix() { + type Build = fn() -> PathMap; + type Locations = Vec<(Vec, Option)>; + let list_val: Build = || { let mut m = PathMap::new(); m.set_val_at(&[5u8, 6, 7], 1); m }; + let list_child: Build = || { let mut m = PathMap::new(); m.set_val_at(&[5u8, 6, 7, 8, 9], 1); m }; + let list_dangling: Build = || { let mut m = PathMap::new(); m.create_path(&[5u8, 6]); m }; + let dense_dangling: Build = || { let mut m = PathMap::new(); for b in [1u8, 2, 3] { m.set_val_at(&[b], 3); } m.create_path(&[5u8, 6]); m }; + let dense_branch: Build = || { let mut m = PathMap::new(); for b in [1u8, 2, 3] { m.set_val_at(&[b], 3); } m.set_val_at(&[5u8, 6, 0], 4); m }; + let dense_value: Build = || { let mut m = PathMap::new(); for b in [1u8, 2, 3] { m.set_val_at(&[b], 3); } m.set_val_at(&[5u8, 6], 4); m }; + assert_eq!(root_tag(&list_val()), LINE_LIST_NODE_TAG); + assert_eq!(root_tag(&dense_dangling()), DENSE_BYTE_NODE_TAG); + + let upto_6: Locations = vec![(vec![], None), (vec![5], None), (vec![5, 6], None)]; + let cases: [(&str, Build, Build, Locations); 7] = [ + ("list value, dense dangling", list_val, dense_dangling, upto_6.clone()), + ("list value, dense branch", list_val, dense_branch, upto_6.clone()), + ("list value, dense value", list_val, dense_value, upto_6.clone()), + ("list child, dense branch", list_child, dense_branch, upto_6.clone()), + ("list child, list dangling", list_child, list_dangling, upto_6.clone()), + ("list value, list dangling", list_val, list_dangling, upto_6.clone()), + ("list dangling, dense value", list_dangling, dense_value, upto_6.clone()), + ]; + let mut failures = vec![]; + for (name, a, b, expected) in cases { + for (order, dst, src) in [("a,b", a, b), ("b,a", b, a)] { + let got_map = all_locations(&dst().meet(&src())); + let mut d = dst(); + let status = d.write_zipper().meet_into(&src().read_zipper(), false); + let got = all_locations(&d); + let expected_status = if all_locations(&dst()) == expected { AlgebraicStatus::Identity } else { AlgebraicStatus::Element }; + if got_map != expected || got != expected || status != expected_status { + failures.push(format!("{name} ({order}): PathMap::meet {got_map:?}, meet_into {status:?} {got:?}; expected {expected_status:?} {expected:?}")); + } + } + } + assert!(failures.is_empty(), "\n{}", failures.join("\n")); + } + + /// A meet with an equal trie reports `Identity`, also when the destination is a dense node and + /// the source a list node, so that the list node does the walking and must say exactly when the + /// result is all of the dense node. Here one byte holds both a value and an onward node, which + /// the list node keeps in two slots under the same key. + #[test] + fn write_zipper_meet_into_dense_with_equal_list_is_identity() { + let build_list = || { + let mut m = PathMap::::new(); + m.set_val_at(&[2u8], 9); + m.set_val_at(&[2u8, 1, 0], 7); + m.set_val_at(&[2u8, 2, 0], 8); + m + }; + let build_dense = || { + let mut m = build_list(); + for b in [1u8, 3] { m.set_val_at(&[b], 1); } + for b in [1u8, 3] { m.remove_val_at(&[b], true); } + m + }; + assert_eq!(root_tag(&build_list()), LINE_LIST_NODE_TAG); + assert_eq!(root_tag(&build_dense()), DENSE_BYTE_NODE_TAG); + assert_eq!(all_locations(&build_dense()), all_locations(&build_list())); + + let mut dst = build_dense(); + assert_eq!(dst.write_zipper().meet_into(&build_list().read_zipper(), false), AlgebraicStatus::Identity); + assert_eq!(all_locations(&dst), all_locations(&build_list())); + let mut dst = build_list(); + assert_eq!(dst.write_zipper().meet_into(&build_dense().read_zipper(), false), AlgebraicStatus::Identity); + + // Anything more in the dense node is not in the result + let mut dst = build_dense(); + dst.create_path(&[2u8, 3]); + assert_eq!(dst.write_zipper().meet_into(&build_list().read_zipper(), false), AlgebraicStatus::Element); + assert_eq!(all_locations(&dst), all_locations(&build_list())); + } + /// `meet_into` never removes its focus: not when the focus value goes because the source has /// none, and not when nothing is left below it, with or without `prune`. `prune` drops only the /// dangling paths below the focus. From f3365befa789b7f282002adc9543be798f2cfc62 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 17:57:10 +0000 Subject: [PATCH 39/73] Give each shrink.py process its own temp file Every candidate input went to one fixed path, lean/.shrink.bin, and $TMPDIR was ignored, so shrinks run concurrently overwrote each other's candidates. Eight parallel shrinks of value-bias inputs gave an IndexError on a garbled trace, "input does not diverge" for an input that does, and a "minimal" input that no longer diverged. The temp file is now per process under tempfile.gettempdir(); the same eight shrink correctly in parallel, to 39-70 bytes each. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- lean/shrink.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lean/shrink.py b/lean/shrink.py index 65f23f64..2d94ae35 100755 --- a/lean/shrink.py +++ b/lean/shrink.py @@ -6,7 +6,7 @@ ./lean/shrink.py [-o out.bin] """ -import argparse, os, re, subprocess, sys +import argparse, os, re, subprocess, sys, tempfile ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # PATHMAP_ORACLE / PATHMAP_TRACE / PATHMAP_ACT_TRACE override the defaults, for @@ -21,7 +21,9 @@ ACT_DEBUG = os.path.join(ROOT, "target", "debug", "act_trace") TRACE = TRACE_RELEASE ORACLE_ARGS = [] -TMP = os.path.join(ROOT, "lean", ".shrink.bin") +# Per process, in $TMPDIR: a shared fixed path let concurrent shrinks overwrite each +# other's candidate inputs, so they shrank toward each other's bugs or none at all. +TMP = os.path.join(tempfile.gettempdir(), "pathmap-shrink-%d.bin" % os.getpid()) def signature(blob): From 98b9dedc621d9579b4f9adafc8a91a6314fd74dd Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 18:12:10 +0000 Subject: [PATCH 40/73] Keep the first k-path's value when drop_head reorders a list node's keys LineListNode::drop_head_dyn, in the case where both keys outlive the dropped bytes, shortens the keys and swaps the two slots when the shortened keys come out in the opposite order (e.g. [0,0,0] and [1,0] dropping 1 byte give [0,0] and [0]). It then called factor_prefix, which always merged slot 0 on the left. After the swap slot 0 holds the payload from the lexicographically *later* k-path, so on a u64 collision its value won, while the spec (Zip.joinKPathInto / PathMap.dropHead) folds the stripped subtries in ascending k-path order and keeps the first value. Whether the swap happens depends on how the subtrie is split across nodes, hence the layout dependence. factor_prefix now takes `slot1_first`; drop_head_dyn passes whether it reordered the slots, and merge_guts runs with slot 1 as the left operand in that case. The Identity branch also now honours COUNTER_IDENT instead of assuming SELF_IDENT. Reproducer (join_k_path_into(2) on {[3,0,0,0]:173, [0]:0, [3,1,0,0]:0, [3,1,0,1,2]:82, [0,0,2]:196, [0,0,3,1]:38}) gave [0,0]=0, spec 173; added as a regression test. Numbers (in_process, crate vs Rust reference, classified via Lean): - 8 shrunk fuzzer inputs: 0/3 crate + 0/5 ACT agree before, 3/3 + 5/5 after - value_bias_by_node_layout before -> after: crate s301 10M ml120 0->0, s302 10M ml300 1->0, s303 5M ml600 2->0, ACT s304 3M ml300 2->0, s305 3M ml600 3->0 - fresh seeds before -> after: crate s411 20M ml300 1->0, s412 20M ml600 7->0, ACT s413 10M ml300 8->0, s414 10M ml600 9->0 - sibling_after_iteration / status_imprecise unchanged (4/1, 22/3, 35/5, s411 46/12, s412 131/27) Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/line_list_node.rs | 30 ++++++++++++++++++++++++------ src/trie_node.rs | 17 +++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/line_list_node.rs b/src/line_list_node.rs index 5dc47fe5..6c519722 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -1074,7 +1074,11 @@ impl LineListNode { /// Ensures that a node is valid by combining an illegal shared prefix between the keys if there is one /// This is currently used by drop_head, because dropping a disjoint prefix may cause downstream paths /// to collide, and thus require merging - fn factor_prefix(&mut self) where V: Clone + Lattice { + /// + /// `slot1_first` makes slot 1 the left operand of the merge, so a colliding value is taken from + /// slot 1. `drop_head_dyn` needs that when shortening the keys reversed their order: slot 1 then + /// holds the payload from the lexicographically first k-path, which must win the collision. + fn factor_prefix(&mut self, slot1_first: bool) where V: Clone + Lattice { let (key0, key1) = self.get_both_keys(); let overlap = find_prefix_overlap(key0, key1); //Overlap of 1 is legal if and only if ONE OF the following two conditions are true: @@ -1086,16 +1090,27 @@ impl LineListNode { //If the overlap is illegal, split the prefix if overlap > 0 && !legal_overlap { - match merge_guts::(overlap, key0, self, key1, self) { + let merged = if slot1_first { + merge_guts::(overlap, key1, self, key0, self) + } else { + merge_guts::(overlap, key0, self, key1, self) + }; + match merged { AlgebraicResult::Element((shared_key, merged_payload)) => { let mut new_node = Self::new_in(self.alloc.clone()); unsafe{ new_node.set_payload_owned::<0>(shared_key, merged_payload) }; *self = new_node; }, AlgebraicResult::Identity(mask) => { - debug_assert!(mask & SELF_IDENT > 0); + //SELF_IDENT names the left operand's slot, COUNTER_IDENT the right one's + let left_slot_wins = mask & SELF_IDENT > 0; + debug_assert!(left_slot_wins || mask & COUNTER_IDENT > 0); let mut new_node = Self::new_in(self.alloc.clone()); - unsafe{ new_node.set_payload_owned::<0>(key0, self.clone_payload::<0>().unwrap()) }; + if left_slot_wins != slot1_first { + unsafe{ new_node.set_payload_owned::<0>(key0, self.clone_payload::<0>().unwrap()) }; + } else { + unsafe{ new_node.set_payload_owned::<0>(key1, self.clone_payload::<1>().unwrap()) }; + } *self = new_node; }, AlgebraicResult::None => {} @@ -2820,7 +2835,8 @@ impl TrieNode for LineListNode let mut new_key0_len = key0_len-byte_cnt; let mut new_key1_len = key1_len-byte_cnt; //Make sure the new keys are in the correctly sorted order - if &key0[byte_cnt..] <= &key1[byte_cnt..] { + let reordered = &key0[byte_cnt..] > &key1[byte_cnt..]; + if !reordered { unsafe { //Shorten key0 let base_ptr = temp_node.key_bytes.as_mut_ptr().cast::(); @@ -2854,7 +2870,9 @@ impl TrieNode for LineListNode core::mem::swap(&mut temp_node.val_or_child0, &mut temp_node.val_or_child1); } temp_node.header = Self::header0(slot0_child, new_key0_len) | Self::header1(slot1_child, new_key1_len); - temp_node.factor_prefix(); + //If the keys were reordered, slot 1 now holds the original slot 0: the lexicographically + // first k-path, whose value must win any collision the prefix merge finds + temp_node.factor_prefix(reordered); debug_assert!(validate_node(&temp_node)); return Some(TrieNodeODRc::new_in(temp_node, self.alloc.clone())) } diff --git a/src/trie_node.rs b/src/trie_node.rs index dd5f2f95..5bdce103 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -3519,6 +3519,23 @@ mod tests { assert_eq!(vals(&m), vec![(vec![7], 0)]); } + /// `LineListNode::drop_head_dyn` with both keys longer than `byte_cnt`: the shortened keys come + /// out in the opposite order ([0,0] from slot 0 and [0] from slot 1), so the slots are swapped + /// before `factor_prefix` merges them. The merge joined the swapped slot 0 (the original + /// slot 1, the later k-path) on the left, so its value won the collision at [0,0]. + #[test] + fn join_k_path_into_keeps_first_value_when_shortened_keys_reorder() { + let mut m = PathMap::::new(); + m.set_val_at(&[3u8, 0, 0, 0], 173); + m.set_val_at(&[0u8], 0); + m.set_val_at(&[3u8, 1, 0, 0], 0); + m.set_val_at(&[3u8, 1, 0, 1, 2], 82); + m.set_val_at(&[0u8, 0, 2], 196); + m.set_val_at(&[0u8, 0, 3, 1], 38); + m.write_zipper().join_k_path_into(2, false); + assert_eq!(vals(&m), vec![(vec![0, 0], 173), (vec![0, 1, 2], 82), (vec![2], 196), (vec![3, 1], 38)]); + } + #[test] fn slim_ptrs_test1() { let map = PathMap::<()>::new(); From a6940e8e989ee92a1697d4f83f3988622c16c842 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 18:43:19 +0000 Subject: [PATCH 41/73] Find the onward child in either slot when stepping to a previous list-node sibling A location holding both a value and a child is stored in a LineListNode as two slots with the same key. `get_sibling_of_child`, stepping backwards, took the onward child only from the slot whose byte it matched first -- the value slot, in that layout -- so `to_prev_sibling_byte` landed on the location without its node: `child_count` reported 0 and nothing below could be reached. From {[2]:0, [2,0]:0, [2,1]:0}, a read zipper at the off-trie [3] stepped to [2] with child_count 0 and `descend_first_byte` returning None. The branch is from e659a96 ("Fixing prev_sibling bugs"). It now looks for the child in both slots holding the sibling's key. This was the `sibling_after_iteration` class: all 117 inputs of it from two fresh fuzz runs (30M inputs) first diverged at to_prev_sibling_byte, and all 117 agree now. Fresh seeds after the fix: crate 711/300 20M, 712/600 10M, 713/120 10M and ACT 714/600 5M show no sibling divergence and no panic; the only remaining class is status_imprecise (3 and 15). Regression test `read_zipper_prev_sibling_onto_a_value_and_child_location` fails without the fix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/line_list_node.rs | 32 ++++++++++++++++++-------------- src/zipper.rs | 24 ++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/line_list_node.rs b/src/line_list_node.rs index 6c519722..bd089ebb 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -2511,21 +2511,25 @@ impl TrieNode for LineListNode && *byte < key[last_key_byte_idx] }) }; - let (sibling_byte, slot) = match key_byte(key1) { - Some(byte) => (byte, 1), - None => match key_byte(key0) { - Some(byte) => (byte, 0), - None => return (None, None), - }, + let sibling_byte = match key_byte(key1).or_else(|| key_byte(key0)) { + Some(byte) => byte, + None => return (None, None), }; - let sib_node = match slot { - 0 if key0.len() == key.len() && self.is_child_ptr::<0>() => { - Some(unsafe { self.child_in_slot::<0>().as_tagged() }) - }, - 1 if key1.len() == key.len() && self.is_child_ptr::<1>() => { - Some(unsafe { self.child_in_slot::<1>().as_tagged() }) - }, - _ => None, + //The sibling's onward child may sit in either slot: a location holding both a value + // and a child is stored as two slots with the same key, so the slot whose byte was + // found first may be the value one. Look for the child in both. + let holds_child = |candidate: &[u8], slot: usize| { + candidate.len() == key.len() + && candidate[last_key_byte_idx] == sibling_byte + && candidate[..last_key_byte_idx] == common_key[..] + && if slot == 0 { self.is_child_ptr::<0>() } else { self.is_child_ptr::<1>() } + }; + let sib_node = if holds_child(key1, 1) { + Some(unsafe { self.child_in_slot::<1>().as_tagged() }) + } else if holds_child(key0, 0) { + Some(unsafe { self.child_in_slot::<0>().as_tagged() }) + } else { + None }; (Some(sibling_byte), sib_node) } diff --git a/src/zipper.rs b/src/zipper.rs index adee0fad..66c66db1 100644 --- a/src/zipper.rs +++ b/src/zipper.rs @@ -6683,6 +6683,30 @@ mod tests { assert_eq!(z.path(), &[3]); } + /// A location holding both a value and a child is stored in a list node as two slots with the same + /// key. Stepping to it as the *previous* sibling looked for the onward child only in the slot + /// whose byte it found first, which can be the value slot, so the zipper landed on the location + /// without its node: `child_count` said 0 and nothing below could be reached. + #[test] + fn read_zipper_prev_sibling_onto_a_value_and_child_location() { + let mut m = PathMap::::new(); + m.set_val_at(&[2u8, 0], 0); + m.set_val_at(&[2u8, 1], 0); + m.set_val_at(&[2u8], 0); + for start in [&[3u8][..], &[9u8]] { + let mut z = m.read_zipper(); + z.descend_to(start); + assert!(!z.path_exists()); + assert_eq!(z.to_prev_sibling_byte(), Some(2), "from {start:?}"); + assert_eq!(z.path(), &[2u8]); + assert_eq!(z.val(), Some(&0)); + assert_eq!(z.child_count(), 2, "from {start:?}"); + assert_eq!(z.child_mask().iter().collect::>(), vec![0u8, 1]); + assert_eq!(z.descend_first_byte(), Some(0), "from {start:?}"); + assert_eq!(z.path(), &[2u8, 0]); + } + } + #[test] fn read_zipper_prev_sibling_cases() { let m: PathMap<()> = [&[10u8][..], &[20], &[70]].into_iter().collect(); // dense node From e28448e5ac744c874b8fc2503bf4595a0d3ce14e Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 18:47:32 +0000 Subject: [PATCH 42/73] Make the repro generator skip and call exactly what the harness does Four ops in the Rust program emitted by `pathmap_trace --repro` did not match what the harness runs: - prune_path / prune_ascend (30, 31): the harness skips them unless the write zipper is at the map root; the repro always called them. - restricting (41): the harness skips it when either focus has no children; the repro always called it. - meet_2 (55): the harness takes the second source as a clone of the read zipper moved down `p`; the repro opened a fresh zipper on map1 instead, which has no history and is at a different place. - to_next_get_val (52): the repro called `to_next_val`. With these changes, compiled repros for 1335 random inputs (seed 55, all ops) end with the same MAP0/MAP1 as the trace. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/repro.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/differential/src/repro.rs b/differential/src/repro.rs index 840eeb68..6c1373ec 100644 --- a/differential/src/repro.rs +++ b/differential/src/repro.rs @@ -172,8 +172,8 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 27 => { let v = g!(d.u8()) as u64; format!("wz.set_val({v});") } 28 => { let _pr = g!(d.boolean()); "wz.remove_val(false);".to_string() } 29 => "wz.create_path();".to_string(), - 30 => "wz.prune_path();".to_string(), - 31 => "wz.prune_ascend();".to_string(), + 30 => if r0.is_empty() { "wz.prune_path();".to_string() } else { "// prune_path off the map root: skipped by the harness".to_string() }, + 31 => if r0.is_empty() { "wz.prune_ascend();".to_string() } else { "// prune_ascend off the map root: skipped by the harness".to_string() }, 32 => { let _pr = g!(d.boolean()); "wz.remove_branches(false);".to_string() } 33 => { let n = g!(d.modn(4)); let m = g!(d.path_n(n)); let _pr = g!(d.boolean()); format!("wz.remove_unmasked_branches(ByteMask::from_iter({}.iter().copied()), false);", rs_mask(&m)) } @@ -185,7 +185,7 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 38 => { let _pr = g!(d.boolean()); "wz.meet_into(&rz, false);".to_string() } 39 => { let _pr = g!(d.boolean()); "wz.subtract_into(&rz, false);".to_string() } 40 => "wz.restrict(&rz);".to_string(), - 41 => "wz.restricting(&rz);".to_string(), + 41 => "if wz.child_count() != 0 && rz.child_count() != 0 { wz.restricting(&rz); }".to_string(), 42 => { let k = g!(d.modn(4)); let _pr = g!(d.boolean()); if k == 0 { "// join_k_path_into(0): skipped by the harness".to_string() } else { format!("wz.join_k_path_into({k}, false);") } } @@ -206,7 +206,7 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { 50 => { let v = g!(d.u8()) as u64; format!("wz.get_val_or_set_mut_with(|| {v});") } 51 => { let t = g!(d.modn(2)); let p = g!(d.path(6)); format!("{{ {z}.val(); {z}.val_at({p}); }}", z = z!(t), p = rs_bytes(&p)) } - 52 => "rz.to_next_val(); // to_next_get_val".to_string(), + 52 => "rz.to_next_get_val();".to_string(), 53 => { let n = g!(d.modn(4)); let m = g!(d.path_n(n)); let ru = g!(d.boolean()); format!("wz.graft_masked_branches(&rz, ByteMask::from_iter({}.iter().copied()), {ru});", rs_mask(&m)) } // As `do_graft_child_maps`: fed the source's own child subtries under the mask. @@ -215,8 +215,7 @@ pub fn emit_repro(bytes: &[u8], upto: usize) -> String { let maps: Vec> = m.iter().map(|b| {{ let mut c = rz.clone(); c.descend_to_byte(b); c.make_map() }}).collect(); \ wz.graft_child_maps(m, maps, {ru}); }}", rs_mask(&m)) } 55 => { let p = g!(d.path(6)); - format!("{{ let mut b = map1.read_zipper_at_path({}); b.descend_to({}); wz.meet_2(&rz, &b); }}", - rs_bytes(&r1), rs_bytes(&p)) } + format!("{{ let mut b = rz.clone(); b.descend_to({}); wz.meet_2(&rz, &b); }}", rs_bytes(&p)) } _ => "// nop".to_string(), }; o.push_str(&format!(" /* {step:3} */ {line}\n")); From 9f382a6bd7f837302a3553d863622ba295da57f7 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 19:09:37 +0000 Subject: [PATCH 43/73] Report Identity when subtract_into drops only an empty link beside a value A byte node location can hold a value together with an empty onward link. The link carries nothing, because the value already holds the location. `CoFree::psubtract` leaves such a link out of the subtraction, but `combine_algebraic_results` then saw a link that was there go away and turned the value's `Identity` into `Element`. So `subtract_into` reported a change for a destination it had not changed. A value of the destination surviving the subtraction now keeps the result `Identity`. This was the `status_imprecise` class, the last divergence class between the crate and the model. All 38 saved inputs from 35M fuzz inputs (seeds 801-803) were `subtract_into` returning Element for Identity, and all 38 agree now. Fresh seeds after the fix: 901/300 20M, 902/600 10M, 903/1000 5M, ACT 904/600 5M, and 905/300 10M with debug assertions: 0 divergences, 0 panics. The regression test builds the layout with join_into, insert_prefix, meet_into and join_into, checks that the empty link is there, and fails without the fix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 10 +++++++++ src/write_zipper.rs | 46 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 798fa530..fb5b5ed2 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -1993,6 +1993,16 @@ impl 0 { + return AlgebraicResult::Identity(SELF_IDENT) + } + } + } self.combine_algebraic_results(other, rec, val) } } diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 1970faf6..51e2d55c 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -4375,6 +4375,52 @@ mod tests { assert_eq!(vals(&dst), vec![(vec![0], 1), (vec![1], 3), (vec![2], 4)]); } + /// `subtract_into` into a byte node whose location holds a value *and* an empty onward link. + /// The link carries nothing, since the value already holds the location, so dropping it while + /// the value survives leaves the trie as it was and the status has to be `Identity`. It used to + /// be `Element`. + #[test] + fn write_zipper_subtract_into_value_beside_an_empty_link_is_identity() { + let mut src = PathMap::::new(); + for (p, v) in [(&[0u8, 0][..], 0), (&[0, 1], 0), (&[0, 3], 0), (&[1], 1)] { src.set_val_at(p, v); } + + //These steps leave `dst` with the content of `src`, and an empty link beside the value at [1] + let mk_dst = || { + let mut dst = PathMap::::new(); + let rz = src.read_zipper(); + let mut wz = dst.write_zipper(); + wz.join_into(&rz); + wz.insert_prefix(&[1u8]); + wz.meet_into(&rz, false); + wz.join_into(&rz); + drop(wz); + let link = dst.root().unwrap().as_tagged().node_get_child(&[1]).map(|(_, child)| child.as_tagged().node_is_empty()); + assert_eq!(link, Some(true), "the layout this test needs"); + dst + }; + let locations = |m: &PathMap| { + let mut z = m.read_zipper(); + let mut locs = vec![(z.path().to_vec(), z.val().copied())]; + while z.to_next_step() { locs.push((z.path().to_vec(), z.val().copied())); } + locs + }; + + //Source {[0]:0, [1]:0, [3]:0}: the value at [1] differs, so nothing annihilates + let mut dst = mk_dst(); + let before = locations(&dst); + let st = dst.write_zipper().subtract_into(&src.read_zipper_at_path(&[0u8]), false); + assert_eq!(st, AlgebraicStatus::Identity); + assert_eq!(locations(&dst), before); + + //An equal value annihilates, and the location goes with it + let mut dst = mk_dst(); + let mut sub = PathMap::::new(); + sub.set_val_at(&[1u8], 1); + let st = dst.write_zipper().subtract_into(&sub.read_zipper(), false); + assert_eq!(st, AlgebraicStatus::Element); + assert_eq!(locations(&dst), vec![(vec![], None), (vec![0], None), (vec![0, 0], Some(0)), (vec![0, 1], Some(0)), (vec![0, 3], Some(0))]); + } + /// Tests how `subtract_into` handles dangling paths, including situations with extraneous empty nodes hanging around #[test] fn write_zipper_subtract_into_test2() { From 342426bffcc1b05f228a80148e684b7a69a456f9 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 21:07:48 +0000 Subject: [PATCH 44/73] Add a crash-only fuzz mode for the API the differential harness cannot reach The differential harness only runs what the Lean model specifies, so much of the crate went unfuzzed: paths outside the 4-letter alphabet and 5-byte limit, value types other than u64, prune = true, most PathMap methods, every zipper kind but the plain read and write zippers, catamorphisms and anamorphisms, and .paths serialization. `differential/src/crash.rs` is a second op table over that surface with no model behind it: an input fails only by panicking (debug assertions included), crashing on a signal, or hanging. Calls with a documented panic are made to meet their precondition (KNOWN_PRECONDITIONS), and known failures -- val_count stubs, TrieRef forks at missing paths, k == 0, to_next_k_path without descend_first_k_path, meet_k_path_into on an empty focus -- are steered around unless --include-known (KNOWN_FAILURES). Front ends: - `crash_fuzz`: random or file inputs over -j threads. A watchdog reports hangs and abandons the stuck thread, a signal handler attributes segfaults and aborts to their input, and --keep-going supervises worker processes and groups failures by site. CRASH_TRACE=1 and CRASH_BACKTRACE=1 are for replaying one input. - `afl_crash`: the same table under AFL. - `crash_shrink.py`: shrinks an input keeping its failure site. Input generation moves from `in_process` to `differential::source` so both in-process front ends share it. A 50k-input survey with debug assertions and overflow checks found 27 failure sites, including undefined behaviour reachable from safe code. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/Cargo.toml | 7 + differential/crash_shrink.py | 89 +++ differential/src/bin/afl_crash.rs | 26 + differential/src/bin/crash_fuzz.rs | 457 ++++++++++++ differential/src/bin/in_process.rs | 56 +- differential/src/crash.rs | 1081 ++++++++++++++++++++++++++++ differential/src/lib.rs | 4 + differential/src/source.rs | 56 ++ 8 files changed, 1721 insertions(+), 55 deletions(-) create mode 100755 differential/crash_shrink.py create mode 100644 differential/src/bin/afl_crash.rs create mode 100644 differential/src/bin/crash_fuzz.rs create mode 100644 differential/src/crash.rs create mode 100644 differential/src/source.rs diff --git a/differential/Cargo.toml b/differential/Cargo.toml index cfbb3b4c..23fefb0e 100644 --- a/differential/Cargo.toml +++ b/differential/Cargo.toml @@ -7,6 +7,8 @@ description = "The Rust side of the differential fuzzing harness for pathmap's z [dependencies] pathmap = { path = "..", features = ["arena_compact"] } +# `crash_fuzz` attributes segfaults and aborts to an input from a signal handler. +libc = "0.2" # Only for `bin/afl_differential.rs`, behind the `afl` feature, so a plain # `cargo build -p differential` neither needs it nor builds it. See that file. afl = { version = "0.18", optional = true } @@ -23,3 +25,8 @@ afl = ["dep:afl"] name = "afl_differential" path = "src/bin/afl_differential.rs" required-features = ["afl"] + +[[bin]] +name = "afl_crash" +path = "src/bin/afl_crash.rs" +required-features = ["afl"] diff --git a/differential/crash_shrink.py b/differential/crash_shrink.py new file mode 100755 index 00000000..f4e7b7d8 --- /dev/null +++ b/differential/crash_shrink.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Shrink an input that makes `crash_fuzz` fail, keeping the failure the same. + + differential/crash_shrink.py [-o out.bin] [--timeout SECS] [--bin PATH] + +The failure is identified by its kind and, for a panic, its site (file:line and +the message with numbers blanked), as `crash_fuzz --keep-going` groups them. A +hang is identified by the kind alone and tested with a short timeout, so shrink +hangs with a timeout comfortably above how long the input takes to get stuck. + +Greedy: delete chunks of halving size from the end towards the start, then try +lowering each byte, keeping any change that preserves the failure. +""" +import argparse, os, re, subprocess, sys, tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def signature(binary, blob, timeout, tmp): + with open(tmp, "wb") as f: + f.write(blob) + try: + p = subprocess.run([binary, tmp, "-j", "1", "--timeout", str(timeout)], + capture_output=True, timeout=timeout + 20) + except subprocess.TimeoutExpired: + return "hang" + err = p.stderr.decode(errors="replace") + m = re.search(r"CRASH .*?kind=(\w+).*?(?:msg=(.*))?$", err, re.M) + if not m: + return None if p.returncode in (0,) else "exit %d" % p.returncode + kind, msg = m.group(1), m.group(2) or "" + if kind == "panic": + site = re.sub(r"\b\d+\b", "N", msg.split(" | ")[0]) + site = re.sub(r"(src/[\w/]+\.rs):N:N", lambda s: s.group(0), site) + loc = re.search(r"panicked at (\S+?):(\d+):(\d+)", msg) + return "panic %s:%s %s" % (loc.group(1), loc.group(2), site.split(": ", 1)[-1]) if loc else "panic " + site + if kind == "signal": + return "signal " + msg.split(" ")[1] if msg else "signal" + return kind + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("input") + ap.add_argument("-o", "--out") + ap.add_argument("--timeout", type=int, default=3) + ap.add_argument("--bin", default=os.path.join(ROOT, "target", "release", "crash_fuzz")) + a = ap.parse_args() + blob = open(a.input, "rb").read() + tmp = os.path.join(tempfile.gettempdir(), "crash-shrink-%d.bin" % os.getpid()) + want = signature(a.bin, blob, a.timeout, tmp) + if want is None: + sys.exit("input does not fail") + print("signature:", want, file=sys.stderr) + + def ok(b): + return signature(a.bin, b, a.timeout, tmp) == want + + chunk = max(1, len(blob) // 2) + while chunk >= 1: + i = len(blob) - chunk + changed = False + while i >= 0: + cand = blob[:i] + blob[i + chunk:] + if ok(cand): + blob = cand + changed = True + i -= chunk + print("chunk %d -> %d bytes" % (chunk, len(blob)), file=sys.stderr) + if not changed: + chunk //= 2 + for i in range(len(blob)): + for v in (0, 1, blob[i] // 2): + if v < blob[i]: + cand = blob[:i] + bytes([v]) + blob[i + 1:] + if ok(cand): + blob = cand + break + out = a.out or re.sub(r"(\.bin)?$", ".min.bin", a.input, count=1) + open(out, "wb").write(blob) + try: + os.unlink(tmp) + except OSError: + pass + print("%s: %d bytes, %s" % (out, len(blob), want)) + + +if __name__ == "__main__": + main() diff --git a/differential/src/bin/afl_crash.rs b/differential/src/bin/afl_crash.rs new file mode 100644 index 00000000..7699fa1a --- /dev/null +++ b/differential/src/bin/afl_crash.rs @@ -0,0 +1,26 @@ +//! Coverage-guided front end for the crash-only table in [`differential::crash`]. +//! +//! ```text +//! cargo afl build --release -p differential --features afl --bin afl_crash +//! mkdir -p out/afl-crash-in && target/release/crash_fuzz --random 64 --dump 0 > out/afl-crash-in/0 +//! cargo afl fuzz -i out/afl-crash-in -o out/afl-crash-out -t 5000 target/release/afl_crash +//! ``` +//! +//! `cargo afl build` turns on debug assertions and overflow checks, so replay +//! findings with a `crash_fuzz` built the same way (see its docs); a plain +//! release `crash_fuzz` runs most of them clean. +//! +//! AFL runs each input in a forked child, so a panic, an abort and a hang each +//! cost one child and are saved (`crashes/`, `hangs/`). Replay them with +//! `crash_fuzz `, which names the panic site, and group them with +//! `crash_fuzz --keep-going `. As with `afl_differential`, check +//! `hangs/` as well as `crashes/`: with a piped `core_pattern`, AFL can file +//! a crash as a hang. + +use differential::crash::run; + +fn main() { + afl::fuzz!(|data: &[u8]| { + run(data); + }); +} diff --git a/differential/src/bin/crash_fuzz.rs b/differential/src/bin/crash_fuzz.rs new file mode 100644 index 00000000..ebfa2d7a --- /dev/null +++ b/differential/src/bin/crash_fuzz.rs @@ -0,0 +1,457 @@ +//! Crash-only fuzzer over [`differential::crash`]: no model, no trace. An input +//! fails by panicking (including a failed `debug_assert!`), aborting, or +//! running longer than `--timeout` seconds. +//! +//! crash_fuzz --random 1000000 -j 56 +//! crash_fuzz --random 1000000 -j 56 --keep-going --save runs/crashes +//! crash_fuzz runs/crashes/*.bin +//! +//! Debug assertions are the point of half of this, so build it with them, and +//! overflow checks, on. That is also what `cargo afl build` does, so it is the +//! build to replay `afl_crash` findings with: +//! +//! CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS=true CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS=true \ +//! CARGO_TARGET_DIR=target/dbgassert cargo build --release -p differential --bin crash_fuzz +//! +//! `CRASH_TRACE=1` prints each operation as it starts and `CRASH_BACKTRACE=1` +//! a backtrace on panic; both are for replaying one saved input. +//! +//! `--include-known` also makes the calls in `differential::crash::KNOWN_FAILURES`, +//! which are otherwise steered around. +//! +//! # Failure handling +//! +//! As in `in_process`, a panic is never caught: `pathmap` is not unwind-safe. +//! The panic hook names the input, saves it, and exits. An abort or a +//! segfault gets no hook at all. +//! +//! A segfault or an abort cannot run a panic hook, but a signal handler can +//! still say which input was in flight on the faulting thread (exit 103); the +//! supervisor saves that input. +//! +//! A hang cannot be interrupted either, but it does not have to end the run: +//! the watchdog reports the input, the stuck thread is abandoned to spin, and +//! the others carry on. The process gives up (exit 102) once half its threads +//! are stuck. +//! +//! Without `--keep-going` the first panic ends the run. With it, this process +//! becomes a supervisor: it runs the fuzzer as a child, collects every failure +//! the child reports, and starts a new child past them, skipping inputs already +//! reported. A death nothing attributed is located by re-running from the +//! child's last low-water mark one input at a time. At the end the +//! failures are grouped by where they panicked. + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Write}; +use std::process::{Command, Stdio}; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use differential::crash::{run, set_include_known}; +use differential::source::Source; + +static IN_FLIGHT: [AtomicUsize; 256] = [const { AtomicUsize::new(usize::MAX) }; 256]; +/// Milliseconds since `EPOCH` at which each slot started its current input. +static STARTED: [AtomicU64; 256] = [const { AtomicU64::new(0) }; 256]; +static EPOCH: OnceLock = OnceLock::new(); +static SOURCE: OnceLock = OnceLock::new(); +static SAVE: OnceLock> = OnceLock::new(); + +thread_local! { + static SLOT: std::cell::Cell = const { std::cell::Cell::new(usize::MAX) }; +} + +fn now_ms() -> u64 { + EPOCH.get().unwrap().elapsed().as_millis() as u64 +} + +/// Report one failure on stderr in the line format the supervisor parses, and +/// save the input. +fn report(idx: usize, kind: &str, msg: &str) { + let one_line = msg.replace('\n', " | "); + let mut saved = String::new(); + if let (Some(Some(dir)), Some(src)) = (SAVE.get(), SOURCE.get()) { + let _ = std::fs::create_dir_all(dir); + let path = std::path::Path::new(dir).join(format!("{idx:08}.bin")); + if std::fs::write(&path, src.get(idx)).is_ok() { + saved = format!(" saved={}", path.display()); + } + } + let name = SOURCE.get().map(|s| s.name(idx)).unwrap_or_default(); + eprintln!("CRASH idx={idx} kind={kind} input={name}{saved} msg={one_line}"); + let _ = std::io::stderr().flush(); +} + +struct Args { + all: Vec, +} + +impl Args { + fn flag(&self, name: &str, default: usize) -> usize { + self.str_flag(name).and_then(|v| v.parse().ok()).unwrap_or(default) + } + fn str_flag(&self, name: &str) -> Option { + self.all.iter().position(|a| a == name).and_then(|i| self.all.get(i + 1)).cloned() + } + fn has(&self, name: &str) -> bool { + self.all.iter().any(|a| a == name) + } +} + +const VALUE_FLAGS: &[&str] = &["--max-failures", "--random", "--from", "--seed", "--maxlen", "-j", "--save", "--dump", "--timeout", "--skip", "--to"]; + +fn main() { + EPOCH.get_or_init(Instant::now); + let args = Args { all: std::env::args().skip(1).collect() }; + let files: Vec = { + let mut v = Vec::new(); + let mut it = args.all.iter(); + while let Some(a) = it.next() { + if a.starts_with('-') { + if VALUE_FLAGS.contains(&a.as_str()) { it.next(); } + } else { + v.push(a.clone()); + } + } + v + }; + let count = args.flag("--random", 0); + if count == 0 && files.is_empty() { + eprintln!("usage: crash_fuzz [--random N] [--seed S] [--maxlen L] [--from I] [--to I] [-j N] \ + [--timeout SECS] [--save DIR] [--keep-going [--max-failures N]] [--include-known] [--dump IDX] [FILES...]"); + std::process::exit(2); + } + let source = if !files.is_empty() { + Source::Files(files) + } else { + Source::Random { seed: args.flag("--seed", 1) as u64, count, maxlen: args.flag("--maxlen", 600) } + }; + + if let Some(i) = args.str_flag("--dump") { + let idx: usize = i.parse().expect("--dump IDX"); + std::io::stdout().write_all(&source.get(idx)).unwrap(); + return; + } + + if args.has("--keep-going") { + supervise(&args, source); + return; + } + worker(&args, source); +} + +// --------------------------------------------------------------------------- +// Worker: runs inputs until the first failure +// --------------------------------------------------------------------------- + +/// Say which input the faulting thread was running, with only async-signal-safe +/// calls, and exit. Runs on the alternate signal stack std sets up for every +/// thread, so a stack overflow reaches it too. +extern "C" fn on_fatal_signal(sig: libc::c_int) { + // One report per process: a second faulting thread waits to be killed by the exit. + static REPORTING: AtomicBool = AtomicBool::new(false); + if REPORTING.swap(true, Ordering::SeqCst) { + loop { unsafe { libc::pause(); } } + } + let slot = SLOT.with(|s| s.get()); + let idx = if slot < 256 { IN_FLIGHT[slot].load(Ordering::Relaxed) } else { usize::MAX }; + let mut buf = [0u8; 128]; + let mut n = 0; + let mut put = |bytes: &[u8]| { + for &b in bytes { + if n < buf.len() { buf[n] = b; n += 1; } + } + }; + fn digits(mut v: u64, out: &mut [u8; 20]) -> usize { + let mut i = out.len(); + loop { + i -= 1; + out[i] = b'0' + (v % 10) as u8; + v /= 10; + if v == 0 { return i } + } + } + let mut d = [0u8; 20]; + put(b"\nCRASH idx="); + if idx == usize::MAX { put(b"none") } else { let i = digits(idx as u64, &mut d); put(&d[i..]); } + put(b" kind=signal msg=signal "); + let i = digits(sig as u64, &mut d); + put(&d[i..]); + put(match sig { libc::SIGSEGV => b" (SIGSEGV)\n" as &[u8], libc::SIGBUS => b" (SIGBUS)\n", libc::SIGABRT => b" (SIGABRT)\n", _ => b"\n" }); + unsafe { + libc::write(2, buf.as_ptr() as *const libc::c_void, n); + libc::_exit(103); + } +} + +fn install_signal_handlers() { + for sig in [libc::SIGSEGV, libc::SIGBUS, libc::SIGABRT, libc::SIGILL] { + unsafe { + let mut sa: libc::sigaction = core::mem::zeroed(); + sa.sa_sigaction = on_fatal_signal as extern "C" fn(libc::c_int) as usize; + sa.sa_flags = libc::SA_ONSTACK; + libc::sigemptyset(&mut sa.sa_mask); + libc::sigaction(sig, &sa, core::ptr::null_mut()); + } + } +} + +fn worker(args: &Args, source: Source) { + install_signal_handlers(); + set_include_known(args.has("--include-known")); + let from = args.flag("--from", 0); + let to = args.flag("--to", usize::MAX).min(source.len()); + let jobs = args.flag("-j", 1).clamp(1, 256); + let timeout_ms = args.flag("--timeout", 10) as u64 * 1000; + let trace_starts = args.has("--trace-starts"); + let skip: std::collections::HashSet = args + .str_flag("--skip") + .map(|s| s.split(',').filter_map(|x| x.parse().ok()).collect()) + .unwrap_or_default(); + SAVE.get_or_init(|| args.str_flag("--save")); + SOURCE.get_or_init(|| source); + let src = SOURCE.get().unwrap(); + + std::panic::set_hook(Box::new(|info| { + // One report per process: a second panicking thread waits for the exit. + static REPORTING: AtomicBool = AtomicBool::new(false); + if REPORTING.swap(true, Ordering::SeqCst) { + loop { std::thread::park(); } + } + let slot = SLOT.with(|s| s.get()); + let idx = if slot < 256 { IN_FLIGHT[slot].load(Ordering::Relaxed) } else { usize::MAX }; + let loc = info.location().map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column())).unwrap_or_default(); + let payload = info + .payload() + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| info.payload().downcast_ref::().cloned()) + .unwrap_or_default(); + if std::env::var_os("CRASH_BACKTRACE").is_some() { + eprintln!("{}", std::backtrace::Backtrace::force_capture()); + } + report(idx, "panic", &format!("panicked at {loc}: {payload}")); + std::process::exit(101); + })); + + // Threads are detached, not scoped: a thread stuck in a hang never returns, + // and a scope would wait for it forever. + static NEXT: AtomicUsize = AtomicUsize::new(0); + static DONE: AtomicUsize = AtomicUsize::new(0); + static RUNNING: AtomicUsize = AtomicUsize::new(0); + static HUNG: [AtomicBool; 256] = [const { AtomicBool::new(false) }; 256]; + NEXT.store(from, Ordering::Relaxed); + RUNNING.store(jobs, Ordering::Relaxed); + let skip: &'static std::collections::HashSet = Box::leak(Box::new(skip)); + let start = Instant::now(); + for slot in 0..jobs { + std::thread::spawn(move || { + SLOT.with(|s| s.set(slot)); + loop { + let idx = NEXT.fetch_add(1, Ordering::Relaxed); + if idx >= to { + IN_FLIGHT[slot].store(usize::MAX, Ordering::Relaxed); + RUNNING.fetch_sub(1, Ordering::Relaxed); + return; + } + if skip.contains(&idx) { continue } + STARTED[slot].store(now_ms(), Ordering::Relaxed); + IN_FLIGHT[slot].store(idx, Ordering::Relaxed); + if trace_starts { + eprintln!("START {idx}"); + } + run(&src.get(idx)); + DONE.fetch_add(1, Ordering::Relaxed); + } + }); + } + + // Watchdog and progress: hangs are reported and their threads written off, + // and the low-water mark lets a supervisor locate an abort. + let mut hung = 0usize; + let mut last_report = Instant::now(); + loop { + std::thread::sleep(Duration::from_millis(100)); + let now = now_ms(); + let mut low = NEXT.load(Ordering::Relaxed); + for slot in 0..jobs { + if HUNG[slot].load(Ordering::Relaxed) { continue } + let idx = IN_FLIGHT[slot].load(Ordering::Relaxed); + if idx == usize::MAX { continue } + low = low.min(idx); + if now.saturating_sub(STARTED[slot].load(Ordering::Relaxed)) > timeout_ms { + HUNG[slot].store(true, Ordering::Relaxed); + hung += 1; + report(idx, "hang", &format!("still running after {}s", timeout_ms / 1000)); + } + } + if last_report.elapsed() > Duration::from_secs(1) { + eprintln!("LOW {low}"); + last_report = Instant::now(); + } + if hung * 2 >= jobs { + eprintln!("{hung} of {jobs} threads hung; giving up at {low}"); + eprintln!("LOW {low}"); + std::process::exit(102); + } + if RUNNING.load(Ordering::Relaxed) <= hung { + break; + } + } + let n = DONE.load(Ordering::Relaxed); + let secs = start.elapsed().as_secs_f64(); + println!("{n} inputs ran clean in {secs:.2}s -> {:.0} inputs/s ({hung} hung)", n as f64 / secs); + // Not a return: hung threads would keep the process alive. + std::process::exit(if hung > 0 { 3 } else { 0 }); +} + +// --------------------------------------------------------------------------- +// Supervisor: restarts workers past failures and collects them +// --------------------------------------------------------------------------- + +struct Crash { + idx: usize, + kind: String, + msg: String, +} + +/// Group key: the panic site, or the kind for hangs and aborts. +fn site(c: &Crash) -> String { + match c.msg.strip_prefix("panicked at ") { + Some(rest) => { + let (loc, what) = rest.split_once(": ").unwrap_or((rest, "")); + // Assertion messages carry values; keep the text before the first value. + let what: String = what.split(" | ").next().unwrap_or("").chars().take(90).collect(); + // Index and length values differ per input; the site does not. + let what: String = what.split(' ').map(|w| if w.chars().all(|c| c.is_ascii_digit()) && !w.is_empty() { "N" } else { w }).collect::>().join(" "); + format!("{loc}: {what}") + } + None if c.kind == "signal" || c.kind == "abort" => format!("{}: {}", c.kind, c.msg.chars().take(60).collect::()), + None => c.kind.clone(), + } +} + +struct ChildResult { + crashes: Vec, + low: usize, + /// The child ran its whole range (exit 0, or 3 when some inputs hung). + completed: bool, + status: String, + last_start: Option, +} + +/// Run one child over `[from, to)` and collect what it reports. +fn run_child(args: &Args, from: usize, to: usize, jobs: usize, skip: &[usize], trace_starts: bool) -> ChildResult { + let exe = std::env::current_exe().unwrap(); + let mut cmd = Command::new(exe); + let mut it = args.all.iter(); + while let Some(a) = it.next() { + match a.as_str() { + "--keep-going" => {} + "--from" | "--to" | "-j" | "--skip" | "--max-failures" => { it.next(); } + _ => { cmd.arg(a); } + } + } + cmd.args(["--from", &from.to_string(), "--to", &to.to_string(), "-j", &jobs.to_string()]); + if !skip.is_empty() { + cmd.args(["--skip", &skip.iter().map(|i| i.to_string()).collect::>().join(",")]); + } + if trace_starts { + cmd.arg("--trace-starts"); + } + cmd.stderr(Stdio::piped()).stdout(Stdio::null()); + let mut child = cmd.spawn().expect("cannot start worker"); + let stderr = child.stderr.take().unwrap(); + let mut r = ChildResult { crashes: Vec::new(), low: from, completed: false, status: String::new(), last_start: None }; + let mut abort_msg = String::new(); + for line in BufReader::new(stderr).lines().map_while(Result::ok) { + if let Some(rest) = line.strip_prefix("CRASH ") { + let field = |k: &str| rest.split(' ').find_map(|f| f.strip_prefix(k)).unwrap_or("").to_string(); + let msg = rest.split_once("msg=").map(|(_, m)| m.to_string()).unwrap_or_default(); + eprintln!("{line}"); + r.crashes.push(Crash { idx: field("idx=").parse().unwrap_or(usize::MAX), kind: field("kind="), msg }); + } else if let Some(l) = line.strip_prefix("LOW ") { + r.low = l.parse().unwrap_or(r.low); + } else if let Some(s) = line.strip_prefix("START ") { + r.last_start = s.parse().ok(); + } else if !line.is_empty() && abort_msg.len() < 300 { + // An abort's own words: a stack overflow, the allocator, a UB check. + abort_msg.push_str(&line); + abort_msg.push(' '); + } + } + let status = child.wait().unwrap(); + r.completed = matches!(status.code(), Some(0) | Some(3)); + let panicked = matches!(status.code(), Some(101) | Some(102) | Some(103)); + if !r.completed && !panicked { + // No hook ran. `idx` is filled in by locating, or from `--trace-starts`. + let msg = format!("{status} {}", abort_msg.trim()); + eprintln!("CRASH kind=abort msg={msg}"); + r.crashes.push(Crash { idx: r.last_start.unwrap_or(usize::MAX), kind: "abort".into(), msg }); + } + r.status = format!("{status}"); + r +} + +fn supervise(args: &Args, source: Source) { + let n = source.len(); + let save = args.str_flag("--save"); + let max_failures = args.flag("--max-failures", 2000); + let jobs = args.flag("-j", 1).clamp(1, 256); + let to = args.flag("--to", usize::MAX).min(n); + let mut from = args.flag("--from", 0); + let mut skip: Vec = Vec::new(); + let mut crashes: Vec = Vec::new(); + let start = Instant::now(); + loop { + let mut r = run_child(args, from, to, jobs, &skip, false); + for c in r.crashes.iter_mut() { + if c.idx != usize::MAX { continue } + // No hook ran: find the input one at a time from the low-water mark. + eprintln!("unattributed failure ({}); locating from {}", c.msg, r.low); + let l = run_child(args, r.low, to, 1, &skip, true); + match l.crashes.into_iter().find(|x| x.idx != usize::MAX) { + Some(found) => { eprintln!(" located at {}", found.idx); c.idx = found.idx; if found.kind == "abort" { c.msg = found.msg; } else { *c = found; } } + None => eprintln!(" could not locate"), + } + } + let progressed = !r.crashes.is_empty() || r.completed; + for c in r.crashes.iter().filter(|c| c.kind != "panic" && c.kind != "hang" && c.idx != usize::MAX) { + // The child saves what its hooks report; a signal or an abort leaves it to us. + if let Some(dir) = &save { + let _ = std::fs::create_dir_all(dir); + let _ = std::fs::write(std::path::Path::new(dir).join(format!("{:08}.bin", c.idx)), source.get(c.idx)); + } + } + for c in r.crashes { + if c.idx != usize::MAX { skip.push(c.idx); } + crashes.push(c); + } + if r.completed { + break; + } + if !progressed { + eprintln!("child died ({}) with nothing to show; stopping", r.status); + break; + } + from = r.low; + if crashes.len() >= max_failures { + eprintln!("{max_failures} failures; stopping (--max-failures)"); + break; + } + } + let mut by_site: BTreeMap> = BTreeMap::new(); + for c in &crashes { + by_site.entry(site(c)).or_default().push(c.idx); + } + println!("--- failures by site ---"); + for (s, idxs) in &by_site { + let shown: Vec = idxs.iter().take(5).map(|i| i.to_string()).collect(); + println!("{:6} {s}\n inputs: {}{}", idxs.len(), shown.join(" "), if idxs.len() > 5 { " ..." } else { "" }); + } + println!("{} failures ({} sites) over inputs {}..{} in {:.1}s", crashes.len(), by_site.len(), args.flag("--from", 0), to, start.elapsed().as_secs_f64()); + if !crashes.is_empty() { + std::process::exit(1); + } +} diff --git a/differential/src/bin/in_process.rs b/differential/src/bin/in_process.rs index 1cf64bb0..742733bf 100644 --- a/differential/src/bin/in_process.rs +++ b/differential/src/bin/in_process.rs @@ -64,61 +64,7 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use differential::act::run_act as crate_run_act; use differential::harness::run as crate_run; use differential::reference::fuzz::run as model_run; - -/// Where inputs come from. -/// -/// `get` is deterministic in `idx` and holds no state between calls, so which -/// thread runs an input cannot change it and a failing input can be re-derived -/// from its index alone. Mirrors `InputSource` in `lean/differential.py`; a -/// queue-backed source drops in here the same way. -enum Source { - Random { seed: u64, count: usize, maxlen: usize }, - Files(Vec), -} - -impl Source { - fn len(&self) -> usize { - match self { - Source::Random { count, .. } => *count, - Source::Files(v) => v.len(), - } - } - - fn name(&self, idx: usize) -> String { - match self { - Source::Random { .. } => format!("random#{idx:06}"), - Source::Files(v) => v[idx].clone(), - } - } - - fn get(&self, idx: usize) -> Vec { - match *self { - Source::Random { seed, maxlen, .. } => { - // splitmix64, seeded per index so generation parallelises without - // changing what gets tested. - let mut s = seed.wrapping_mul(0x9E3779B97F4A7C15) - ^ (idx as u64).wrapping_mul(0xBF58476D1CE4E5B9); - let mut next = || { - s = s.wrapping_add(0x9E3779B97F4A7C15); - let mut z = s; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - }; - // The same *distribution* as `RandomInputs.get` in - // `lean/differential.py` -- `randrange(8, maxlen)`, i.e. uniform - // on `[8, maxlen)` -- so a divergence rate measured here is - // directly comparable to one measured there. The bit stream - // differs (splitmix64 against Python's Mersenne Twister) and is - // meant to: two independent samples of the same population. - let span = maxlen.saturating_sub(8).max(1); - let n = 8 + (next() as usize) % span; - (0..n).map(|_| next() as u8).collect() - } - Source::Files(ref v) => std::fs::read(&v[idx]).expect("cannot read input"), - } - } -} +use differential::source::Source; /// The index each thread is currently working on, so a panic or an abort can be /// attributed. diff --git a/differential/src/crash.rs b/differential/src/crash.rs new file mode 100644 index 00000000..13d70415 --- /dev/null +++ b/differential/src/crash.rs @@ -0,0 +1,1081 @@ +//! A second op table for `pathmap`, with no model behind it: an input only has +//! to run to the end without a panic, a failed debug assertion, an abort or a +//! hang. +//! +//! [`crate::harness`] is differential, so it can only reach what the Lean model +//! specifies, and that leaves a lot of the crate alone. This table covers that +//! surface. It has no trace, no lockstep partner and nothing to agree with, so +//! it can grow freely: +//! +//! * **paths** outside the harness's 4-letter alphabet and 5-byte limit: bytes +//! spread over all four `ByteMask` words, full bytes, and keys up to 67 bytes, +//! longer than a list node holds inline; +//! * **value types** other than `u64`: `()`, `bool` and `u16`, whose lattice +//! impls report identities differently; +//! * **`prune = true`** on every operation that takes it, and `prune_path` / +//! `prune_ascend` off the map root; +//! * **`PathMap` methods** the harness never calls: `insert`/`remove`/`get_mut`, +//! `remove_branches_at`, map-level `join`/`meet`/`subtract`/`restrict` and the +//! `Lattice` impls, clones and copy-on-write, `iter`/`from_iter`, `merkleize`, +//! `new_from_ana`; +//! * **zipper kinds**: owned read and write zippers, `read_zipper_at_borrowed_path`, +//! forks, `TrieRef`, `ZipperHead` and `ZipperHeadOwned` with several zippers +//! live at once, `ProductZipper`, `ProductZipperG`, +//! `DependentProductZipperG`, `OverlayZipper`, `PrefixZipper`, `EmptyZipper` +//! and the ACT zipper; +//! * **operations** absent from the harness: `join_into_take`, `drop_head`, +//! `graft_map` on arbitrary maps, `meet_2` over two independent sources, +//! `take_map` without restoring, `meet_k_path_into` on any focus, `k = 0`, +//! witnesses, the `_observed` variants, `descend_indexed_branch`, +//! `reserve_buffers`, `get_focus`, `try_borrow_focus`; +//! * **catamorphisms** in all four flavours, fallible ones stopping early, and +//! `hash`; +//! * **`.paths` serialization**, round trips and decoding of corrupted streams. +//! +//! Calls with a *documented* panic are made to satisfy the documented +//! precondition (see [`KNOWN_PRECONDITIONS`]); anything else that panics is a +//! finding. +//! +//! # Wire format +//! +//! Byte-oriented like the harness's, so AFL mutations land on operands: every +//! operand is one or a few bytes reduced at the point of use, and a truncated +//! input is a shorter valid program. The first byte picks the value type, then +//! come three seeded maps, then the op stream. + +use std::io::Cursor; + +use pathmap::PathMap; +use pathmap::arena_compact::ArenaCompactTree; +use pathmap::morphisms::Catamorphism; +use pathmap::paths_serialization::{deserialize_paths, for_each_deserialized_path, serialize_paths}; +use pathmap::ring::{AlgebraicResult, DistributiveLattice, Lattice, SELF_IDENT}; +use pathmap::utils::{BitMask, ByteMask}; +use pathmap::zipper::*; + +use crate::harness::{Dec, hex_path}; + +/// `CRASH_TRACE=1` prints every operation to stderr as it starts, so the last +/// line before a hang or a panic names the call that did it. +static TRACE: std::sync::OnceLock = std::sync::OnceLock::new(); + +macro_rules! note { + ($($arg:tt)*) => { + if *TRACE.get_or_init(|| std::env::var_os("CRASH_TRACE").is_some()) { + eprintln!($($arg)*); + } + }; +} + +/// Total work per input, counted in operations including those inside episodes. +pub const MAX_STEPS: usize = 512; +/// How many steps one episode (one zipper's lifetime) may take. +const EPISODE_STEPS: usize = 48; +/// Maps in play. +const NMAPS: usize = 3; +/// Bound on any loop this file drives itself (iteration walks and the like). +const WALK: usize = 64; +/// A map that grows past this many values stops accepting growth ops, so a run +/// of `insert_prefix` and joins cannot turn one input into a memory benchmark. +const MAX_VALS: usize = 4096; + +/// Known failures the table steers around unless [`set_include_known`] says +/// otherwise, so a survey reports what is new. Each is either a documented +/// stub or a documented open issue. +pub const KNOWN_FAILURES: &[&str] = &[ + "val_count is todo!/unimplemented! on OverlayZipper, ProductZipperG and DependentProductZipperG", + "TrieRef::fork_read_zipper panics on a TrieRef at a non-existent path (GOAT, issue #96)", + "k == 0 is degenerate for k-path iteration, join_k_path_into and drop_head (the harness skips it as \ + skip:k0): it spins on several zipper kinds and trips `debug_assert!(byte_cnt > 0)` in drop_head", + "to_next_k_path without a preceding descend_first_k_path continues state nothing set up; the harness \ + only runs whole walks, and alone it underflows path_len (and read_zipper_at_borrowed_path's)", + "meet_k_path_into spins forever when the focus has no children, and escapes it when k == 0 \ + (see Zip.meetKPathUnspecified in the Lean model)", +]; + +static INCLUDE_KNOWN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Run the calls listed in [`KNOWN_FAILURES`] as well. +pub fn set_include_known(on: bool) { + INCLUDE_KNOWN.store(on, std::sync::atomic::Ordering::Relaxed); +} + +fn include_known() -> bool { + INCLUDE_KNOWN.load(std::sync::atomic::Ordering::Relaxed) +} + +thread_local! { + /// Set while driving a zipper type whose `val_count` is a stub. + static VAL_COUNT_STUB: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Marks the current episode as driving a `val_count` stub, until dropped. +struct ValCountStub; + +impl ValCountStub { + fn new() -> Self { + VAL_COUNT_STUB.with(|c| c.set(true)); + ValCountStub + } +} + +impl Drop for ValCountStub { + fn drop(&mut self) { + VAL_COUNT_STUB.with(|c| c.set(false)); + } +} + +fn val_count_ok() -> bool { + include_known() || !VAL_COUNT_STUB.with(|c| c.get()) +} + +/// Preconditions this table deliberately satisfies, because the crate documents +/// a panic when they are broken. Anything else that panics is a finding. +pub const KNOWN_PRECONDITIONS: &[&str] = &[ + "ProductZipper::new: secondary factors must be at node roots (factors are map-root zippers)", + "TrieBuilder::push / push_byte / graft_at_byte: first bytes strictly increasing", + "TrieBuilder::graft_at_byte: the source focus must hold a node (called only with children below)", + "graft_child_maps: at least one map per set bit of the mask", +]; + +/// A value type the table can run over. +pub trait CrashValue: + Clone + Send + Sync + Unpin + Lattice + DistributiveLattice + core::hash::Hash + core::fmt::Debug + Default + 'static +{ + fn from_byte(b: u8) -> Self; + fn to_u64(&self) -> u64; +} + +impl CrashValue for u64 { + fn from_byte(b: u8) -> Self { b as u64 } + fn to_u64(&self) -> u64 { *self } +} +impl CrashValue for u16 { + fn from_byte(b: u8) -> Self { (b as u16) << (b % 9) } + fn to_u64(&self) -> u64 { *self as u64 } +} +impl CrashValue for bool { + fn from_byte(b: u8) -> Self { b & 1 == 1 } + fn to_u64(&self) -> u64 { *self as u64 } +} +impl CrashValue for () { + fn from_byte(_: u8) -> Self {} + fn to_u64(&self) -> u64 { 0 } +} + +// --------------------------------------------------------------------------- +// Operand decoding +// --------------------------------------------------------------------------- + +/// A path over one of three alphabets, chosen by the low two bits of its header +/// byte: `0..4` (so paths share prefixes, as in the harness), sixteen bytes +/// spread over the whole range (so all four mask words are used), or any byte. +/// The rest of the header picks the length: mostly `0..7`, sometimes 7 to 67. +fn path(d: &mut Dec) -> Option> { + let h = d.u8()?; + let s = (h >> 2) as usize; + let len = if s < 48 { s % 7 } else { 7 + (s - 48) * 4 }; + let mut v = Vec::with_capacity(len); + for _ in 0..len { + let b = d.u8()?; + v.push(match h & 3 { + 0 | 1 => b % 4, + 2 => (b % 16) * 17, + _ => b, + }); + } + Some(v) +} + +/// A short path, for places where a long one only costs time. +fn short_path(d: &mut Dec) -> Option> { + let mut p = path(d)?; + p.truncate(8); + Some(p) +} + +/// One byte, mostly from the small alphabet so it hits existing children. +fn byte(d: &mut Dec) -> Option { + let b = d.u8()?; + if b < 160 { Some(b % 4) } else { d.u8() } +} + +fn mask(d: &mut Dec) -> Option { + let n = d.modn(6)?; + let mut m = ByteMask::EMPTY; + for _ in 0..n { + m.set_bit(byte(d)?); + } + Some(m) +} + +fn val(d: &mut Dec) -> Option { + Some(V::from_byte(d.u8()?)) +} + +/// Result of a map-level lattice operation, as a map. +fn resolve(r: AlgebraicResult>, a: &PathMap, b: &PathMap) -> PathMap { + match r { + AlgebraicResult::None => PathMap::new(), + AlgebraicResult::Identity(m) => if m & SELF_IDENT != 0 { a.clone() } else { b.clone() }, + AlgebraicResult::Element(x) => x, + } +} + +/// Count one step against the input's budget; `None` ends the input. +fn tick(steps: &mut usize) -> Option<()> { + *steps += 1; + if *steps > MAX_STEPS { None } else { Some(()) } +} + +/// The per-input state: the maps, and the step budget shared by everything. +pub struct State { + pub maps: [PathMap; NMAPS], + steps: usize, +} + +impl State { + fn small(&self, m: usize) -> bool { + self.maps[m].val_count() < MAX_VALS + } +} + +// --------------------------------------------------------------------------- +// Steps shared by every zipper kind +// --------------------------------------------------------------------------- + +// Observers are started from the zipper's current path: a `Vec` or `usize` +// observer mirrors the path, and an ascent above where it started would underflow it. + +/// One movement or query. Everything a `ZipperMoving + ZipperPath` offers. +fn move_step(d: &mut Dec, z: &mut Z) -> Option<()> { + let __op = d.modn(34)?; + note!("move {__op} at {}", hex_path(z.path())); + match __op { + 0 => { let p = path(d)?; note!(" p={}", hex_path(&p)); z.descend_to(&p); } + 1 => { let b = byte(d)?; z.descend_to_byte(b); } + 2 => { let p = path(d)?; let _ = z.descend_to_check(&p); } + 3 => { let p = path(d)?; let _ = z.descend_to_existing(&p); } + 4 => { let p = path(d)?; let _ = z.descend_to_val(&p); } + 5 => { let b = byte(d)?; let _ = z.descend_to_existing_byte(b); } + 6 => { let i = d.modn(8)?; let _ = z.descend_indexed_byte(i); } + #[allow(deprecated)] + 7 => { let i = d.modn(8)?; let _ = z.descend_indexed_branch(i); } + 8 => { let _ = z.descend_first_byte(); } + 9 => { let _ = z.descend_last_byte(); } + 10 => { let _ = z.descend_until(); } + 11 => { let mut o = z.path().to_vec(); let _ = z.descend_until_observed(&mut o); } + 12 => { let n = d.modn(12)?; let _ = z.descend_until_max_bytes(n); } + 13 => { let n = d.modn(12)?; let mut o = z.path().len(); let _ = z.descend_until_max_bytes_observed(n, &mut o); } + 14 => { let n = d.modn(12)?; let _ = z.ascend(n); } + 15 => { let _ = z.ascend_byte(); } + 16 => { let _ = z.ascend_until(); } + 17 => { let _ = z.ascend_until_branch(); } + 18 => { let _ = z.to_next_sibling_byte(); } + 19 => { let _ = z.to_prev_sibling_byte(); } + 20 => { let _ = z.to_next_step(); } + 21 => { let mut o = z.path().to_vec(); let _ = z.to_next_step_observed(&mut o); } + 22 => { let p = path(d)?; let _ = z.move_to_path(&p); } + 23 => z.reset(), + 24 => { let _ = (z.path_exists(), z.is_val(), z.child_count(), z.child_mask()); } + 25 => { let _ = (z.depth(), z.at_root(), z.focus_byte(), z.path().len()); } + 26 => { if val_count_ok() { let _ = z.val_count(); } } + 27 => { + // A sibling walk, the way callers enumerate children. + if z.descend_first_byte().is_some() { + let mut n = 0; + while n < WALK && z.to_next_sibling_byte().is_some() { n += 1; } + let _ = z.ascend_byte(); + } + } + 28 => { + if z.descend_last_byte().is_some() { + let mut n = 0; + while n < WALK && z.to_prev_sibling_byte().is_some() { n += 1; } + let _ = z.ascend_byte(); + } + } + 29 => { let mut n = 0; while n < WALK && z.to_next_step() { n += 1; } } + #[allow(deprecated)] + 30 => { let _ = z.is_value(); } + #[allow(deprecated)] + 31 => { let p = path(d)?; let _ = z.descend_to_value(&p); } + _ => { let b = byte(d)?; z.descend_to_byte(b); let _ = z.ascend_byte(); } + } + Some(()) +} + +/// A k for k-path iteration: `0..n`, or `1..n` unless known failures are included. +fn kpath_k(d: &mut Dec, n: usize) -> Option { + let k = d.modn(n)?; + Some(if k == 0 && !include_known() { 1 } else { k }) +} + +/// One iteration step, or a movement. +fn iter_step(d: &mut Dec, z: &mut Z) -> Option<()> { + let __op = d.modn(14)?; + note!("iter {__op} at {}", hex_path(z.path())); + match __op { + 0 => { let _ = z.to_next_val(); } + 1 => { let mut o = z.path().to_vec(); let _ = z.to_next_val_observed(&mut o); } + 2 => { let _ = z.descend_last_path(); } + 3 => { let mut o = z.path().len(); let _ = z.descend_last_path_observed(&mut o); } + 4 => { let k = kpath_k(d, 6)?; note!(" k={k}"); let _ = z.descend_first_k_path(k); } + 5 => { + let k = kpath_k(d, 6)?; + note!(" k={k}"); + if include_known() { let _ = z.to_next_k_path(k); } else if z.descend_first_k_path(k) { let _ = z.to_next_k_path(k); } + } + 6 => { + // A whole k-path walk. + let k = kpath_k(d, 5)?; + note!(" k={k}"); + if z.descend_first_k_path(k) { + let mut n = 0; + while n < WALK && z.to_next_k_path(k) { n += 1; } + } + } + 7 => { + let k = kpath_k(d, 5)?; + note!(" k={k}"); + let mut o = z.path().to_vec(); + if z.descend_first_k_path_observed(k, &mut o) { + let mut n = 0; + while n < WALK && z.to_next_k_path_observed(k, &mut o) { n += 1; } + } + } + 8 => { let mut n = 0; while n < WALK && z.to_next_val() { n += 1; } } + _ => move_step(d, z)?, + } + Some(()) +} + +/// A read program over a zipper with iteration and values. +fn read_program(d: &mut Dec, steps: &mut usize, z: &mut Z) -> Option<()> +where + V: Clone, + Z: ZipperMoving + ZipperPath + ZipperIteration + ZipperValues, +{ + let n = d.modn(EPISODE_STEPS)?; + for _ in 0..n { + tick(steps)?; + match d.modn(8)? { + 0 => { let _ = z.val().cloned(); } + #[allow(deprecated)] + 1 => { let _ = z.value().cloned(); } + _ => iter_step(d, z)?, + } + } + Some(()) +} + +/// A read program over a zipper that moves and has values, but cannot iterate. +fn move_program(d: &mut Dec, steps: &mut usize, z: &mut Z) -> Option<()> +where + V: Clone, + Z: ZipperMoving + ZipperPath + ZipperValues, +{ + let n = d.modn(EPISODE_STEPS)?; + for _ in 0..n { + tick(steps)?; + match d.modn(8)? { + 0 => { let _ = z.val().cloned(); } + _ => move_step(d, z)?, + } + } + Some(()) +} + +/// The read-only value accessors: references that outlive the borrow of the +/// zipper, and iteration that hands them out. +macro_rules! ro_extras { + ($d:expr, $z:expr) => {{ + match $d.modn(4)? { + 0 => { let _ = $z.get_val().cloned(); } + 1 => { let p = path($d)?; let _ = $z.get_val_at(&p).cloned(); } + 2 => { let mut n = 0; while n < WALK && $z.to_next_get_val().is_some() { n += 1; } } + _ => { let mut o = $z.path().to_vec(); let _ = $z.to_next_get_val_observed(&mut o).cloned(); } + } + }}; +} + +/// Everything a subtrie-capable read zipper offers beyond moving: witnesses, +/// focus borrowing, forks, trie refs, buffer management. +macro_rules! sub_extras { + ($d:expr, $steps:expr, $z:expr) => {{ + match $d.modn(12)? { + 0 => { let w = $z.witness(); let _ = $z.get_val_with_witness(&w).cloned(); } + 1 => { + let w = $z.witness(); + let mut n = 0; + while n < WALK && $z.to_next_get_val_with_witness(&w).is_some() { n += 1; } + } + 2 => { let _ = $z.make_map().val_count(); } + 3 => { let _ = $z.try_make_map().map(|m| m.val_count()); } + 4 => { let p = path($d)?; let t = $z.trie_ref_at_path(&p); trie_ref_ops($d, $steps, &t)?; } + 5 => { let _ = ($z.is_shared(), $z.shared_node_id()); } + 6 => { let _ = $z.get_focus(); let p = path($d)?; let _ = $z.get_focus_at(&p); let _ = $z.try_borrow_focus().is_some(); } + 7 => { let (a, b) = ($d.modn(80)?, $d.modn(24)?); $z.reserve_buffers(a, b); $z.prepare_buffers(); } + 8 => { let mut f = $z.fork_read_zipper(); read_program($d, $steps, &mut f)?; } + 9 => { let _ = ($z.origin_path().len(), $z.root_prefix_path().len()); } + 10 => { let _ = $z.get_trie_ref().child_count(); } + _ => { let _ = $z.native_subtries(); let _ = $z.trie_ref().is_some(); let p = path($d)?; let _ = $z.val_at(&p).cloned(); } + } + }}; +} + +/// Operations on a `TrieRef`, which is a zipper that cannot move. +fn trie_ref_ops(d: &mut Dec, steps: &mut usize, t: &T) -> Option<()> +where + V: Clone + Send + Sync + Unpin, + T: ZipperValuesAt + Zipper + ZipperInfallibleSubtries + ZipperConcrete, +{ + let n = d.modn(8)?; + for _ in 0..n { + tick(steps)?; + match d.modn(6)? { + 0 => { let _ = (t.val().cloned(), t.path_exists(), t.is_val(), t.child_count(), t.child_mask()); } + 1 => { let p = path(d)?; let _ = t.val_at(&p).cloned(); } + 2 => { let _ = t.make_map().val_count(); } + 3 => { let _ = (t.is_shared(), t.shared_node_id()); } + 4 => { let p = path(d)?; let _ = t.get_focus_at(&p); let _ = t.get_trie_ref().child_count(); } + _ => { let _ = t.try_borrow_focus().is_some(); } + } + } + Some(()) +} + +// --------------------------------------------------------------------------- +// Writing +// --------------------------------------------------------------------------- + +/// One write, or a movement. `srcs` are clones of the maps taken when the +/// episode began: sources that share nodes with the destination, which is what +/// most callers of the algebraic operations hand in. +fn write_step(d: &mut Dec, z: &mut W, srcs: &[PathMap; NMAPS]) -> Option<()> +where + V: CrashValue, + W: ZipperWriting + ZipperMoving + ZipperPath + ZipperValues, +{ + let src = |d: &mut Dec| -> Option<&PathMap> { Some(&srcs[d.modn(NMAPS)?]) }; + let op = d.modn(40)?; + note!("write {op} at {}", hex_path(z.path())); + // Growth is refused below an oversized focus; everything else still runs. + let grows = matches!(op, 9 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 23 | 26 | 28 | 29 | 30); + if grows && z.val_count() > MAX_VALS { + return Some(()); + } + match op { + 0 => { let v = val(d)?; let _ = z.set_val(v); } + 1 => { let pr = d.boolean()?; let _ = z.remove_val(pr); } + 2 => { let v = val(d)?; if let Some(slot) = z.get_val_mut() { *slot = v; } } + 3 => { let v = val(d)?; let _ = z.get_val_or_set_mut(v).clone(); } + 4 => { let v = val(d)?; let _ = z.get_val_or_set_mut_with(|| v).clone(); } + 5 => { let _ = z.create_path(); } + 6 => { let _ = z.prune_path(); } + 7 => { let _ = z.prune_ascend(); } + 8 => { let pr = d.boolean()?; let _ = z.remove_branches(pr); } + 9 => { let s = src(d)?; let p = short_path(d)?; z.graft(&s.read_zipper_at_path(&p)); } + 10 => { let s = src(d)?; let p = short_path(d)?; z.graft_map(s.read_zipper_at_path(&p).make_map()); } + 11 => { let s = src(d)?; let p = short_path(d)?; let q = path(d)?; z.graft_src_at(&s.read_zipper_at_path(&p), &q); } + 12 => { let s = src(d)?; let p = short_path(d)?; let _ = z.join_into(&s.read_zipper_at_path(&p)); } + 13 => { let s = src(d)?; let p = short_path(d)?; let _ = z.join_map_into(s.read_zipper_at_path(&p).make_map()); } + 14 => { let s = src(d)?; let p = short_path(d)?; let pr = d.boolean()?; let _ = z.meet_into(&s.read_zipper_at_path(&p), pr); } + 15 => { let s = src(d)?; let p = short_path(d)?; let pr = d.boolean()?; let _ = z.subtract_into(&s.read_zipper_at_path(&p), pr); } + 16 => { let s = src(d)?; let p = short_path(d)?; let _ = z.restrict(&s.read_zipper_at_path(&p)); } + 17 => { let s = src(d)?; let p = short_path(d)?; let _ = z.restricting(&s.read_zipper_at_path(&p)); } + 18 => { + // Two independent sources, possibly different maps. + let (a, pa) = (src(d)?, short_path(d)?); + let (b, pb) = (src(d)?, short_path(d)?); + let _ = z.meet_2(&a.read_zipper_at_path(&pa), &b.read_zipper_at_path(&pb)); + } + 19 => { + // The source is a write zipper on a private copy, emptied by the join. + let s = src(d)?.clone(); + let p = short_path(d)?; + let pr = d.boolean()?; + let mut sw = s.into_write_zipper(&p); + let _ = z.join_into_take(&mut sw, pr); + let _ = sw.into_map().val_count(); + } + 20 => { let k = kpath_k(d, 6)?; let pr = d.boolean()?; let _ = z.join_k_path_into(k, pr); } + 21 => { + let k = d.modn(6)?; + let pr = d.boolean()?; + if include_known() || (k > 0 && z.child_count() > 0) { + let _ = z.meet_k_path_into(k, pr); + } + } + #[allow(deprecated)] + 22 => { let k = kpath_k(d, 6)?; let _ = z.drop_head(k); } + 23 => { let p = path(d)?; let _ = z.insert_prefix(&p); } + 24 => { let n = d.modn(10)?; let _ = z.remove_prefix(n); } + 25 => { let pr = d.boolean()?; let _ = z.take_map(pr).map(|m| m.val_count()); } + 26 => { + let pr = d.boolean()?; + if let Some(m) = z.take_map(pr) { + let p = short_path(d)?; + let _ = z.descend_to_existing(&p); + z.graft_map(m); + } + } + 27 => { let m = mask(d)?; let pr = d.boolean()?; z.remove_unmasked_branches(m, pr); } + 28 => { let s = src(d)?; let p = short_path(d)?; let m = mask(d)?; let ru = d.boolean()?; z.graft_masked_branches(&s.read_zipper_at_path(&p), m, ru); } + 29 => { + let s = src(d)?; + let m = mask(d)?; + let ru = d.boolean()?; + let extra = d.modn(3)?; + // One map per set bit, as documented, plus possibly a few spare ones. + let maps: Vec> = m.iter().map(|b| s.read_zipper_at_path([b]).make_map()) + .chain((0..extra).map(|_| s.clone())).collect(); + z.graft_child_maps(m, maps, ru); + } + 30 => { + // `.paths` into the focus, from a serialized copy of a source. + let s = src(d)?; + let mut buf = Vec::new(); + if serialize_paths(s.read_zipper(), &mut buf).is_ok() { + let v = val(d)?; + let _ = deserialize_paths(&mut *z, Cursor::new(&buf[..]), v); + } + } + _ => move_step(d, z)?, + } + Some(()) +} + +/// A write program over one write zipper. +fn write_program(d: &mut Dec, steps: &mut usize, z: &mut W, srcs: &[PathMap; NMAPS]) -> Option<()> +where + V: CrashValue, + W: ZipperWriting + ZipperMoving + ZipperPath + ZipperValues, +{ + let n = d.modn(EPISODE_STEPS)?; + for _ in 0..n { + tick(steps)?; + write_step(d, z, srcs)?; + } + Some(()) +} + +fn snapshot(maps: &[PathMap; NMAPS]) -> [PathMap; NMAPS] { + [maps[0].clone(), maps[1].clone(), maps[2].clone()] +} + +// --------------------------------------------------------------------------- +// Episodes: one zipper kind, created, driven, dropped +// --------------------------------------------------------------------------- + +fn read_episode(d: &mut Dec, st: &mut State) -> Option<()> { + let m = d.modn(NMAPS)?; + let p = short_path(d)?; + let map = st.maps[m].clone(); + let steps = &mut st.steps; + let __kind = d.modn(12)?; + note!("read episode {__kind} map {m} at {}", hex_path(&p)); + match __kind { + 0 => { + let mut z = map.read_zipper_at_path(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { 0 => ro_extras!(d, z), 1 => sub_extras!(d, steps, z), _ => iter_step(d, &mut z)? } + } + read_program(d, steps, &mut z)?; + } + 1 => { + let mut z = map.read_zipper_at_borrowed_path(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { 0 => ro_extras!(d, z), 1 => sub_extras!(d, steps, z), _ => iter_step(d, &mut z)? } + } + read_program(d, steps, &mut z)?; + } + 2 => { + let mut z = map.into_read_zipper(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + if d.boolean()? { sub_extras!(d, steps, z) } else { iter_step(d, &mut z)? } + } + read_program(d, steps, &mut z)?; + let mut c = z.clone(); + read_program(d, steps, &mut c)?; + } + 3 => { + let t = map.trie_ref_at_path(&p); + trie_ref_ops(d, steps, &t)?; + let q = path(d)?; + let t2 = t.trie_ref_at_path(&q); + trie_ref_ops(d, steps, &t2)?; + if include_known() || t2.path_exists() { + let mut f = t2.fork_read_zipper(); + read_program(d, steps, &mut f)?; + } + } + 4 => { + // A prefix in front of a zipper, possibly rooted part way into it. + let prefix = path(d)?; + let mut z = PrefixZipper::new(&prefix[..], map.read_zipper_at_path(&p)); + if d.boolean()? { + let cut = d.modn(prefix.len() + 1)?; + note!(" prefix={} cut={cut}", hex_path(&prefix)); + let _ = z.set_root_prefix_path(&prefix[..cut]); + } + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { 0 => ro_extras!(d, z), 1 => sub_extras!(d, steps, z), _ => iter_step(d, &mut z)? } + } + read_program(d, steps, &mut z)?; + } + 5 => { + let other = st.maps[d.modn(NMAPS)?].clone(); + let q = short_path(d)?; + let _stub = ValCountStub::new(); + let mut z = OverlayZipper::new(map.read_zipper_at_path(&p), other.read_zipper_at_path(&q)); + move_program(d, &mut st.steps, &mut z)?; + } + 6 => { + // Secondary factors are map-root zippers: see KNOWN_PRECONDITIONS. + let others: Vec> = (0..d.modn(4)?).map(|_| d.modn(NMAPS).map(|i| st.maps[i].clone())).collect::>()?; + let more = st.maps[d.modn(NMAPS)?].clone(); + let steps = &mut st.steps; + let mut z = ProductZipper::new(map.read_zipper_at_path(&p), others.iter().map(|o| o.read_zipper())); + if d.boolean()? { + z.new_factors([more.read_zipper()]); + } + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(4)? { + 0 => { let _ = (z.focus_factor(), z.factor_count(), z.path_indices().len()); } + 1 => { let w = z.witness(); let _ = z.get_val_with_witness(&w).cloned(); } + 2 => { let _ = (z.is_shared(), z.shared_node_id(), z.origin_path().len()); } + _ => iter_step(d, &mut z)?, + } + } + read_program(d, steps, &mut z)?; + } + 7 => { + let others: Vec> = (0..d.modn(4)?).map(|_| d.modn(NMAPS).map(|i| st.maps[i].clone())).collect::>()?; + let steps = &mut st.steps; + let _stub = ValCountStub::new(); + let mut z = ProductZipperG::new(map.read_zipper_at_path(&p), others.iter().map(|o| o.read_zipper_at_path(&[]))); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { + 0 => { let _ = (z.focus_factor(), z.factor_count(), z.path_indices().len()); } + _ => iter_step(d, &mut z)?, + } + } + read_program(d, steps, &mut z)?; + } + 8 => { + // Factors enrolled as the zipper walks: a map-root zipper on one of + // the maps, chosen from the path and the depth, a bounded number of times. + let pool = snapshot(&st.maps); + let steps = &mut st.steps; + let sel = d.u8()?; + let budget = d.modn(4)?; + let _stub = ValCountStub::new(); + let mut z = DependentProductZipperG::new_enroll( + map.read_zipper_at_path(&p), + budget, + move |left: usize, path: &[u8], depth: usize| { + let pick = (path.iter().fold(sel as usize, |a, &b| a.wrapping_mul(31).wrapping_add(b as usize)) + depth) % (NMAPS + 1); + if left == 0 || pick == NMAPS { + (left, None) + } else { + (left - 1, Some(pool[pick].clone().into_read_zipper(&[]))) + } + }, + ); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { + 0 => { let _ = (z.focus_factor(), z.path_indices().len()); } + _ => iter_step(d, &mut z)?, + } + } + read_program(d, steps, &mut z)?; + } + 9 => { + let mut z = EmptyZipper::new_at_path(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(3)? { 0 => { let _: Option<&V> = z.get_val(); } _ => iter_step(d, &mut z)? } + } + read_program::(d, steps, &mut z)?; + } + 10 => { + // A fork taken part way through a walk, driven while the parent lives. + let mut z = map.read_zipper_at_path(&p); + move_program(d, steps, &mut z)?; + let mut f = z.fork_read_zipper(); + read_program(d, steps, &mut f)?; + drop(f); + read_program(d, steps, &mut z)?; + } + _ => { + // The arena-compact form of the map. + let act = ArenaCompactTree::from_zipper(map.read_zipper(), |v: &V| v.to_u64()); + let mut z = act.read_zipper_at_path_u64(&p); + for _ in 0..d.modn(8)? { + tick(steps)?; + match d.modn(4)? { + 0 => { let _ = act.get_val_at(&p); } + 1 => { let mut n = 0; for _ in act.iter() { n += 1; if n > WALK { break } } } + 2 => { let mut f = z.fork_read_zipper(); read_program(d, steps, &mut f)?; } + _ => iter_step(d, &mut z)?, + } + } + read_program(d, steps, &mut z)?; + let mut u = act.read_zipper_at_path(&p); + read_program(d, steps, &mut u)?; + } + } + Some(()) +} + +fn write_episode(d: &mut Dec, st: &mut State) -> Option<()> { + let m = d.modn(NMAPS)?; + let p = path(d)?; + let srcs = snapshot(&st.maps); + let State { maps, steps } = st; + let __kind = d.modn(7)?; + note!("write episode {__kind} map {m} at {}", hex_path(&p)); + match __kind { + 0 => { + let mut z = maps[m].write_zipper_at_path(&p); + write_program(d, steps, &mut z, &srcs)?; + } + 1 => { + let mut z = maps[m].write_zipper(); + z.descend_to(&p); + write_program(d, steps, &mut z, &srcs)?; + // A fork of the write zipper, read while the writer is still live. + { + let mut f = z.fork_read_zipper(); + read_program(d, steps, &mut f)?; + } + write_program(d, steps, &mut z, &srcs)?; + } + 2 => { + // An owned write zipper, turned back into the map afterwards. + let map = std::mem::take(&mut maps[m]); + let mut z = map.into_write_zipper(&p); + let r = write_program(d, steps, &mut z, &srcs); + maps[m] = z.into_map(); + r?; + } + 3 => { + let zh = maps[m].zipper_head(); + zh_program(d, steps, &zh, &srcs)?; + } + 4 => { + // A ZipperHead handed out by a write zipper at its focus. + let mut z = maps[m].write_zipper_at_path(&p); + write_program(d, steps, &mut z, &srcs)?; + { + let zh = z.zipper_head(); + zh_program(d, steps, &zh, &srcs)?; + } + write_program(d, steps, &mut z, &srcs)?; + } + 5 => { + let map = std::mem::take(&mut maps[m]); + let zh = map.into_zipper_head(&p); + let r = zh_program(d, steps, &zh, &srcs); + maps[m] = zh.into_map(); + r?; + } + _ => { + // A write zipper and a read zipper on the same map at once, through a head. + let zh = maps[m].zipper_head(); + let q = short_path(d)?; + if let (Ok(mut w), Ok(mut r)) = (zh.write_zipper_at_exclusive_path(&p), zh.read_zipper_at_path(&q)) { + for _ in 0..d.modn(EPISODE_STEPS)? { + tick(steps)?; + if d.boolean()? { write_step(d, &mut w, &srcs)? } else { iter_step(d, &mut r)? } + } + } + } + } + Some(()) +} + +/// Several zippers from one head, live at once and driven in turn. Requests +/// that conflict are refused with `Err`, which is the documented outcome. +fn zh_program<'t, V, H>(d: &mut Dec, steps: &mut usize, zh: &H, srcs: &[PathMap; NMAPS]) -> Option<()> +where + V: CrashValue, + H: ZipperCreation<'t, V>, +{ + let paths: Vec> = (0..3).map(|_| short_path(d)).collect::>()?; + let mut w0 = zh.write_zipper_at_exclusive_path(&paths[0]).ok(); + let mut w1 = zh.write_zipper_at_exclusive_path(&paths[1]).ok(); + let mut r0 = zh.read_zipper_at_path(&paths[2]).ok(); + let mut r1 = zh.read_zipper_at_borrowed_path(&paths[2]).ok(); + let n = d.modn(EPISODE_STEPS)?; + for _ in 0..n { + tick(steps)?; + match d.modn(7)? { + 0 => if let Some(z) = w0.as_mut() { write_step(d, z, srcs)? }, + 1 => if let Some(z) = w1.as_mut() { write_step(d, z, srcs)? }, + 2 => if let Some(z) = r0.as_mut() { iter_step(d, z)? }, + 3 => if let Some(z) = r1.as_mut() { if d.boolean()? { sub_extras!(d, steps, z) } else { iter_step(d, z)? } }, + 4 => { drop(w0.take()); let q = short_path(d)?; w0 = zh.write_zipper_at_exclusive_path(&q).ok(); } + 5 => { drop(r0.take()); let q = short_path(d)?; r0 = zh.read_zipper_at_path(&q).ok(); } + _ => { w1 = None; r1 = None; } + } + } + Some(()) +} + +// --------------------------------------------------------------------------- +// Whole-map operations +// --------------------------------------------------------------------------- + +fn map_op(d: &mut Dec, st: &mut State) -> Option<()> { + let m = d.modn(NMAPS)?; + let __op = d.modn(20)?; + note!("map {__op}"); + match __op { + 0 => { let p = path(d)?; let v = val(d)?; if st.small(m) { let _ = st.maps[m].set_val_at(&p, v); } } + 1 => { let p = path(d)?; let v = val(d)?; if st.small(m) { let _ = st.maps[m].insert(&p, v); } } + 2 => { let p = path(d)?; let pr = d.boolean()?; let _ = st.maps[m].remove_val_at(&p, pr); } + 3 => { let p = path(d)?; let _ = st.maps[m].remove(&p); } + 4 => { + let p = path(d)?; + let v = val(d)?; + match d.modn(3)? { + 0 => { if let Some(slot) = st.maps[m].get_val_mut_at(&p) { *slot = v; } } + 1 => { let _ = st.maps[m].get_val_or_set_mut_at(&p, v).clone(); } + _ => { let _ = st.maps[m].get_val_or_set_mut_with_at(&p, || v).clone(); } + } + } + 5 => { + let p = path(d)?; + let map = &st.maps[m]; + let _ = (map.get(&p).cloned(), map.contains(&p), map.path_exists_at(&p), map.is_empty()); + #[allow(deprecated)] + let _ = map.contains_path(&p); + } + 6 => { let p = path(d)?; let _ = st.maps[m].create_path(&p); } + 7 => { let p = path(d)?; let _ = st.maps[m].prune_path(&p); } + 8 => { let p = path(d)?; let pr = d.boolean()?; let _ = st.maps[m].remove_branches_at(&p, pr); } + 9 => { let to = d.modn(NMAPS)?; st.maps[to] = st.maps[m].clone(); } + 10 => { + let (b, to) = (d.modn(NMAPS)?, d.modn(NMAPS)?); + let (x, y) = (&st.maps[m], &st.maps[b]); + let r = match d.modn(8)? { + 0 => x.join(y), + 1 => x.meet(y), + 2 => x.subtract(y), + 3 => x.restrict(y), + 4 => resolve(x.pjoin(y), x, y), + 5 => resolve(x.pmeet(y), x, y), + 6 => resolve(x.psubtract(y), x, y), + _ => y.restrict(x), + }; + if r.val_count() <= MAX_VALS { st.maps[to] = r; } + } + 11 => { + // A comb: many children under one prefix, over a chosen alphabet, so + // list nodes split and dense nodes form. + let prefix = path(d)?; + let n = d.modn(40)?; + let v: V = val(d)?; + if st.small(m) { + for _ in 0..n { + let mut k = prefix.clone(); + k.push(byte(d)?); + k.extend(short_path(d)?); + let _ = st.maps[m].set_val_at(&k, v.clone()); + } + } + } + 12 => { + let to = d.modn(NMAPS)?; + let rebuilt: PathMap = st.maps[m].iter().take(MAX_VALS).map(|(k, v)| (k, v.clone())).collect(); + let _ = st.maps[m].clone().into_iter().take(WALK).count(); + st.maps[to] = rebuilt; + } + 13 => { let _ = st.maps[m].merkleize(); } + 14 => { let to = d.modn(NMAPS)?; st.maps[to] = ana(d, &st.maps)?; } + 15 => cata(d, &st.maps[m])?, + 16 => { + // `.paths` round trip, then decoding a damaged copy, which must fail + // cleanly or decode something, not panic. + let p = short_path(d)?; + let mut buf = Vec::new(); + let _ = serialize_paths(st.maps[m].read_zipper_at_path(&p), &mut buf); + let to = d.modn(NMAPS)?; + let q = short_path(d)?; + let v = val(d)?; + if st.small(to) { + let _ = deserialize_paths(st.maps[to].write_zipper_at_path(&q), Cursor::new(&buf[..]), v); + } + let flips = d.modn(4)?; + let cut = d.u8()? as usize; + let mut bad = buf.clone(); + for _ in 0..flips { + if bad.is_empty() { break } + let i = d.u8()? as usize * 7 % bad.len(); + bad[i] ^= d.u8()? | 1; + } + if cut & 1 == 1 { bad.truncate(cut % (bad.len() + 1)); } + decode_bounded(&bad); + } + 17 => { + // Decoding arbitrary bytes from the input. + let len = d.modn(64)?; + let raw: Vec = (0..len).map(|_| d.u8()).collect::>()?; + decode_bounded(&raw); + } + 18 => { + let p = path(d)?; + let map = &st.maps[m]; + let _ = (map.is_shared(), map.shared_node_id(), map.val().cloned(), map.val_at(&p).cloned(), map.goat_val_count()); + } + _ => { + // A clone mutated while the original is read: copy-on-write. + let mut c = st.maps[m].clone(); + let p = path(d)?; + let v = val(d)?; + if st.small(m) { + let _ = c.set_val_at(&p, v); + let q = path(d)?; + let _ = c.remove_branches_at(&q, d.boolean()?); + } + let _ = (st.maps[m].val_count(), c.val_count()); + } + } + Some(()) +} + +fn decode_bounded(bytes: &[u8]) { + let mut n = 0usize; + let _ = for_each_deserialized_path(Cursor::new(bytes), |_, _| { + n += 1; + if n > MAX_VALS { Err(std::io::Error::other("enough")) } else { Ok(()) } + }); +} + +/// A trie built by an anamorphism. `W` is the remaining depth; the children +/// pushed at each step come from the input, sorted and de-duplicated by first +/// byte as `TrieBuilder` requires, and sometimes grafted from one of the maps. +fn ana(d: &mut Dec, maps: &[PathMap; NMAPS]) -> Option> { + let depth = d.modn(5)?; + let spec: Vec = (0..32).map(|_| d.u8()).collect::>()?; + let graft_from = maps[d.modn(NMAPS)?].clone(); + let mut calls = 0usize; + Some(PathMap::::new_from_ana(depth, |left: usize, v: &mut Option, children, path: &[u8]| { + calls += 1; + let h = spec[(calls + path.len()) % spec.len()]; + if h & 1 == 1 { + *v = Some(V::from_byte(h)); + } + if left == 0 || calls > 256 { + return; + } + let fan = (h >> 1) as usize % 4; + let mut firsts: Vec<(u8, Vec)> = (0..fan) + .map(|i| { + let s = spec[(calls * 7 + i * 3) % spec.len()]; + let first = if s & 0x80 == 0 { s % 4 } else { s }; + let tail = (0..(s as usize % 3)).map(|j| spec[(i + j + calls) % spec.len()] % 4).collect(); + (first, tail) + }) + .collect(); + firsts.sort_by_key(|(b, _)| *b); + firsts.dedup_by_key(|(b, _)| *b); + for (i, (first, tail)) in firsts.into_iter().enumerate() { + let rz = graft_from.read_zipper_at_path([first]); + if (h >> 4) as usize % 4 == i && rz.child_count() > 0 { + children.graft_at_byte(first, &rz); + } else { + let mut sub = vec![first]; + sub.extend(tail); + children.push(&sub, left - 1); + } + } + })) +} + +/// The four catamorphism flavours, some stopping early with an error, and `hash`. +fn cata(d: &mut Dec, map: &PathMap) -> Option<()> { + let p = short_path(d)?; + let stop = d.modn(32)?; + let z = map.read_zipper_at_path(&p); + match d.modn(9)? { + 0 => { let _: usize = z.into_cata_side_effect(|_m, ch: &mut [usize], v: Option<&V>, path: &[u8]| ch.iter().sum::() + v.is_some() as usize + path.len()); } + 1 => { let _: usize = z.into_cata_jumping_side_effect(|_m, ch: &mut [usize], jump, v: Option<&V>, _p: &[u8]| ch.iter().sum::() + jump + v.is_some() as usize); } + 2 => { let _: usize = z.into_cata_cached(|_m, ch: &mut [usize], v: Option<&V>| ch.iter().sum::() + v.map_or(0, |v| v.to_u64() as usize)); } + 3 => { let _: usize = z.into_cata_jumping_cached(|_m, ch: &mut [usize], v: Option<&V>, sub: &[u8]| ch.iter().sum::() + sub.len() + v.is_some() as usize); } + 4 => { + let mut n = 0; + let _: Result = z.into_cata_side_effect_fallible(|_m, ch: &mut [usize], _v: Option<&V>, _p: &[u8]| { n += 1; if n > stop { Err(()) } else { Ok(ch.len()) } }); + } + 5 => { + let mut n = 0; + let _: Result = z.into_cata_jumping_side_effect_fallible(|_m, ch: &mut [usize], j, _v: Option<&V>, _p: &[u8]| { n += 1; if n > stop { Err(()) } else { Ok(ch.len() + j) } }); + } + 6 => { let _: Result = z.into_cata_cached_fallible(|m: &ByteMask, ch: &mut [usize], _v: Option<&V>| if m.count_bits() > stop % 4 { Err(()) } else { Ok(ch.len()) }); } + 7 => { let _: Result = z.into_cata_jumping_cached_fallible(|_m, ch: &mut [usize], _v: Option<&V>, sub: &[u8]| if sub.len() > stop { Err(()) } else { Ok(ch.len()) }); } + _ => { let _ = z.hash(); } + } + Some(()) +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +fn seed(d: &mut Dec) -> Option> { + let mut st = State { maps: Default::default(), steps: 0 }; + for m in 0..NMAPS { + for _ in 0..d.modn(12)? { + let p = path(d)?; + let v = val(d)?; + st.maps[m].set_val_at(&p, v); + } + } + Some(st) +} + +fn run_typed(d: &mut Dec) { + let Some(mut st) = seed::(d) else { return }; + let _ = (|| -> Option<()> { + loop { + tick(&mut st.steps)?; + match d.modn(8)? { + 0 | 1 | 2 => map_op(d, &mut st)?, + 3 | 4 => read_episode(d, &mut st)?, + _ => write_episode(d, &mut st)?, + } + } + })(); + // Tear down in a varied order: dropping shared nodes is part of the surface. + let order = d.u8().unwrap_or(0) as usize; + for i in 0..NMAPS { + let m = (order + i) % NMAPS; + let _ = st.maps[m].val_count(); + st.maps[m] = PathMap::new(); + } +} + +/// Run one input. Returns normally unless the crate panics, aborts or hangs. +/// +/// Anything this reaches with `debug_assertions` off is also reachable with +/// them on; build with them on to turn internal invariant checks into failures. +pub fn run(bytes: &[u8]) { + let mut d = Dec { bytes, pos: 0 }; + match d.u8().map(|b| b % 4) { + None => {} + Some(0) | Some(1) => run_typed::(&mut d), + Some(2) => run_typed::<()>(&mut d), + Some(_) => { + if d.boolean().unwrap_or(false) { run_typed::(&mut d) } else { run_typed::(&mut d) } + } + } +} diff --git a/differential/src/lib.rs b/differential/src/lib.rs index 436335e7..72f3c0a7 100644 --- a/differential/src/lib.rs +++ b/differential/src/lib.rs @@ -7,6 +7,8 @@ //! table are a contract shared with `lean/PathMapModel/Fuzz.lean`. //! * [`server`] is the resident-process protocol the driver speaks. //! * [`repro`] turns an input back into standalone `pathmap` calls. +//! * [`source`] generates random inputs by index, for the in-process front ends. +//! * [`crash`] is a second op table that only has to not crash; see `bin/crash_fuzz.rs`. //! * [`act`] is the `ArenaCompactTree` read source behind `act_trace`. //! * [`reference`] is a second executable model: a Rust transcription of the //! same Lean specification, sharing no code with `pathmap`. It is what @@ -18,6 +20,8 @@ pub mod harness; pub mod reference; pub mod repro; pub mod server; +pub mod source; +pub mod crash; pub use act::*; pub use harness::*; diff --git a/differential/src/source.rs b/differential/src/source.rs new file mode 100644 index 00000000..ab0a16ab --- /dev/null +++ b/differential/src/source.rs @@ -0,0 +1,56 @@ +//! Where fuzzer inputs come from, shared by `in_process` and `crash_fuzz`. + +/// Where inputs come from. +/// +/// `get` is deterministic in `idx` and holds no state between calls, so which +/// thread runs an input cannot change it and a failing input can be re-derived +/// from its index alone. Mirrors `InputSource` in `lean/differential.py`; a +/// queue-backed source drops in here the same way. +pub enum Source { + Random { seed: u64, count: usize, maxlen: usize }, + Files(Vec), +} + +impl Source { + pub fn len(&self) -> usize { + match self { + Source::Random { count, .. } => *count, + Source::Files(v) => v.len(), + } + } + + pub fn name(&self, idx: usize) -> String { + match self { + Source::Random { .. } => format!("random#{idx:06}"), + Source::Files(v) => v[idx].clone(), + } + } + + pub fn get(&self, idx: usize) -> Vec { + match *self { + Source::Random { seed, maxlen, .. } => { + // splitmix64, seeded per index so generation parallelises without + // changing what gets tested. + let mut s = seed.wrapping_mul(0x9E3779B97F4A7C15) + ^ (idx as u64).wrapping_mul(0xBF58476D1CE4E5B9); + let mut next = || { + s = s.wrapping_add(0x9E3779B97F4A7C15); + let mut z = s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + }; + // The same *distribution* as `RandomInputs.get` in + // `lean/differential.py` -- `randrange(8, maxlen)`, i.e. uniform + // on `[8, maxlen)` -- so a divergence rate measured here is + // directly comparable to one measured there. The bit stream + // differs (splitmix64 against Python's Mersenne Twister) and is + // meant to: two independent samples of the same population. + let span = maxlen.saturating_sub(8).max(1); + let n = 8 + (next() as usize) % span; + (0..n).map(|_| next() as u8).collect() + } + Source::Files(ref v) => std::fs::read(&v[idx]).expect("cannot read input"), + } + } +} From 10a6332ee694d87d3303282d0d00a5d44150a91c Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 16 Sep 2026 21:52:46 +0000 Subject: [PATCH 45/73] Write down the crash-only fuzzer's findings CRASH_FINDINGS.md lists the 29 failure sites from two 50k-input surveys (debug assertions and release) and a 10k-input all_dense_nodes survey: what each one is, how often it was hit, the public call that reaches it, and either a minimal Rust reproducer or a (seed, index) input to replay. Three are undefined behaviour reachable from safe code (ZipperHeadOwned exclusive paths, as_dense_unchecked in PathMap::join, heap corruption after ZipperHead writes); the rest are panics, failed debug assertions and hangs, plus the known failures the table steers around. crash_repros compiles the eight minimal reproducers, one selectable case each. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/CRASH_FINDINGS.md | 360 +++++++++++++++++++++++++++ differential/src/bin/crash_repros.rs | 109 ++++++++ 2 files changed, 469 insertions(+) create mode 100644 differential/CRASH_FINDINGS.md create mode 100644 differential/src/bin/crash_repros.rs diff --git a/differential/CRASH_FINDINGS.md b/differential/CRASH_FINDINGS.md new file mode 100644 index 00000000..32db7e93 --- /dev/null +++ b/differential/CRASH_FINDINGS.md @@ -0,0 +1,360 @@ +# Crash-only fuzzing findings + +Failures `crash_fuzz` found in `pathmap` on 2026-09-16, at commit `7c61024` on +`fuzz-fixes-v2`. The op table is `differential/src/crash.rs`; see +`differential/src/bin/crash_fuzz.rs` for the runner. Nothing here has been +fixed. + +Each finding is an input that makes the crate panic, fail a debug assertion, +abort, segfault or hang **through its public API**, with every documented +precondition met. Calls the crate documents as panicking, stubs, and failures +already known from the differential work are steered around unless +`--include-known` is passed; they are listed at the end. + +## How the surveys were run + +| Survey | Build | Inputs | Seed | Time | Failures | Sites | +|---|---|---|---|---|---|---| +| A | debug assertions + overflow checks | 50,000 | 12 | 222 s | 7,165 | 27 | +| B | release | 50,000 | 11 | 134 s | 2,043 | 17 | +| C | release, `--features pathmap/all_dense_nodes` | 10,000 | 13 | 62 s | 845 | 11 | + +```sh +# A +CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS=true CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS=true \ + CARGO_TARGET_DIR=target/dbgassert cargo build --release -p differential --bin crash_fuzz +target/dbgassert/release/crash_fuzz --random 50000 --seed 12 -j 56 --timeout 8 \ + --keep-going --max-failures 20000 --save runs/crash-a +# B +cargo build --release -p differential --bin crash_fuzz +target/release/crash_fuzz --random 50000 --seed 11 -j 56 --timeout 5 --keep-going --save runs/crash-b +# C +CARGO_TARGET_DIR=target/alldense cargo build --release -p differential --bin crash_fuzz \ + --features pathmap/all_dense_nodes +target/alldense/release/crash_fuzz --random 10000 --seed 13 -j 56 --timeout 5 --keep-going +``` + +Survey B's release build was taken before the last `KNOWN_FAILURES` entry +(stand-alone `to_next_k_path`) existed. The site that entry covers is listed +under "Known failures" below. + +**Reproducing a fuzz input.** Inputs are deterministic in (seed, index), so +"A #3620" means: + +```sh +target/dbgassert/release/crash_fuzz --random 50000 --seed 12 --dump 3620 > in.bin +CRASH_TRACE=1 CRASH_BACKTRACE=1 target/dbgassert/release/crash_fuzz in.bin +``` + +`CRASH_TRACE=1` prints every op as it starts, so the last lines before the +failure name the call. Indices are only valid for the op table at `7c61024`. +Replay a survey A input with the debug build and a B input with the release +build; the other build may fail at a different site or not at all. + +**Minimal reproducers** for eight of the findings are compiled in +`differential/src/bin/crash_repros.rs`: + +```sh +cargo run -p differential --bin crash_repros -- --list +cargo run -p differential --bin crash_repros -- +``` + +## Summary + +Counts are hits in survey A (debug) / survey B (release); `-` means none in +that survey. Hangs are counted separately at the end. + +| # | Site | A | B | Reached through | Repro | +|---|---|---|---|---|---| +| 1 | `zipper.rs:3616` unreachable_unchecked / SIGSEGV | 33 | 23 | `ZipperHeadOwned::write_zipper_at_exclusive_path` | `owned_head_second_exclusive_path` | +| 2 | `trie_node.rs:1021` unreachable_unchecked / SIGILL | 7 | 4 | `PathMap::join`, `WriteZipper::join_map_into` | A #3620, B #6905 | +| 3 | `malloc(): unaligned tcache chunk detected` (SIGABRT) | 1 | - | `ZipperHead`, then a read zipper | A #8814 | +| 4 | `trie_node.rs:3213` misaligned pointer dereference | 1 (earlier run) | - | `ZipperHead::write_zipper_at_exclusive_path` | - | +| 5 | `write_zipper.rs:2869` slice out of range | 176 | 128 | `ZipperHeadOwned::write_zipper_at_exclusive_path` | A #464 | +| 6 | `zipper_head.rs:347` unwrap on `None` | 24 | 27 | `write_zipper_at_exclusive_path` on a head from `WriteZipper::zipper_head` or `into_zipper_head` | `write_zipper_head_second_exclusive_path` | +| 7 | `write_zipper.rs:1337` assertion `origin_path` not a slice | 1,593 | - | `WriteZipper::zipper_head` on a zipper made by `write_zipper_at_path` | `write_zipper_head_second_exclusive_path` (debug) | +| 8 | `trie_node.rs:3209` make_unique on an empty sentinel | 1 | 16 | `ZipperHead::write_zipper_at_exclusive_path` | A #36325 | +| 9 | `write_zipper.rs:1284` assertion `at_root` | 4 | - | `ZipperHead::write_zipper_at_exclusive_path` | A #69 | +| 10 | `write_zipper.rs:1435` unwrap on `None` | - | 2 | `set_val` on a zipper from a nested `ZipperHead` | B #8504 | +| 11 | `write_zipper.rs:1453` unwrap on `None` | - | 7 | `remove_val` (via `deserialize_paths`) on a zipper from a nested `ZipperHead` | B #5364 | +| 12 | `zipper.rs:1871` explicit panic | 617 | 341 | `get_trie_ref` on a read zipper from a `ZipperHead` | `head_read_zipper_get_trie_ref` | +| 13 | `zipper.rs:2644` unwrap on `None` | 126 | 57 | `ProductZipper::is_shared` | `product_zipper_is_shared` | +| 14 | `zipper.rs:3048` unwrap on `None` | 3 | 3 | `ProductZipper::val_count` | `product_zipper_val_count` | +| 15 | `product_zipper.rs:178` assertion `focus_factor() == factor_count() - 1` | 77 | - | `ProductZipper::val_count` | `product_zipper_val_count` (debug) | +| 16 | `product_zipper.rs:502` assertion "must ascend" | 1,148 | - | `ProductZipperG` sibling moves, `to_next_val`, k-path walks | A #31, A #8704 | +| 17 | `dependent_zipper.rs:166` assertion "must ascend" | 1,163 | - | `DependentProductZipperG` sibling moves, `to_next_val`, k-path walks | A #16 | +| 18 | `overlay_zipper.rs:163` assertion (`focus_byte` of the two sides) | 884 | - | `OverlayZipper::to_next_step`, `to_next_step_observed` | A #17 | +| 19 | `overlay_zipper.rs:152` assertion (`depth` of the two sides) | 2 | - | `OverlayZipper` | A #9317 | +| 20 | `overlay_zipper.rs:330` assertion (`path` of the two sides) | 1 | - | `OverlayZipper::ascend_until` | A #21316 | +| 21 | `zipper.rs:557` subtract with overflow | 12 | - | `OverlayZipper::to_next_step_observed` | A #269 | +| 22 | `prefix_zipper.rs:448` assertion `path_exists()` | 64 | - | `PrefixZipper::descend_indexed_byte`, `descend_last_byte`, `to_next_step_observed` | A #400 | +| 23 | `prefix_zipper.rs:355` index out of bounds | 1 | 3 | `PrefixZipper::to_next_step`, `descend_indexed_byte` | A #46237, B #3769 | +| 24 | `write_zipper.rs:2557` / `2560` slice out of range | 32 | 20 | `graft_child_maps` under a root path of 48+ bytes | `graft_child_maps_long_root` | +| 25 | `zipper.rs:3328` assertion / `zipper.rs:3332` / `3330` slice out of range | 12 | 10 | `get_val_with_witness` on a `ReadZipperOwned` | `owned_read_zipper_witness` | +| 26 | `write_zipper.rs:1304` assertion `at_root` | 1 | - | `join_k_path_into` | A #22746 | +| 27 | `line_list_node.rs:1818` unwrap on `None` | 2 | - | `PathMap::merkleize` | A #4250, A #45491 | +| 28 | `trie_ref.rs:171` subtract with overflow | 2 | - | `trie_ref_at_path`, `get_focus_at` | A #17556, A #49829 | +| 29 | `write_zipper.rs:1132` slice out of range | - | - | `WriteZipperUntracked::path`, only with `all_dense_nodes` (survey C #3135) | C #3135 | + +## Undefined behaviour from safe code + +### 1. `ZipperHeadOwned` rooted below the map root, second exclusive path + +```rust +let zh = sample().into_zipper_head(&[1u8]); +drop(zh.write_zipper_at_exclusive_path(&[])); +let _ = zh.write_zipper_at_exclusive_path(&[9u8]); +``` + +Release: SIGSEGV. Debug: `unsafe precondition(s) violated: +hint::unreachable_unchecked must never be reached` at `zipper.rs:3616`. The +first exclusive zipper at the head's own root, once dropped, leaves the head in +a state the next request walks off. Sites 5 and 6 are panics from the same +function (`prepare_exclusive_write_path`), reached from differing states; +`write_zipper.rs:2869` is `KeyFields::root_prefix_path` slicing past the end of +the prefix buffer. (`sample()` is the map `{[0], [1,2,1], [1,2,1,0], +[1,2,1,3,3], [2,2]}`, all values 7.) + +### 2. `as_dense_unchecked` on a node that is not dense + +Reached from `PathMap::join` and from `WriteZipper::join_map_into` (A #3620, +A #17025, B #6905). `TaggedNodeRef::as_dense_unchecked` hits +`unreachable_unchecked` (`trie_node.rs:1021`); a release build executes it as +an illegal instruction (SIGILL). In A #3620 the destination zipper is at a +path below a map built by earlier writes; no minimal repro yet. The most likely +candidate is a node type the join dispatch assumes cannot occur there (e.g. a +`CellByteNode`, which `ZipperHead` creates), but that is not confirmed. + +### 3. Heap corruption after `ZipperHead` writes + +A #8814: a `ZipperHead` over a map, an exclusive write zipper doing +`join_k_path_into` and `subtract_into`, then a read zipper from the same head +calling `descend_last_path`. The process aborts with `malloc(): unaligned +tcache chunk detected`. Reproduces alone, in both builds (release: SIGABRT with the same message). + +### 4. Misaligned pointer dereference + +`trie_node.rs:3213`: `misaligned pointer dereference: address must be a +multiple of 0x4 but is 0xff19cac196ee4b9`, one hit in an earlier debug survey +(seed 11, #8513) against an older revision of the op table, in the same code +as site 8 (`make_unique`), from `ZipperHead::write_zipper_at_exclusive_path`. +That index no longer reproduces at `7c61024`, and survey A did not hit it. + +## Panics with a minimal reproducer + +### 6, 7. `zipper_head()` on a write zipper made with a borrowed path + +```rust +let mut map = sample(); +let mut wz = map.write_zipper_at_path(&[1u8]); +let zh = wz.zipper_head(); +drop(zh.write_zipper_at_exclusive_path(&[])); +let _ = zh.write_zipper_at_exclusive_path(&[]); +``` + +Debug fails at once, in `zipper_head()`: +`!self.key.origin_path.is_slice() || self.key.origin_path.len() == 0` +(`write_zipper.rs:1337`, `as_static_path_zipper`). `write_zipper_at_path` takes +the path by reference, so any non-empty root trips it. Release carries on +and panics on the second exclusive path at `zipper_head.rs:347` (`root_val` +unwrap in `prepare_exclusive_write_path`). Site 7 is the most frequent +failure in survey A. + +### 12. `get_trie_ref` on a read zipper from a `ZipperHead` + +```rust +let mut map = sample(); +let zh = map.zipper_head(); +let rz = zh.read_zipper_at_path(&[1u8]).unwrap(); +let _ = rz.get_trie_ref(); +``` + +`focus_parent_borrowed` calls `OwnedOrBorrowed::as_borrowed_ref`, which panics +on the owned root node a head's read zipper holds (`zipper.rs:1871`). The same +path is reached from `get_focus_at` on those zippers. Most frequent panic in +survey B. + +### 13. `ProductZipper::is_shared` + +```rust +let (a, b) = (sample(), sample()); +let mut z = ProductZipper::new(a.read_zipper_at_path(&[1u8, 2, 1]), [b.read_zipper()]); +while z.to_next_step() { + let _ = z.is_shared(); +} +``` + +Once the focus is in the second factor, `ReadZipperCore::is_shared` looks the +focus up in the parent node of the *primary* trie and unwraps `None` +(`zipper.rs:2644`). + +### 14, 15. `ProductZipper::val_count` outside the last factor + +```rust +let mut a = PathMap::::new(); +a.set_val_at([1u8, 2], 1); +let mut b = PathMap::::new(); +b.set_val_at([3u8, 4], 1); +let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper()]); +loop { + let _ = z.val_count(); + if !z.to_next_step() { break } +} +``` + +Debug: `focus_factor() == factor_count() - 1` (`product_zipper.rs:178`), i.e. +`val_count` is only implemented for the last factor. Release: +`get_focus_at` unwraps `None` (`zipper.rs:3048`). + +### 24. `graft_child_maps` under a long root path + +```rust +let mut map = PathMap::::new(); +let mut wz = map.write_zipper_at_path(&[0u8; 48]); +wz.graft_child_maps(ByteMask::from_iter([1u8]), [PathMap::single([2u8], 5)], false); +``` + +`range end index 49 out of range for slice of length 48` +(`write_zipper.rs:2560`; with a longer root, `2557`). Roots of 47 bytes or +fewer work. A fixed 48-byte buffer is being indexed by the full path length. + +### 25. `get_val_with_witness` on an owned read zipper + +```rust +let mut map = PathMap::::new(); +map.set_val_at([1u8], 1); +let z = map.into_read_zipper(&[]); +let w = z.witness(); +let _ = z.get_val_with_witness(&w); +``` + +Debug: `root_parent_key_start < usize::MAX` (`zipper.rs:3328`). Release: +`range start index 18446744073709551615 out of range` (`zipper.rs:3332`, and +`3330` in another state). The root path here is empty; a 3-byte root path +returns `None` without panicking. + +### 23 (and a k = 0 case). `PrefixZipper` + +```rust +let mut map = PathMap::::new(); +map.set_val_at([0x22u8], 1); +let mut z = PrefixZipper::new(&[2u8, 3][..], map.read_zipper()); +z.descend_to_byte(2); +z.descend_last_path(); +z.descend_last_path(); +z.descend_first_k_path(0); +``` + +`prefix_zipper.rs:571`, slice out of range: `descend_first_k_path` computes the +untaken part of the prefix as if the focus were still inside it, although the +zipper has moved into the source. +`k = 0` is in `KNOWN_FAILURES` (degenerate) and is skipped by default, but a +slice panic is not a degenerate answer. Related, without minimal repros: +site 22, `descend_indexed_byte` / `descend_last_byte` landing on a path that +does not exist (`prefix_zipper.rs:448`, A #400); site 23, index out of bounds in +`to_next_step` / `descend_indexed_byte` (`prefix_zipper.rs:355`, B #3769: "the +len is 0 but the index is 0"). + +## Findings with fuzz inputs only + +### 16, 17. `ProductZipperG` and `DependentProductZipperG`: "must ascend" + +`to_sibling_byte` gets `Some` from `focus_byte()` and then `ascend(1)` returns +0 (`product_zipper.rs:502`, `dependent_zipper.rs:166`). Reached from +`to_next_sibling_byte`, `to_prev_sibling_byte`, `to_next_step`, `to_next_val` +and k-path walks (A #31, #8704; A #16). In release the assertion is gone, and +these zippers hang instead: most hangs in survey B are in these two types. + +### 18–21. `OverlayZipper`: the two sides drift apart + +The overlay moves both source zippers in step and asserts they agree. They +stop agreeing on `focus_byte` (site 18, A #17), `depth` (19, A #9317) and +`path` (20, A #21316: `[0, 0, 2, 0, 1]` against `[2, 0, 1]`). From there, +`to_next_step_observed` reports more ascent to the observer than it descended +(21, A #269: `Vec` observer underflow at `zipper.rs:557`, with the observer +started from the zipper's current path), and in release `to_next_step` hangs. +Not diagnosed; a move that succeeds on one source and not the other is the +obvious suspect. + +### 8, 9. `ZipperHead` exclusive paths + +- Site 8, `Attempted to make_unique on an empty sentinel node` + (`trie_node.rs:3209`, from `make_cell_node` in `prepare_exclusive_write_path`): + A #36325, B #1730. Seen after an exclusive zipper did `take_map`, + `graft_child_maps` or `set_val` at a path and was dropped. A direct attempt + (`take_map` then a new exclusive zipper at the same path) did not reproduce. +- Site 9, assertion `self.at_root()` (`write_zipper.rs:1284`): A #69. + +### 10, 11. Writes through a zipper from a nested `ZipperHead` + +A head made by `WriteZipper::zipper_head()` on a write zipper at a long root, +then `set_val` (`write_zipper.rs:1435`, B #8504) or `remove_val` inside +`deserialize_paths` (`write_zipper.rs:1453`, B #5364) on a zipper it hands out. +Both unwrap `None`. Release only: in a debug build these inputs stop earlier +at site 7. + +### 26. `join_k_path_into`: assertion `at_root` + +`write_zipper.rs:1304`, A #22746. + +### 27. `PathMap::merkleize` + +`line_list_node.rs:1818`, `node_replace_child` unwraps `None` from +`get_child_mut`: A #4250 (`PathMap`), A #45491 (`PathMap`). +The list node cannot find the child `merkleize` asks it to replace. + +### 28. `trie_ref_at_path` / `get_focus_at`: subtract with overflow + +`trie_ref.rs:171`, A #17556, A #49829. Release wraps silently. + +### 29. `all_dense_nodes`: `WriteZipperUntracked::path` + +`write_zipper.rs:1132`, `range start index 3 out of range for slice of length +2`, survey C #3135 only. The default build passes this input. + +## Hangs + +| Survey | Hangs | Where (last op before the timeout) | +|---|---|---| +| A (debug) | 1,178 | as below; many B hangs fail an assertion (16–21) in A instead | +| B (release) | 1,392 | classified on a 3,000-input sample: `ProductZipperG` / `DependentProductZipperG` sibling moves and k-path walks; `OverlayZipper::to_next_step`; `ProductZipper` k-path walks (k ≥ 1) | +| C (all_dense) | 581 | same kinds | + +Hangs are infinite rather than slow: ten sampled inputs were all still running +after 40 s single-threaded, and one after 5 minutes, where a `to_next_step` or a `k ≤ 5` k-path step +should take microseconds. The simple cases (two small maps, a whole +`to_next_step` or k-path walk over a product or overlay zipper) do not hang; +the hanging inputs move the zipper first. Shrunk inputs are in +`.fuzzcorpus/crash/shrunk-h*.min.bin` locally; they were not minimised to Rust. + +## Not reproduced + +- A #30871: SIGABRT in the survey, runs clean alone (3 tries). Possibly + caused by another thread's failure in the same process. + +## Known failures (steered around by default) + +These are `KNOWN_FAILURES` in `differential/src/crash.rs`. `--include-known` +runs them. + +- `val_count` is `todo!()` on `OverlayZipper` (`overlay_zipper.rs:173`) and + `unimplemented!()` on `ProductZipperG` (`product_zipper.rs:699`) and + `DependentProductZipperG` (`dependent_zipper.rs:360`). +- `TrieRef::fork_read_zipper` at a missing path unwraps `None` + (`trie_ref.rs:322`; commented upstream as issue #96). +- `k == 0` for k-path iteration, `join_k_path_into` and `drop_head`: spins on + several zipper kinds, trips `debug_assert!(byte_cnt > 0)` in + `LineListNode::drop_head_dyn`, and panics in `PrefixZipper` (above). +- `to_next_k_path` without a preceding `descend_first_k_path`: underflows + `path_len` (`zipper.rs:2841`, overflow check) and reaches + `unreachable!()` in `EmptyNode` from `k_path_internal` on a fork of a write + zipper (`empty_node.rs:70`, survey B, 9 hits). +- `meet_k_path_into` spins when the focus has no children, and escapes it + when `k == 0`. + +## Not covered by the crash table + +`viz`, `old_cursor`, `bridge_nodes`, ACT file operations +(`dump_from_zipper`, `open_mmap`, `merge_zipper_into_file`), a `ZipperHead` +shared between threads, custom allocators, `PolyZipper`, `SplitCata` and +`counters`. diff --git a/differential/src/bin/crash_repros.rs b/differential/src/bin/crash_repros.rs new file mode 100644 index 00000000..3e12b82d --- /dev/null +++ b/differential/src/bin/crash_repros.rs @@ -0,0 +1,109 @@ +//! Minimal reproducers for failures `crash_fuzz` turned up. See +//! `differential/CRASH_FINDINGS.md` for the write-up. +//! +//! cargo run -p differential --bin crash_repros -- --list +//! cargo run -p differential --bin crash_repros -- +//! +//! Each case ends the process (panic, abort or segfault), so run them one at a +//! time. Where a debug build fails earlier on an assertion, the case says so. + +use pathmap::PathMap; +use pathmap::utils::ByteMask; +use pathmap::zipper::*; + +fn sample() -> PathMap { + let mut m = PathMap::new(); + for p in [&[1u8, 2, 1][..], &[1, 2, 1, 0], &[1, 2, 1, 3, 3], &[0], &[2, 2]] { + m.set_val_at(p, 7); + } + m +} + +fn owned_head_second_exclusive_path() { + let zh = sample().into_zipper_head(&[1u8]); + drop(zh.write_zipper_at_exclusive_path(&[])); + let _ = zh.write_zipper_at_exclusive_path(&[9u8]); // UB: segfault / unreachable_unchecked +} +fn write_zipper_head_second_exclusive_path() { + let mut map = sample(); + let mut wz = map.write_zipper_at_path(&[1u8]); + let zh = wz.zipper_head(); + drop(zh.write_zipper_at_exclusive_path(&[])); + let _ = zh.write_zipper_at_exclusive_path(&[]); // zipper_head.rs:347 +} +fn head_read_zipper_get_trie_ref() { + let mut map = sample(); + let zh = map.zipper_head(); + let rz = zh.read_zipper_at_path(&[1u8]).unwrap(); + let _ = rz.get_trie_ref(); // zipper.rs:1871 +} +fn product_zipper_is_shared() { + let (a, b) = (sample(), sample()); + let mut z = ProductZipper::new(a.read_zipper_at_path(&[1u8, 2, 1]), [b.read_zipper()]); + while z.to_next_step() { + let _ = z.is_shared(); // zipper.rs:2644 once the focus is in the second factor + } +} +fn product_zipper_val_count() { + let mut a = PathMap::::new(); + a.set_val_at([1u8, 2], 1); + let mut b = PathMap::::new(); + b.set_val_at([3u8, 4], 1); + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper()]); + loop { + let _ = z.val_count(); // zipper.rs:3048 (debug: product_zipper.rs:178) + if !z.to_next_step() { break } + } +} +fn graft_child_maps_long_root() { + let mut map = PathMap::::new(); + let mut wz = map.write_zipper_at_path(&[0u8; 48]); + wz.graft_child_maps(ByteMask::from_iter([1u8]), [PathMap::single([2u8], 5)], false); // write_zipper.rs:2557/2560 +} +fn owned_read_zipper_witness() { + let mut map = PathMap::::new(); + map.set_val_at([1u8], 1); + let z = map.into_read_zipper(&[]); + let w = z.witness(); + let _ = z.get_val_with_witness(&w); // zipper.rs:3332 (debug: zipper.rs:3328) +} +fn prefix_zipper_k0_after_last_path() { + let mut map = PathMap::::new(); + map.set_val_at([0x22u8], 1); + let mut z = PrefixZipper::new(&[2u8, 3][..], map.read_zipper()); + z.descend_to_byte(2); + z.descend_last_path(); + z.descend_last_path(); + z.descend_first_k_path(0); // prefix_zipper.rs:571 +} + +const CASES: &[(&str, fn(), &str)] = &[ + ("owned_head_second_exclusive_path", owned_head_second_exclusive_path, "undefined behaviour: segfault in release, unreachable_unchecked in debug"), + ("write_zipper_head_second_exclusive_path", write_zipper_head_second_exclusive_path, "zipper_head.rs:347 unwrap (debug: write_zipper.rs:1337 assertion)"), + ("head_read_zipper_get_trie_ref", head_read_zipper_get_trie_ref, "zipper.rs:1871 explicit panic"), + ("product_zipper_is_shared", product_zipper_is_shared, "zipper.rs:2644 unwrap"), + ("product_zipper_val_count", product_zipper_val_count, "zipper.rs:3048 unwrap (debug: product_zipper.rs:178 assertion)"), + ("graft_child_maps_long_root", graft_child_maps_long_root, "write_zipper.rs:2560 slice out of range"), + ("owned_read_zipper_witness", owned_read_zipper_witness, "zipper.rs:3332 slice out of range (debug: zipper.rs:3328 assertion)"), + ("prefix_zipper_k0_after_last_path", prefix_zipper_k0_after_last_path, "prefix_zipper.rs:571 slice out of range"), +]; + +fn main() { + let arg = std::env::args().nth(1).unwrap_or_else(|| "--list".into()); + if arg == "--list" { + for (name, _, what) in CASES { + println!("{name:40} {what}"); + } + return; + } + match CASES.iter().find(|(name, _, _)| *name == arg) { + Some((_, case, _)) => { + case(); + println!("{arg}: no failure (fixed?)"); + } + None => { + eprintln!("unknown case {arg}; see --list"); + std::process::exit(2); + } + } +} From 00b51748e9d42ac7166addb070993a0aa219f310 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:31:13 +0000 Subject: [PATCH 46/73] Fix ZipperHead exclusive path at the head's own root With an empty path and the head's root inside a node, the walk popped a byte of the head's own path and never put it back. The next request then read past its key buffer (UB in release) or panicked. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/zipper_head.rs | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/zipper_head.rs b/src/zipper_head.rs index 1ab41673..d66b56b0 100644 --- a/src/zipper_head.rs +++ b/src/zipper_head.rs @@ -368,7 +368,11 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + let cell_node = end_node.make_mut().into_cell_node().unwrap(); let (exclusive_node, val) = cell_node.prepare_cf(last_path_byte); + //With an empty `path`, the popped byte was the zipper's own; put it back z.key.prefix_buf.truncate(original_path_len); + if z.key.prefix_buf.len() < original_path_len { + z.key.prefix_buf.push(last_path_byte); + } return (exclusive_node, val) }, @@ -409,6 +413,9 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + } else { //CASE 4 z.key.prefix_buf.truncate(original_path_len); + if z.key.prefix_buf.len() < original_path_len { + z.key.prefix_buf.push(last_path_byte); + } //If the node on top of the stack is not a cell node, we need to upgrade it if !z.focus_stack.top().unwrap().is_cell_node() { @@ -1494,4 +1501,42 @@ mod tests { assert_eq!(rz.is_val(), false); drop(rz); } + + /// An exclusive zipper at the head's own root, requested more than once, from a head whose + /// root sits partway into a node + #[test] + fn exclusive_path_at_head_root_twice() { + let sample = || { + let mut m = PathMap::::new(); + for p in [&[1u8, 2, 1][..], &[1, 2, 1, 0], &[1, 2, 1, 3, 3], &[0], &[2, 2]] { m.set_val_at(p, 7); } + m + }; + + let zh = sample().into_zipper_head(&[1u8]); + for (path, v) in [(&[][..], 1), (&[9u8][..], 2), (&[][..], 3)] { + let mut wz = zh.write_zipper_at_exclusive_path(path).unwrap(); + wz.descend_to(&[5u8]); + wz.set_val(v); + } + let map = zh.into_map(); + assert_eq!(map.get_val_at(&[1u8, 5]), Some(&3)); + assert_eq!(map.get_val_at(&[1u8, 9, 5]), Some(&2)); + assert_eq!(map.get_val_at(&[1u8, 2, 1, 0]), Some(&7)); + assert_eq!(map.val_count(), 7); + + let mut map = sample(); + { + let mut wz = map.write_zipper(); + wz.descend_to(&[1u8]); + let zh = wz.zipper_head(); + for (path, v) in [(&[][..], 1), (&[][..], 2), (&[9u8][..], 3)] { + let mut child = zh.write_zipper_at_exclusive_path(path).unwrap(); + child.descend_to(&[5u8]); + child.set_val(v); + } + } + assert_eq!(map.get_val_at(&[1u8, 5]), Some(&2)); + assert_eq!(map.get_val_at(&[1u8, 9, 5]), Some(&3)); + assert_eq!(map.val_count(), 7); + } } From 8c937f82ce994005e91c4bc2e581e6e6abb09dc3 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:32:38 +0000 Subject: [PATCH 47/73] Fix zipper_head on a write zipper with a borrowed path as_static_path_zipper asserted the zipper held no borrowed origin path, but write_zipper_at_path always does. Once buffers are prepared the path lives in prefix_buf, so drop the borrowed slice instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/write_zipper.rs | 4 ++++ src/zipper_head.rs | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 51e2d55c..446acc83 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -1343,6 +1343,10 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC /// Internal method to re-borrow a WriteZipperCore without the `'path` lifetime fn as_static_path_zipper(&mut self) -> &mut WriteZipperCore<'a, 'static, V, A> { self.prepare_buffers(); + //The path is in `prefix_buf` now, so drop the borrowed copy + if self.key.origin_path.len() > 0 { + self.key.origin_path = SliceOrLen::new_owned(self.key.origin_path.len()); + } debug_assert!(!self.key.origin_path.is_slice() || self.key.origin_path.len() == 0); unsafe{ &mut *(self as *mut WriteZipperCore).cast() } } diff --git a/src/zipper_head.rs b/src/zipper_head.rs index d66b56b0..36c72d0e 100644 --- a/src/zipper_head.rs +++ b/src/zipper_head.rs @@ -1539,4 +1539,27 @@ mod tests { assert_eq!(map.get_val_at(&[1u8, 9, 5]), Some(&3)); assert_eq!(map.val_count(), 7); } + + /// A `ZipperHead` from a write zipper made with a borrowed path, and the zipper used afterwards + #[test] + fn zipper_head_from_write_zipper_at_borrowed_path() { + let mut map = PathMap::::new(); + map.set_val_at(&[1u8, 2, 3], 7); + { + let path = [1u8, 2]; + let mut wz = map.write_zipper_at_path(&path); + { + let zh = wz.zipper_head(); + let mut child = zh.write_zipper_at_exclusive_path(&[4u8]).unwrap(); + child.set_val(1); + } + assert_eq!(wz.origin_path(), &[1u8, 2]); + wz.descend_to(&[5u8]); + wz.set_val(2); + assert_eq!(wz.origin_path(), &[1u8, 2, 5]); + } + assert_eq!(map.get_val_at(&[1u8, 2, 4]), Some(&1)); + assert_eq!(map.get_val_at(&[1u8, 2, 5]), Some(&2)); + assert_eq!(map.get_val_at(&[1u8, 2, 3]), Some(&7)); + } } From 3158a51aeac6f2191956f1fb4fa24961382196fc Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:33:49 +0000 Subject: [PATCH 48/73] Fix get_trie_ref on a read zipper that owns its root ZipperHead read zippers own their root node, and get_trie_ref borrowed the root for the zipper's whole lifetime, which panics for an owned root. The TrieRef only needs to borrow the zipper. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/zipper.rs | 2 +- src/zipper_head.rs | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/zipper.rs b/src/zipper.rs index 66c66db1..b4aafed3 100644 --- a/src/zipper.rs +++ b/src/zipper.rs @@ -2016,7 +2016,7 @@ pub(crate) mod read_zipper_core { PathMap::new_with_root_in(root_node, root_val, self.alloc.clone()) } fn get_trie_ref(&self) -> TrieRef<'_, V, A> { - TrieRefBorrowed::new_with_key_and_path_in(self.focus_parent_borrowed(), || self.val(), self.node_key(), b"", self.alloc.clone()).into() + TrieRefBorrowed::new_with_key_and_path_in(self.focus_parent(), || self.val(), self.node_key(), b"", self.alloc.clone()).into() } fn get_focus(&self) -> OpaqueAbstractNodeRef<'_, V, A> { self.get_focus_at([]) diff --git a/src/zipper_head.rs b/src/zipper_head.rs index 36c72d0e..8acf19f2 100644 --- a/src/zipper_head.rs +++ b/src/zipper_head.rs @@ -1562,4 +1562,26 @@ mod tests { assert_eq!(map.get_val_at(&[1u8, 2, 5]), Some(&2)); assert_eq!(map.get_val_at(&[1u8, 2, 3]), Some(&7)); } + + /// `get_trie_ref`, `get_focus` and forks from a head's read zipper, which owns its root node + #[test] + fn head_read_zipper_trie_refs() { + let mut map = PathMap::::new(); + for p in [&[1u8, 2, 1][..], &[1, 2, 1, 0], &[1, 3], &[0]] { map.set_val_at(p, 7); } + let zh = map.zipper_head(); + for path in [&[][..], &[1u8], &[1u8, 2], &[9u8]] { + let mut rz = zh.read_zipper_at_path(path).unwrap(); + for step in [&[][..], &[2u8], &[2u8, 1]] { + rz.reset(); + rz.descend_to(step); + let tr = rz.get_trie_ref(); + assert_eq!(tr.val(), rz.val(), "{path:?} {step:?}"); + assert_eq!(tr.child_mask(), rz.child_mask(), "{path:?} {step:?}"); + let _ = rz.get_focus(); + let fork = rz.fork_read_zipper(); + assert_eq!(fork.get_trie_ref().val(), rz.val(), "{path:?} {step:?}"); + assert_eq!(fork.trie_ref_at_path(&[0u8]).val(), rz.val_at(&[0u8]), "{path:?} {step:?}"); + } + } + } } From c0f178a65df87951de4d105f6f740e2adbc44c29 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:35:25 +0000 Subject: [PATCH 49/73] Fix ProductZipper is_shared at a factor root A secondary factor's root isn't a child of the node above it, so the core zipper's parent lookup unwrapped None. Ask the factor's TrieRef instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/product_zipper.rs | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/product_zipper.rs b/src/product_zipper.rs index 9f2be3e9..0dce0faf 100644 --- a/src/product_zipper.rs +++ b/src/product_zipper.rs @@ -148,6 +148,14 @@ impl<'factor_z, 'trie, V: Clone + Send + Sync + Unpin, A: Allocator> ProductZipp self.enroll_next_factor(); } } + /// The secondary factor whose root node is the focus, if any. Its node is not a child of the + /// node above it, so the core zipper can't look it up. + fn factor_root(&self) -> Option<&TrieRef<'trie, V, A>> { + match self.factor_paths.last() { + Some(&start) if start == self.depth() => self.secondaries.get(self.factor_paths.len() - 1), + _ => None + } + } /// Internal method to make sure `self.factor_paths` is correct after an ascend method #[inline] fn fix_after_ascend(&mut self) { @@ -362,8 +370,19 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper } impl ZipperConcrete for ProductZipper<'_, '_, V, A> { - fn shared_node_id(&self) -> Option { self.z.shared_node_id() } - fn is_shared(&self) -> bool { self.z.is_shared() } + fn shared_node_id(&self) -> Option { + match self.factor_root() { + Some(_) if self.z.is_val() => None, + Some(factor) => factor.shared_node_id(), + None => self.z.shared_node_id(), + } + } + fn is_shared(&self) -> bool { + match self.factor_root() { + Some(factor) => factor.is_shared(), + None => self.z.is_shared(), + } + } } impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> ZipperPathBuffer for ProductZipper<'_, 'trie, V, A> { @@ -1975,6 +1994,24 @@ mod tests { |btm: &mut PathMap<()>, path: &[u8]| -> _ { ProductZipperG::new::<[ReadZipperUntracked<()>; 0]>(btm.read_zipper_at_path(path), []) }); + + /// `is_shared` and `shared_node_id` across factor boundaries + #[test] + fn product_zipper_is_shared_across_factors() { + let mut a = PathMap::::new(); + for p in [&[1u8, 2, 1][..], &[1, 2, 1, 0], &[1, 2, 1, 3, 3], &[0], &[2, 2]] { a.set_val_at(p, 7); } + let b = a.clone(); + let mut z = ProductZipper::new(a.read_zipper_at_path(&[1u8, 2, 1]), [b.read_zipper()]); + let mut factor_roots = 0; + while z.to_next_step() { + let _ = (z.is_shared(), z.shared_node_id()); + if z.factor_root().is_some() { + factor_roots += 1; + assert!(z.is_shared(), "{:?}", z.path()); + } + } + assert!(factor_roots > 0); + } } //POSSIBLE FUTURE DIRECTION: From 374dba7c000d8385a7ad3c7003aa81d9b612d10a Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:37:09 +0000 Subject: [PATCH 50/73] Fix ProductZipper::val_count outside the last factor val_count delegated to the core zipper, which only sees the current factor, and panicked elsewhere. Count by walking a copy of the product zipper below the focus. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/product_zipper.rs | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/product_zipper.rs b/src/product_zipper.rs index 0dce0faf..6ea30ff2 100644 --- a/src/product_zipper.rs +++ b/src/product_zipper.rs @@ -183,8 +183,22 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper self.z.reset() } fn val_count(&self) -> usize { - debug_assert!(self.focus_factor() == self.factor_count() - 1); - self.z.val_count() + //Values below the focus can be spread over later factors, so walk a copy of the zipper + let mut walker = ProductZipper { + z: self.z.clone(), + secondaries: self.secondaries.clone(), + factor_paths: self.factor_paths.clone(), + source_zippers: Vec::new(), + }; + let focus = self.path(); + let mut count = self.is_val() as usize; + while walker.to_next_val() { + if walker.path().len() <= focus.len() || !walker.path().starts_with(focus) { + break + } + count += 1; + } + count } fn descend_to_existing>(&mut self, k: K) -> usize { let k = k.as_ref(); @@ -1995,6 +2009,31 @@ mod tests { ProductZipperG::new::<[ReadZipperUntracked<()>; 0]>(btm.read_zipper_at_path(path), []) }); + /// `val_count` counts the values below the focus in every later factor + #[test] + fn product_zipper_val_count_in_every_factor() { + let mut a = PathMap::::new(); + for p in [&[1u8, 2][..], &[1u8], &[3u8]] { a.set_val_at(p, 1); } + let mut b = PathMap::::new(); + for p in [&[4u8][..], &[4u8, 5], &[6u8]] { b.set_val_at(p, 2); } + let mut c = PathMap::::new(); + for p in [&[7u8][..], &[8u8, 9]] { c.set_val_at(p, 3); } + + //Every value path in the product, from a full walk + let mut all = vec![]; + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper(), c.read_zipper()]); + while z.to_next_val() { all.push(z.path().to_vec()); } + assert!(!all.is_empty()); + + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper(), c.read_zipper()]); + loop { + let focus = z.path().to_vec(); + let expected = all.iter().filter(|p| p.starts_with(&focus)).count(); + assert_eq!(z.val_count(), expected, "{focus:?}"); + if !z.to_next_step() { break } + } + } + /// `is_shared` and `shared_node_id` across factor boundaries #[test] fn product_zipper_is_shared_across_factors() { From 3cddba740286fd01e74b655ff40eba68964ab9b5 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:38:34 +0000 Subject: [PATCH 51/73] Fix graft_child_maps below a long root path with_node_at_path joined the focus's node key and the child path in a fixed MAX_NODE_KEY_BYTES buffer, so a root path of 48 bytes or more overflowed it. Fall back to a heap buffer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/write_zipper.rs | 48 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 446acc83..495437e4 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -2570,13 +2570,17 @@ impl <'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> WriteZipperC { self.in_zipper_mut_static_result( |focus_node, partial_key| { - let mut key_buf = [0u8; MAX_NODE_KEY_BYTES]; - key_buf[0..partial_key.len()].copy_from_slice(partial_key); - //GOAT, currently this will panic if the path is too long to fit in the buffer, which means this internal API - // isn't suitable for general-purpose path-based ops yet, but we're using it to deal with single-byte ops - key_buf[partial_key.len()..partial_key.len()+path.len()].copy_from_slice(path); - let full_key = &key_buf[0..partial_key.len()+path.len()]; - node_f(focus_node, full_key) + let full_len = partial_key.len() + path.len(); + if full_len <= MAX_NODE_KEY_BYTES { + let mut key_buf = [0u8; MAX_NODE_KEY_BYTES]; + key_buf[0..partial_key.len()].copy_from_slice(partial_key); + key_buf[partial_key.len()..full_len].copy_from_slice(path); + node_f(focus_node, &key_buf[0..full_len]) + } else { + //Too long for the stack buffer + let full_key = [partial_key, path].concat(); + node_f(focus_node, &full_key) + } }, retry_f ) @@ -6770,6 +6774,36 @@ mod tests { /// (absent, or a dangling path) therefore empties an existing branch -- which then survives /// as a dangling path, exactly as after `graft` of an empty source -- and leaves an absent /// branch absent. `remove_unset` removes the unset branches outright first. + /// `graft_child_maps` and `graft_masked_branches` below a root path too long for one node key + #[test] + fn graft_child_maps_long_root() { + for root_len in [47usize, 48, 60, 200] { + let root = vec![0u8; root_len]; + let mut map = PathMap::::new(); + { + let mut wz = map.write_zipper_at_path(&root); + wz.graft_child_maps(ByteMask::from_iter([1u8, 3]), [PathMap::single([2u8], 5), PathMap::single([], 6)], false); + } + let mut want = root.clone(); + want.extend([1u8, 2]); + assert_eq!(map.get_val_at(&want), Some(&5), "root {root_len}"); + want.truncate(root_len); + want.push(3); + assert_eq!(map.get_val_at(&want).is_some(), cfg!(feature = "graft_root_vals"), "root {root_len}"); + + let mut src = PathMap::::new(); + src.set_val_at([4u8, 4], 9); + let mut map = PathMap::::new(); + { + let mut wz = map.write_zipper_at_path(&root); + wz.graft_masked_branches(&src.read_zipper(), ByteMask::from_iter([4u8]), false); + } + let mut want = root.clone(); + want.extend([4u8, 4]); + assert_eq!(map.get_val_at(&want), Some(&9), "root {root_len}"); + } + } + #[test] fn graft_child_maps_dense() { use crate::utils::BitMask; From b20bcfbe20bd12f1a848c66292b6535e96f7e836 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:39:59 +0000 Subject: [PATCH 52/73] Fix get_val_with_witness at an owned zipper's root With no root value and the zipper root at the root node, the lookup used an unset parent key and sliced out of range. The value there is just root_val. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/zipper.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/zipper.rs b/src/zipper.rs index b4aafed3..24907a96 100644 --- a/src/zipper.rs +++ b/src/zipper.rs @@ -2603,7 +2603,8 @@ pub(crate) mod read_zipper_core { if let Some((parent, _iter_tok, _prefix_offset)) = self.ancestors.last() { parent.node_get_val(self.parent_key()) } else { - if self.root_val.is_some() { + if self.root_val.is_some() || self.root_parent_key_start == usize::MAX { + //No parent key: the zipper root is the root node itself, and its value is `root_val` self.root_val } else { //We know the node in the witness and the node in self.root_node are the same, @@ -3410,6 +3411,21 @@ pub(crate) mod read_zipper_core { } } + /// `get_val_with_witness` agrees with `val` on owned read zippers, including at a root without a value + #[test] + fn read_zipper_owned_get_val_with_witness() { + let mut map = PathMap::::new(); + for p in [&[1u8][..], &[1, 2], &[3, 4, 5]] { map.set_val_at(p, p.len() as u64); } + for root in [&[][..], &[1u8], &[3u8], &[3u8, 4], &[9u8]] { + let mut z = map.clone().into_read_zipper(root); + loop { + let w = z.witness(); + assert_eq!(z.get_val_with_witness(&w), z.val(), "{root:?} {:?}", z.path()); + if !z.to_next_step() { break } + } + } + } + /// Validate we don't accidentially reallocate the path buffer when we don't need to #[test] fn read_zipper_reserve_buffer_test() { From 59edba984c0f090751b7022b09454ba019d121bf Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:41:13 +0000 Subject: [PATCH 53/73] Fix PrefixZipper::descend_first_k_path with k = 0 in the source k = 0 is documented to return false, but with the focus in the source the method sliced the prefix past its end. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/prefix_zipper.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/prefix_zipper.rs b/src/prefix_zipper.rs index 14823612..9e4290e0 100644 --- a/src/prefix_zipper.rs +++ b/src/prefix_zipper.rs @@ -558,7 +558,7 @@ impl<'prefix, Z> ZipperIteration for PrefixZipper<'prefix, Z> } fn descend_first_k_path_observed(&mut self, k: usize, obs: &mut Obs) -> bool { - if self.position.is_invalid() { + if k == 0 || self.position.is_invalid() { return false; } //The prefix is a single forced path, so the bytes it contributes always exist and never @@ -954,4 +954,25 @@ mod tests { //...so `descend_until` must report that it moved assert_eq!(moved, true); } + + /// `descend_first_k_path(0)` returns `false` without moving, wherever the focus is + #[test] + fn prefix_zipper_descend_first_k_path_zero() { + let mut map = PathMap::::new(); + map.set_val_at(&[0x22u8], 1); + map.set_val_at(&[0x22u8, 1], 2); + let mut z = PrefixZipper::new(&[2u8, 3][..], map.read_zipper()); + for setup in 0..4 { + z.reset(); + match setup { + 0 => {}, + 1 => { z.descend_to_byte(2); }, + 2 => { z.descend_to_byte(2); z.descend_last_path(); z.descend_last_path(); }, + _ => { z.descend_to(&[2u8, 3, 0x22]); }, + } + let path = z.path().to_vec(); + assert!(!z.descend_first_k_path(0), "setup {setup}"); + assert_eq!(z.path(), &path[..], "setup {setup}"); + } + } } From 8905e7cce14a4f67fd97bd3054d6270a24adafa6 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:46:56 +0000 Subject: [PATCH 54/73] Fix UB joining a list node with a cell node LineListNode::pjoin_dyn's CellByteNode arm cast the other node with as_dense_unchecked, which is undefined behaviour for a cell node. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/line_list_node.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/line_list_node.rs b/src/line_list_node.rs index bd089ebb..6b027ae9 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -2669,7 +2669,7 @@ impl TrieNode for LineListNode unimplemented!() }, CELL_BYTE_NODE_TAG => { - let other_dense_node = unsafe{ other.as_dense_unchecked() }; + let other_dense_node = unsafe{ other.as_cell_unchecked() }; let mut new_node = other_dense_node.clone(); match new_node.merge_from_list_node(self, true) { //See the DENSE_BYTE_NODE_TAG arm: two empty nodes join to an empty result @@ -4099,4 +4099,30 @@ mod tests { m.write_zipper().restrict(&o.read_zipper()); assert_eq!(keys(&m), ["bx"]); } + + /// Joining a list node with a `CellByteNode` (left behind by a `ZipperHead`) + #[test] + fn list_node_join_with_cell_node() { + use crate::PathMap; + use crate::zipper::*; + let mut cell = PathMap::::new(); + { + let zh = cell.zipper_head(); + for b in [1u8, 2] { + let mut wz = zh.write_zipper_at_exclusive_path(&[b]).unwrap(); + wz.set_val(b as u64); + } + } + assert!(cell.root().unwrap().as_tagged().tag() == CELL_BYTE_NODE_TAG, "the layout this test needs"); + let mut list = PathMap::::new(); + list.set_val_at(&[0u8, 5], 7); + assert!(list.root().unwrap().as_tagged().tag() == LINE_LIST_NODE_TAG, "the layout this test needs"); + + let joined = list.join(&cell); + let vals: Vec<(Vec, u64)> = joined.iter().map(|(k, v)| (k, *v)).collect(); + assert_eq!(vals, vec![(vec![0, 5], 7), (vec![1], 1), (vec![2], 2)]); + let mut into = list.clone(); + into.write_zipper().join_map_into(cell.clone()); + assert_eq!(into.iter().map(|(k, v)| (k, *v)).collect::>(), vals); + } } From 6c56544a0e73f70806bffb349f3bf78b1cbf459b Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 03:58:57 +0000 Subject: [PATCH 55/73] Fix ZipperHead exclusive paths over an empty node make_cell_node called make_mut on the empty sentinel, which panics. Replace the sentinel with a new CellByteNode instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/trie_node.rs | 8 ++++++-- src/zipper_head.rs | 44 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/trie_node.rs b/src/trie_node.rs index 5bdce103..2ef8c8c3 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -2652,8 +2652,12 @@ pub(crate) fn node_along_path_mut<'a, 'k, V: Clone + Send + Sync, A: Allocator>( /// Ensures the node is a CellByteNode /// /// Returns `true` if the node was upgraded and `false` if it already was a CellByteNode -pub(crate) fn make_cell_node(node: &mut TrieNodeODRc) -> bool { - if !node.as_tagged().is_cell_node() { +pub(crate) fn make_cell_node(node: &mut TrieNodeODRc, alloc: A) -> bool { + if node.is_empty() { + //The empty sentinel can't be made mutable; there is nothing in it to keep + *node = TrieNodeODRc::new_in(crate::dense_byte_node::CellByteNode::new_in(alloc.clone()), alloc); + true + } else if !node.as_tagged().is_cell_node() { let replacement = node.make_mut().convert_to_cell_node(); *node = replacement; true diff --git a/src/zipper_head.rs b/src/zipper_head.rs index 8acf19f2..9363b0ec 100644 --- a/src/zipper_head.rs +++ b/src/zipper_head.rs @@ -343,7 +343,7 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + debug_assert_eq!(z.focus_stack.depth(), 1); z.focus_stack.to_root(); let stack_root = z.focus_stack.root_mut().unwrap(); - make_cell_node(stack_root); + make_cell_node(stack_root, z.alloc.clone()); let root_val = z.root_val.as_mut().unwrap(); return (stack_root, unsafe{ &mut **root_val }) } @@ -386,7 +386,7 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + |node, key| { let new_node = if key.len() > 0 { if let Some(mut remaining) = node.take_node_at_key(key, false) { - make_cell_node(&mut remaining); + make_cell_node(&mut remaining, alloc.clone()); remaining } else { TrieNodeODRc::new_in(CellByteNode::new_in(alloc.clone()), alloc) @@ -419,8 +419,9 @@ pub(crate) fn prepare_exclusive_write_path<'a, 'trie: 'a, 'path: 'a, V: Clone + //If the node on top of the stack is not a cell node, we need to upgrade it if !z.focus_stack.top().unwrap().is_cell_node() { + let alloc = z.alloc.clone(); swap_top_node(&mut z.focus_stack, &z.key, |mut existing_node| { - make_cell_node(&mut existing_node); + make_cell_node(&mut existing_node, alloc); existing_node }); } @@ -439,9 +440,9 @@ fn prepare_node_at_path_end<'a, V: Clone + Send + Sync, A: Allocator>(start_node let mut node_ref = node.make_mut(); let mut new_parent = match node_ref.take_node_at_key(remaining_key, false) { Some(downward_node) => downward_node, - None => TrieNodeODRc::new_in(CellByteNode::new_in(alloc.clone()), alloc) + None => TrieNodeODRc::new_in(CellByteNode::new_in(alloc.clone()), alloc.clone()) }; - make_cell_node(&mut new_parent); + make_cell_node(&mut new_parent, alloc.clone()); let result = node_ref.node_set_branch(remaining_key, new_parent); match result { Ok(_) => { }, @@ -452,7 +453,7 @@ fn prepare_node_at_path_end<'a, V: Clone + Send + Sync, A: Allocator>(start_node node = child_node; } else { //Otherwise just upgrade node - make_cell_node(node); + make_cell_node(node, alloc); } node } @@ -1584,4 +1585,35 @@ mod tests { } } } + + /// Exclusive paths from a head whose focus node is the empty sentinel + #[test] + fn exclusive_path_over_empty_node() { + let setups: [fn(&mut PathMap); 3] = [ + |m| { m.write_zipper_at_path(&[0u8, 0]).remove_branches(false); }, + |m| { let e = PathMap::::new(); m.write_zipper_at_path(&[0u8, 0]).graft(&e.read_zipper()); }, + |m| { m.write_zipper_at_path(&[0u8, 0]).take_map(false); }, + ]; + for (i, setup) in setups.iter().enumerate() { + for paths in [[&[][..], &[5u8][..]], [&[5u8, 6][..], &[][..]], [&[0u8, 0][..], &[1u8][..]]] { + let mut map = PathMap::::new(); + map.set_val_at(&[0u8, 0, 1, 2], 9); + map.set_val_at(&[7u8], 9); + setup(&mut map); + { + let mut wz = map.write_zipper_at_path(&[0u8, 0]); + let zh = wz.zipper_head(); + for (n, p) in paths.iter().enumerate() { + let mut w = zh.write_zipper_at_exclusive_path(p).unwrap(); + w.set_val(n as u64); + } + } + assert_eq!(map.get(&[7u8]), Some(&9), "setup {i} {paths:?}"); + for (n, p) in paths.iter().enumerate() { + let full: Vec = [&[0u8, 0][..], p].concat(); + assert_eq!(map.get(&full), Some(&(n as u64)), "setup {i} {paths:?}"); + } + } + } + } } From 7f9c59a561bb72b80b3cb4576bb91fde07dad8c2 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 04:20:33 +0000 Subject: [PATCH 56/73] Fix the default k-path walk looping forever at a leaf With nothing below the base and no sibling, k_path_default_internal never reached its exit and spun. It also let k = 0 step sideways from the base. Stop when back at the base. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/zipper.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/zipper.rs b/src/zipper.rs index 24907a96..22f34027 100644 --- a/src/zipper.rs +++ b/src/zipper.rs @@ -1136,6 +1136,8 @@ fn k_path_default_internal(z: &mut if z.depth() == base_idx + k { return true } } } + //Back at the base: nothing (more) below it, and its own siblings are out of bounds + if z.depth() == base_idx { return false } //A sibling step replaces the last byte rather than adding one, so the observer sees the old //byte retracted before the new one arrives if let Some(byte) = z.to_next_sibling_byte() { @@ -3411,6 +3413,26 @@ pub(crate) mod read_zipper_core { } } + /// The default k-path walk ends when there is nothing below its base, and `k = 0` returns `false` + #[test] + fn default_k_path_walk_at_a_leaf() { + use crate::zipper::ProductZipperG; + let mut leaf = PathMap::::new(); + leaf.set_val_at(&[1u8], 1); + let empty = PathMap::::new(); + for (map, path) in [(&empty, &[][..]), (&leaf, &[1u8][..]), (&leaf, &[][..])] { + for k in 0..3 { + let mut z = ProductZipperG::new(map.read_zipper(), [empty.read_zipper()]); + z.descend_to(path); + let found = z.descend_first_k_path(k); + assert_eq!(found, map.val_count() > 0 && path.is_empty() && k == 1, "{path:?} k={k}"); + if !found { + assert_eq!(z.path(), path, "{path:?} k={k}"); + } + } + } + } + /// `get_val_with_witness` agrees with `val` on owned read zippers, including at a root without a value #[test] fn read_zipper_owned_get_val_with_witness() { From 96bfa487ace484737453ecb929c6a6291f2fda28 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 04:21:17 +0000 Subject: [PATCH 57/73] Fix ProductZipper factor bookkeeping at the root A factor was entered below a primary path that doesn't exist, and a sibling step at depth 0 dropped the factor record although the core zipper can't leave the factor there. Later moves then saw factor roots the zipper didn't know about, and is_shared unwrapped None. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/product_zipper.rs | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/product_zipper.rs b/src/product_zipper.rs index 6ea30ff2..72b064ac 100644 --- a/src/product_zipper.rs +++ b/src/product_zipper.rs @@ -136,7 +136,7 @@ impl<'factor_z, 'trie, V: Clone + Send + Sync + Unpin, A: Allocator> ProductZipp /// `product_zipper_test4` for more discussion. #[inline] fn ensure_descend_next_factor(&mut self) { - if self.factor_paths.len() < self.secondaries.len() && self.z.child_count() == 0 { + if self.factor_paths.len() < self.secondaries.len() && self.z.child_count() == 0 && self.z.path_exists() { //We don't want to push the same factor on the stack twice if let Some(factor_path_len) = self.factor_paths.last() { @@ -288,7 +288,8 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper moved } fn to_next_sibling_byte(&mut self) -> Option { - if self.factor_paths.last().cloned().unwrap_or(0) == self.depth() { + //Stepping sideways leaves a factor entered at this depth, but at the root there's no sideways + if self.depth() > 0 && self.factor_paths.last().cloned() == Some(self.depth()) { self.factor_paths.pop(); } let moved = self.z.to_next_sibling_byte(); @@ -296,7 +297,8 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper moved } fn to_prev_sibling_byte(&mut self) -> Option { - if self.factor_paths.last().cloned().unwrap_or(0) == self.depth() { + //Stepping sideways leaves a factor entered at this depth, but at the root there's no sideways + if self.depth() > 0 && self.factor_paths.last().cloned() == Some(self.depth()) { self.factor_paths.pop(); } let moved = self.z.to_prev_sibling_byte(); @@ -2034,6 +2036,34 @@ mod tests { } } + /// k-path walks and sibling steps keep factor bookkeeping in step with the core zipper, including + /// with a primary rooted at a missing path or at a leaf + #[test] + fn product_zipper_k_path_walk_from_the_root() { + let mut a = PathMap::<()>::new(); + for p in [&[2u8, 2, 0, 0][..], &[0xe7]] { a.set_val_at(p, ()); } + let b = a.clone(); + for root in [&[0xaau8, 0x77][..], &[0xe7u8][..], &[][..]] { + for k in 1..4 { + let mut z = ProductZipper::new(a.read_zipper_at_path(root), [b.read_zipper()]); + let mut paths = vec![]; + if z.descend_first_k_path(k) { + paths.push(z.path().to_vec()); + while paths.len() < 64 && z.to_next_k_path(k) { paths.push(z.path().to_vec()); } + } + assert_eq!(z.path(), &[] as &[u8], "{root:?} k={k}"); + let _ = (z.is_shared(), z.shared_node_id(), z.child_mask()); + assert!(paths.iter().all(|p| p.len() == k), "{root:?} k={k}: {paths:?}"); + if root == &[0xaau8, 0x77][..] { + assert!(paths.is_empty(), "a missing primary has no paths: {paths:?}"); + } + let mut z = ProductZipper::new(a.read_zipper_at_path(root), [b.read_zipper()]); + let _ = (z.to_next_sibling_byte(), z.to_prev_sibling_byte()); + let _ = (z.is_shared(), z.child_mask(), z.descend_first_byte(), z.is_shared()); + } + } + } + /// `is_shared` and `shared_node_id` across factor boundaries #[test] fn product_zipper_is_shared_across_factors() { From 169afc989cb64019925f35df42abf169ea5c6438 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 04:30:53 +0000 Subject: [PATCH 58/73] Fix product zipper sibling steps at the root ProductZipperG and DependentProductZipperG read focus_byte at the root, which is the primary's root path byte, and tried to ascend from there. The root has no siblings. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dependent_zipper.rs | 20 ++++++++++++++++++++ src/product_zipper.rs | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/dependent_zipper.rs b/src/dependent_zipper.rs index 5620e2a4..2df291f8 100644 --- a/src/dependent_zipper.rs +++ b/src/dependent_zipper.rs @@ -161,6 +161,10 @@ impl<'trie, PrimaryZ, SecondaryZ, V, C, F : Clone + for <'a> FnOnce(C, &'a [u8], /// a combination between `to_next_sibling` and `to_prev_sibling` fn to_sibling_byte(&mut self, next: bool) -> Option { + //`focus_byte` is unspecified at the root, and the root has no siblings + if self.depth() == 0 { + return None; + } let byte = self.focus_byte()?; let ascended = self.ascend(1); debug_assert_eq!(ascended, 1, "must ascend"); @@ -568,4 +572,20 @@ rubiconrubicon rubicundusrubicundus ") } + + /// Sibling steps at the root when the primary is rooted below the map root + #[test] + fn dependent_product_zipper_sibling_step_at_root() { + let mut a = PathMap::::new(); + for p in [&[1u8, 0, 2, 7][..], &[1, 0, 2, 8], &[9]] { a.set_val_at(p, 1); } + let mut z = DependentProductZipperG::new_enroll(a.read_zipper_at_path(&[1u8, 0, 2]), (), |_, _, _| ((), None::>)); + assert_eq!(z.to_next_sibling_byte(), None); + assert_eq!(z.to_prev_sibling_byte(), None); + assert_eq!(z.descend_first_byte(), Some(7)); + assert_eq!(z.to_next_sibling_byte(), Some(8)); + z.reset(); + let mut n = 0; + while z.to_next_val() { n += 1; assert!(n < 100); } + assert_eq!(n, 2); + } } diff --git a/src/product_zipper.rs b/src/product_zipper.rs index 72b064ac..0636eaf9 100644 --- a/src/product_zipper.rs +++ b/src/product_zipper.rs @@ -532,6 +532,10 @@ impl<'trie, PrimaryZ, SecondaryZ, V> ProductZipperG<'trie, PrimaryZ, SecondaryZ, /// a combination between `to_next_sibling` and `to_prev_sibling` fn to_sibling_byte(&mut self, next: bool) -> Option { + //`focus_byte` is unspecified at the root, and the root has no siblings + if self.depth() == 0 { + return None; + } let byte = self.focus_byte()?; let ascended = self.ascend(1); debug_assert_eq!(ascended, 1, "must ascend"); @@ -2064,6 +2068,22 @@ mod tests { } } + /// Sibling steps and value walks at the root of a `ProductZipperG` whose primary is rooted below the map root + #[test] + fn product_zipper_g_sibling_step_at_root() { + let mut a = PathMap::::new(); + for p in [&[2u8, 3, 2, 1, 0, 7][..], &[2, 3, 2, 1, 0, 8, 1], &[9]] { a.set_val_at(p, 1); } + let b = a.clone(); + let mut z = ProductZipperG::new(a.read_zipper_at_path(&[2u8, 3, 2, 1, 0]), [b.read_zipper()]); + assert_eq!(z.to_next_sibling_byte(), None); + assert_eq!(z.to_prev_sibling_byte(), None); + assert_eq!(z.path(), &[] as &[u8]); + let mut n = 0; + while z.to_next_val() { n += 1; assert!(n < 100); } + assert!(n > 0); + assert_eq!(z.path(), &[] as &[u8]); + } + /// `is_shared` and `shared_node_id` across factor boundaries #[test] fn product_zipper_is_shared_across_factors() { From bfd2d0e1859ccfe09b1241786b01bc8ce090088e Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 04:32:12 +0000 Subject: [PATCH 59/73] Fix OverlayZipper focus_byte at the root Its sources may be rooted at different paths, so at the root their focus bytes differ and the debug assert failed; sibling steps then tried to move the root. Report None at the root. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/overlay_zipper.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/overlay_zipper.rs b/src/overlay_zipper.rs index 211c8aa3..f462cd15 100644 --- a/src/overlay_zipper.rs +++ b/src/overlay_zipper.rs @@ -159,6 +159,10 @@ impl ZipperMoving #[inline] fn focus_byte(&self) -> Option { + //The sources may be rooted at different paths, so at the root their bytes differ + if self.depth() == 0 { + return None; + } let byte = self.a.focus_byte(); debug_assert_eq!(byte, self.b.focus_byte()); byte @@ -577,4 +581,21 @@ mod tests { assert_eq!(moved, true); assert_eq!(observed, oz.path(), "observer must match the resulting path"); } + + /// Sources rooted at different paths: no focus byte, and no sibling step, at the root + #[test] + fn overlay_sources_at_different_roots() { + use crate::zipper::ZipperIteration; + let mut a = PathMap::::new(); + for p in [&[1u8, 5][..], &[1, 6], &[2, 5, 1]] { a.set_val_at(p, 1); } + let mut z = OverlayZipper::new(a.read_zipper_at_path(&[1u8]), a.read_zipper_at_path(&[2u8])); + assert_eq!(z.focus_byte(), None); + assert_eq!(z.to_next_sibling_byte(), None); + assert_eq!(z.to_prev_sibling_byte(), None); + let mut steps = vec![]; + while z.to_next_step() { steps.push(z.path().to_vec()); assert!(steps.len() < 16); } + assert_eq!(steps, vec![vec![5], vec![5, 1], vec![6]]); + let mut o = Vec::new(); + while z.to_next_val_observed(&mut o) { assert_eq!(&o[..], z.path()); } + } } From 3c4e277d30ecce37e53a4725f5c3acf8a694c43c Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 04:33:04 +0000 Subject: [PATCH 60/73] Fix OverlayZipper::descend_to_val when the second source stops first The branch meant to bring the second source down to the first one's depth descended the first source again, leaving them at different depths. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/overlay_zipper.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/overlay_zipper.rs b/src/overlay_zipper.rs index f462cd15..d9f1ae94 100644 --- a/src/overlay_zipper.rs +++ b/src/overlay_zipper.rs @@ -215,7 +215,7 @@ impl ZipperMoving self.a.ascend(depth_a - depth_o); depth_o } else { - self.a.descend_to(&path[depth_o..depth_a]); + self.b.descend_to(&path[depth_o..depth_a]); depth_a } } else { @@ -598,4 +598,21 @@ mod tests { let mut o = Vec::new(); while z.to_next_val_observed(&mut o) { assert_eq!(&o[..], z.path()); } } + + /// `descend_to_val` keeps both sources at the same place when the second one stops first + #[test] + fn overlay_descend_to_val_second_stops_first() { + let mut a = PathMap::::new(); + a.set_val_at(&[1u8, 2, 3], 1); + let mut b = PathMap::::new(); + b.set_val_at(&[1u8, 7], 2); + for (x, y) in [(&a, &b), (&b, &a)] { + let mut z = OverlayZipper::new(x.read_zipper(), y.read_zipper()); + assert_eq!(z.descend_to_val(&[1u8, 2, 3, 4, 5]), 3); + assert_eq!(z.depth(), 3); + assert_eq!(z.path(), &[1u8, 2, 3]); + assert_eq!(z.ascend(3), 3); + assert_eq!(z.depth(), 0); + } + } } From 0080b69bf6e6c8bc00c5ac3e4c7883343d66c3ee Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 04:38:18 +0000 Subject: [PATCH 61/73] Fix PrefixZipper over a source rooted at a missing path The prefix always reported itself as existing, with one child, so a walk descended through it into a source that doesn't exist. The prefix now exists only when the source's root does. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/prefix_zipper.rs | 46 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/src/prefix_zipper.rs b/src/prefix_zipper.rs index 9e4290e0..7eb1218d 100644 --- a/src/prefix_zipper.rs +++ b/src/prefix_zipper.rs @@ -122,6 +122,9 @@ impl<'prefix, Z> PrefixZipper<'prefix, Z> /// Returns `true` if the focus moved. The descended bytes are appended to this zipper's path /// buffer and reported to `obs`. Does nothing if the focus is already within the source. fn consume_prefix(&mut self, obs: &mut Obs) -> bool { + if !self.source.path_exists() { + return false; + } match self.position.prefixed_depth() { Some(prefixed_depth) => { let prefix_rest = &self.prefix[self.origin_depth + prefixed_depth..]; @@ -331,7 +334,9 @@ impl<'prefix, Z> Zipper for PrefixZipper<'prefix, Z> { fn path_exists(&self) -> bool { match self.position { - PrefixPos::Prefix {..} => true, + //The source stays at its root while the focus is in the prefix, and the prefix only + // exists if the source's root does + PrefixPos::Prefix {..} => self.source.path_exists(), PrefixPos::PrefixOff {..} => false, PrefixPos::Source => self.source.path_exists(), } @@ -344,17 +349,18 @@ impl<'prefix, Z> Zipper for PrefixZipper<'prefix, Z> } fn child_count(&self) -> usize { match self.position { - PrefixPos::Prefix {..} => 1, + PrefixPos::Prefix {..} => self.source.path_exists() as usize, PrefixPos::PrefixOff {..} => 0, PrefixPos::Source => self.source.child_count(), } } fn child_mask(&self) -> ByteMask { match self.position { - PrefixPos::Prefix { valid } => { + PrefixPos::Prefix { valid } if self.source.path_exists() => { let byte = self.prefix[self.origin_depth + valid]; ByteMask::from(byte) }, + PrefixPos::Prefix {..} => ByteMask::EMPTY, PrefixPos::PrefixOff {..} => ByteMask::EMPTY, PrefixPos::Source => self.source.child_mask(), } @@ -404,7 +410,7 @@ impl<'prefix, Z> ZipperMoving for PrefixZipper<'prefix, Z> if let PrefixPos::Prefix { valid } = &self.position { let valid = *valid; let rest_prefix = &self.prefix[self.origin_depth + valid..]; - let overlap = find_prefix_overlap(rest_prefix, path); + let overlap = if self.source.path_exists() { find_prefix_overlap(rest_prefix, path) } else { 0 }; path = &path[overlap..]; self.set_valid(valid + overlap); descended += overlap; @@ -558,7 +564,7 @@ impl<'prefix, Z> ZipperIteration for PrefixZipper<'prefix, Z> } fn descend_first_k_path_observed(&mut self, k: usize, obs: &mut Obs) -> bool { - if k == 0 || self.position.is_invalid() { + if k == 0 || self.position.is_invalid() || !self.source.path_exists() { return false; } //The prefix is a single forced path, so the bytes it contributes always exist and never @@ -975,4 +981,34 @@ mod tests { assert_eq!(z.path(), &path[..], "setup {setup}"); } } + + /// A prefix in front of a source rooted at a missing path doesn't exist either + #[test] + fn prefix_zipper_over_missing_source() { + let mut map = PathMap::::new(); + map.set_val_at(&[0u8], 1); + let mut z = PrefixZipper::new(&[0u8, 7][..], map.read_zipper_at_path(&[5u8])); + assert!(!z.path_exists()); + assert_eq!(z.child_count(), 0); + assert_eq!(z.descend_first_byte(), None); + assert!(!z.to_next_step()); + assert!(!z.to_next_val()); + assert!(!z.descend_until()); + assert!(!z.descend_first_k_path(1)); + assert_eq!(z.path(), &[] as &[u8]); + assert_eq!(z.descend_to_existing(&[0u8, 7]), 0); + z.descend_to(&[0u8]); + assert!(!z.path_exists()); + assert_eq!(z.descend_first_byte(), None); + + //With the source present the prefix exists as before + let mut z = PrefixZipper::new(&[0u8, 7][..], map.read_zipper()); + assert!(z.path_exists()); + assert_eq!(z.descend_first_byte(), Some(0)); + assert!(z.path_exists()); + let mut steps = vec![]; + z.reset(); + while z.to_next_step() { steps.push(z.path().to_vec()); } + assert_eq!(steps, vec![vec![0], vec![0, 7], vec![0, 7, 0]]); + } } From 3de791475f39edca2a1d7189a68484e5b5d80b27 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 04:40:09 +0000 Subject: [PATCH 62/73] Fix TrieRef lookups with a node key longer than its buffer new_with_key_and_path_in copied the focus's node key into a fixed stack buffer, overflowing it (UB in release) when the key was longer, and truncated key + path when the two together didn't fit. Fall back to a heap buffer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/trie_ref.rs | 57 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/src/trie_ref.rs b/src/trie_ref.rs index 67b59d70..12bf0afe 100644 --- a/src/trie_ref.rs +++ b/src/trie_ref.rs @@ -101,7 +101,7 @@ fn trie_ref_from_key_and_path_in<'a, 'paths, V, A, R, RootValF, BuildF, InvalidF mut node: &'a TrieNodeODRc, root_val_f: RootValF, node_key: &'paths [u8], - mut path: &'paths [u8], + path: &'paths [u8], alloc: A, build: BuildF, invalid: InvalidF, @@ -115,31 +115,28 @@ where { // A temporary buffer on the stack, if we need to assemble a combined key from both the `node_key` and `path`. let mut temp_key_buf: [MaybeUninit; MAX_NODE_KEY_BYTES] = [MaybeUninit::uninit(); MAX_NODE_KEY_BYTES]; + let heap_key: Vec; + let mut path: &[u8] = path; let node_key_len = node_key.len(); let path_len = path.len(); // Copy the existing node key and the first chunk of the path into the temporary buffer, then try to descend one step. if node_key_len > 0 && path_len > 0 { - let next_node_path = unsafe { - // SAFETY: `temp_key_buf` has capacity for `MAX_NODE_KEY_BYTES` bytes. We copy exactly - // `node_key_len` bytes from `node_key`, which is a valid slice, then append at most the - // remaining buffer capacity from the valid slice `path`. Both destination ranges are - // within the stack buffer and do not overlap the sources. - let src_ptr = node_key.as_ptr(); - let dst_ptr = temp_key_buf.as_mut_ptr().cast::(); - core::ptr::copy_nonoverlapping(src_ptr, dst_ptr, node_key_len); - - let remaining_len = (MAX_NODE_KEY_BYTES - node_key_len).min(path_len); - let src_ptr = path.as_ptr(); - let dst_ptr = temp_key_buf.as_mut_ptr().cast::().add(node_key_len); - core::ptr::copy_nonoverlapping(src_ptr, dst_ptr, remaining_len); - - let total_buf_len = node_key_len + remaining_len; - // SAFETY: The first `total_buf_len` bytes of `temp_key_buf` were initialized by the - // copies above, and `total_buf_len <= MAX_NODE_KEY_BYTES`, so this slice is valid for - // reads for the duration of this function. - core::slice::from_raw_parts(temp_key_buf.as_mut_ptr().cast::(), total_buf_len) + let next_node_path: &[u8] = if node_key_len + path_len <= MAX_NODE_KEY_BYTES { + unsafe { + // SAFETY: `temp_key_buf` holds `MAX_NODE_KEY_BYTES` bytes and we copy + // `node_key_len + path_len <= MAX_NODE_KEY_BYTES` bytes from two valid slices into it, + // so the resulting slice is initialized and in bounds. + let dst_ptr = temp_key_buf.as_mut_ptr().cast::(); + core::ptr::copy_nonoverlapping(node_key.as_ptr(), dst_ptr, node_key_len); + core::ptr::copy_nonoverlapping(path.as_ptr(), dst_ptr.add(node_key_len), path_len); + core::slice::from_raw_parts(dst_ptr, node_key_len + path_len) + } + } else { + //Too long for the stack buffer + heap_key = [node_key, path].concat(); + &heap_key }; match node.as_tagged().node_get_child(next_node_path) { @@ -1448,4 +1445,24 @@ mod tests { assert!(!trie_ref.is_shared()); assert_eq!(trie_ref.shared_node_id(), None); } + + /// `val_at` and `trie_ref_at_path` where the focus key plus the path are longer than a node key + #[test] + fn trie_ref_long_node_key_and_path() { + let mut map = PathMap::::new(); + map.set_val_at(&[0u8; 70], 5); + map.set_val_at(&[1u8], 6); + for (focus, rest) in [(10usize, 60usize), (30, 40), (47, 23), (69, 1)] { + let mut rz = map.read_zipper(); + rz.descend_to(&vec![0u8; focus]); + assert_eq!(rz.val_at(&vec![0u8; rest]), Some(&5), "{focus}+{rest}"); + assert_eq!(rz.trie_ref_at_path(&vec![0u8; rest]).val(), Some(&5), "{focus}+{rest}"); + assert_eq!(rz.val_at(&vec![0u8; rest + 1]), None, "{focus}+{rest}"); + } + //A focus far below anything in the trie + let mut rz = map.read_zipper(); + rz.descend_to(&[7u8; 60]); + assert_eq!(rz.val_at(&[1u8]), None); + assert_eq!(rz.val_at(&[7u8; 60]), None); + } } From 6dfde49d7eff9cc725197052e462f60577c333b5 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 04:48:01 +0000 Subject: [PATCH 63/73] Fix merkleize with several dangling paths merkleize recursed into empty child links, and the second one found the first in its memo and tried to replace it in the parent, which refuses to hand out an empty child. Empty children have nothing to share, so skip them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/merkleization.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/merkleization.rs b/src/merkleization.rs index 6840abb3..99382831 100644 --- a/src/merkleization.rs +++ b/src/merkleization.rs @@ -64,7 +64,8 @@ pub(crate) fn merkleize_impl( it = next; path.hash(&mut hasher); let (child_hash, replace); - if let Some(child) = child { + //An empty child (a dangling path) has nothing to share, and can't be replaced in its parent + if let Some(child) = child.filter(|child| !child.is_empty()) { (child_hash, replace) = merkleize_impl(counters, memo, child, val); if let Some(replace) = replace { let node = replacement.get_or_insert_with(|| { @@ -136,4 +137,18 @@ mod tests { eprintln!("```mermaid\n{}```", std::str::from_utf8(&after).unwrap()); } } + + /// Several dangling paths, which are links to the empty node + #[test] + fn merkleize_with_dangling_paths() { + use crate::zipper::*; + let mut map = crate::PathMap::::new(); + for p in [&[1u8, 1, 5][..], &[2, 2, 5], &[3, 3, 5], &[4]] { map.set_val_at(p, 1); } + for p in [&[1u8, 1, 7][..], &[2, 2, 7], &[3, 3, 7], &[1, 9], &[2, 9]] { map.create_path(p); } + let before: Vec> = { let mut z = map.read_zipper(); let mut v = vec![]; while z.to_next_step() { v.push(z.path().to_vec()) } v }; + let _ = map.merkleize(); + let after: Vec> = { let mut z = map.read_zipper(); let mut v = vec![]; while z.to_next_step() { v.push(z.path().to_vec()) } v }; + assert_eq!(before, after); + assert_eq!(map.val_count(), 4); + } } From 503a43c1b1a900b670ce87e766104e71da6f7f40 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 05:03:56 +0000 Subject: [PATCH 64/73] Fix exclusive zippers rooted at an emptied link prepare_cf returned an existing link even when it was the empty sentinel, left by remove_branches, take_map or grafting nothing. A write zipper rooted there couldn't move or read. Give it a fresh node, as for a missing link. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 13 +++++-------- src/zipper_head.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index fb5b5ed2..b6db17c1 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -734,14 +734,11 @@ impl CellByteNode { } let cf = self.get_mut(k).unwrap(); let (rec, val) = cf.both_mut_refs(); - let rec = match rec { - Some(rec) => rec, - None => { - *rec = Some(TrieNodeODRc::new_allocated_in(0, 0, alloc)); - rec.as_mut().unwrap() - } - }; - (rec, val) + //A zipper can't be rooted at the empty sentinel, so an emptied link gets a real node too + if rec.as_ref().map_or(true, |rec| rec.is_empty()) { + *rec = Some(TrieNodeODRc::new_allocated_in(0, 0, alloc)); + } + (rec.as_mut().unwrap(), val) } } diff --git a/src/zipper_head.rs b/src/zipper_head.rs index 9363b0ec..38aa0cac 100644 --- a/src/zipper_head.rs +++ b/src/zipper_head.rs @@ -1616,4 +1616,34 @@ mod tests { } } } + + /// An exclusive zipper rooted at a link emptied by `remove_branches`, `take_map` or a graft of nothing + #[test] + fn exclusive_zipper_at_emptied_link() { + let preps: [fn(&mut WriteZipperUntracked); 4] = [ + |z| { z.remove_branches(false); }, + |z| { z.take_map(false); }, + |z| { let e = PathMap::::new(); z.graft(&e.read_zipper()); }, + |z| { let e = PathMap::::new(); z.restrict(&e.read_zipper()); }, + ]; + for (i, prep) in preps.iter().enumerate() { + let mut map = PathMap::::new(); + for p in [&[0u8, 0, 1][..], &[0, 0, 1, 2], &[0]] { map.set_val_at(p, 1); } + { + let mut wz = map.write_zipper_at_path(&[0u8, 0]); + prep(&mut wz); + let zh = wz.zipper_head(); + let mut w = zh.write_zipper_at_exclusive_path(&[]).unwrap(); + assert!(!w.descend_to_existing_byte(1), "prep {i}"); + w.descend_to(&[7u8, 7]); + assert!(!w.path_exists(), "prep {i}"); + assert_eq!(w.ascend_until(), 2, "prep {i}"); + w.descend_to(&[7u8, 7]); + w.set_val(5); + } + assert_eq!(map.get(&[0u8, 0, 7, 7]), Some(&5), "prep {i}"); + assert_eq!(map.get(&[0u8]), Some(&1), "prep {i}"); + assert_eq!(map.val_count(), 2, "prep {i}"); + } + } } From 24f3125c27ce88497b2c3020cc0be6536728631f Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 05:29:14 +0000 Subject: [PATCH 65/73] Fix PrefixZipper fork rooted at the focus Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/prefix_zipper.rs | 56 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/src/prefix_zipper.rs b/src/prefix_zipper.rs index 7eb1218d..25c4244d 100644 --- a/src/prefix_zipper.rs +++ b/src/prefix_zipper.rs @@ -59,6 +59,8 @@ pub struct PrefixZipper<'prefix, Z> { prefix: Cow<'prefix, [u8]>, origin_depth: usize, position: PrefixPos, + /// The zipper's own root is off the trie, as for a fork taken past a diverged prefix + off_root: bool, } impl<'prefix, Z> PrefixZipper<'prefix, Z> @@ -83,6 +85,7 @@ impl<'prefix, Z> PrefixZipper<'prefix, Z> prefix, origin_depth: 0, position, + off_root: false, } } @@ -110,7 +113,9 @@ impl<'prefix, Z> PrefixZipper<'prefix, Z> fn set_valid(&mut self, valid: usize) { debug_assert!(valid <= self.prefix.len(), "valid prefix can't be outside prefix"); - self.position = if valid == self.prefix.len() - self.origin_depth { + self.position = if self.off_root { + PrefixPos::PrefixOff { valid: 0, invalid: 0 } + } else if valid == self.prefix.len() - self.origin_depth { PrefixPos::Source } else { PrefixPos::Prefix { valid } @@ -379,7 +384,7 @@ impl<'prefix, Z> ZipperMoving for PrefixZipper<'prefix, Z> fn at_root(&self) -> bool { match self.position { PrefixPos::Prefix { valid } => valid == 0, - PrefixPos::PrefixOff {..} => false, + PrefixPos::PrefixOff { valid, invalid } => self.off_root && valid == 0 && invalid == 0, PrefixPos::Source => self.prefix.len() <= self.origin_depth && self.source.at_root(), } } @@ -629,12 +634,19 @@ impl<'prefix, Z, V> ZipperForking for PrefixZipper<'prefix, Z> { type ReadZipperT<'a> = PrefixZipper<'prefix, Z::ReadZipperT<'a>> where Self: 'a; fn fork_read_zipper<'a>(&'a self) -> >::ReadZipperT<'a> { + //The fork is rooted at the focus: in the source, partway along the prefix, or off the trie + let (prefix, position, off_root) = match self.position { + PrefixPos::Source => (Cow::Borrowed(&[][..]), PrefixPos::Source, false), + PrefixPos::Prefix { valid } => (Cow::Owned(self.prefix[self.origin_depth + valid..].to_vec()), PrefixPos::Prefix { valid: 0 }, false), + PrefixPos::PrefixOff {..} => (Cow::Borrowed(&[][..]), PrefixPos::PrefixOff { valid: 0, invalid: 0 }, true), + }; PrefixZipper { path: Vec::new(), - position: PrefixPos::Prefix { valid: 0 }, + position, source: self.source.fork_read_zipper(), - prefix: self.prefix.clone(), + prefix, origin_depth: 0, + off_root, } } } @@ -1011,4 +1023,40 @@ mod tests { while z.to_next_step() { steps.push(z.path().to_vec()); } assert_eq!(steps, vec![vec![0], vec![0, 7], vec![0, 7, 0]]); } + + /// A fork is rooted at the focus, wherever the focus is + #[test] + fn prefix_zipper_fork_at_focus() { + use crate::zipper::ZipperForking; + let mut map = PathMap::::new(); + map.set_val_at(&[5u8], 1); + map.set_val_at(&[5u8, 6], 2); + fn steps(z: &mut Z) -> Vec> { + let mut v = vec![]; + while z.to_next_step() { v.push(z.path().to_vec()); assert!(v.len() < 16); } + v + } + let mut z = PrefixZipper::new(&[2u8, 3][..], map.read_zipper()); + for (at, want) in [ + (&[][..], vec![vec![2], vec![2, 3], vec![2, 3, 5], vec![2, 3, 5, 6]]), + (&[2u8][..], vec![vec![3], vec![3, 5], vec![3, 5, 6]]), + (&[2u8, 3, 5][..], vec![vec![6]]), + (&[9u8][..], vec![]), + ] { + z.reset(); + z.descend_to(at); + let mut f = z.fork_read_zipper(); + assert!(f.at_root(), "{at:?}"); + assert_eq!(f.path_exists(), z.path_exists(), "{at:?}"); + assert_eq!(steps(&mut f), want, "{at:?}"); + f.descend_to(&[1u8, 1]); + assert_eq!(f.ascend(5), 2, "{at:?}"); + assert!(f.at_root(), "{at:?}"); + } + //An empty prefix + let mut z = PrefixZipper::new(&[][..], map.read_zipper()); + z.set_root_prefix_path(&[]).unwrap(); + let mut f = z.fork_read_zipper(); + assert_eq!(steps(&mut f), vec![vec![5], vec![5, 6]]); + } } From 9954e445544ae162ab4728dd97be18dd8a505bdc Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 05:36:09 +0000 Subject: [PATCH 66/73] Allow empty child nodes when joining dangling slots Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/dense_byte_node.rs | 5 +++-- src/write_zipper.rs | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index b6db17c1..cafd5d55 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -2148,8 +2148,9 @@ impl, Other AlgebraicResult::None => { //Both nodes hold a dangling path (no value, no onward node) at this byte, e.g. after // `remove_branches(prune = false)`; it stays dangling in the join and is an identity for both - debug_assert!(!lv.has_rec() && !lv.has_val()); - debug_assert!(!rv.has_rec() && !rv.has_val()); + // An empty onward node counts as dangling too + debug_assert!(!lv.has_val() && lv.rec().map_or(true, |n| n.as_tagged().node_is_empty())); + debug_assert!(!rv.has_val() && rv.rec().map_or(true, |n| n.as_tagged().node_is_empty())); unsafe { new_v.get_unchecked_mut(c).write(Cf::new(None, None)) }; }, AlgebraicResult::Identity(mask) => { diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 495437e4..2b24156a 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -6774,6 +6774,21 @@ mod tests { /// (absent, or a dangling path) therefore empties an existing branch -- which then survives /// as a dangling path, exactly as after `graft` of an empty source -- and leaves an absent /// branch absent. `remove_unset` removes the unset branches outright first. + /// Joining child slots whose onward nodes are empty, as left by dropped head writers + #[test] + fn join_k_path_into_empty_child_nodes() { + let mut map = PathMap::::new(); + map.set_val_at(&[0, 0], 1); + { + let zh = map.zipper_head(); + let _w0 = zh.write_zipper_at_exclusive_path(&[0, 2]).unwrap(); + let _w1 = zh.write_zipper_at_exclusive_path(&[1, 2]).unwrap(); + } + map.write_zipper().join_k_path_into(1, false); + assert_eq!(map.get_val_at(&[0]), Some(&1)); + assert_eq!(map.val_count(), 1); + } + /// `graft_child_maps` and `graft_masked_branches` below a root path too long for one node key #[test] fn graft_child_maps_long_root() { From bd41c799dad4ba5ba5bc8f4f40750e1fd33b2f46 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 05:49:03 +0000 Subject: [PATCH 67/73] Fix use-after-free between ZipperHead readers and writers A head reader cloned an ancestor node, so the next exclusive writer copied it and left existing writers pointing into the old copy. Readers now own a private root holding only their own entry. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/zipper.rs | 74 +++++++++++++++++++++++++++++++++++++++------- src/zipper_head.rs | 30 ++++++++++++++++--- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/zipper.rs b/src/zipper.rs index 22f34027..59e67e17 100644 --- a/src/zipper.rs +++ b/src/zipper.rs @@ -1511,10 +1511,13 @@ impl<'a, V: Clone + Send + Sync + Unpin + 'a, A: Allocator + 'a> ZipperReadOnlyP } impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ReadZipperTracked<'a, 'path, V, A> { - /// See [ReadZipperCore::new_with_node_and_path] - pub(crate) fn new_with_node_and_path_in(root_node: &'a TrieNodeODRc, owned_root: bool, path: &'path [u8], root_prefix_len: usize, root_key_start: usize, root_val: Option<&'a V>, alloc: A, tracker: Option>) -> Self { - let core = ReadZipperCore::new_with_node_and_path_in(root_node, owned_root, path, root_prefix_len, root_key_start, root_val, alloc); - Self { z: core, tracker } + /// See [ReadZipperCore::new_isolated_in] + pub(crate) fn new_isolated_in(root_node: &'a TrieNodeODRc, path: &'path [u8], root_val: Option<&'a V>, alloc: A, tracker: Option>) -> Self { + Self { z: ReadZipperCore::new_isolated_in(root_node, path, root_val, alloc), tracker } + } + /// See [ReadZipperCore::new_isolated_cloned_path_in] + pub(crate) fn new_isolated_cloned_path_in(root_node: &'a TrieNodeODRc, path: &[u8], root_val: Option<&'a V>, alloc: A, tracker: Option>) -> Self { + Self { z: ReadZipperCore::new_isolated_cloned_path_in(root_node, path, root_val, alloc), tracker } } /// See [ReadZipperCore::new_with_node_and_cloned_path] pub(crate) fn new_with_node_and_cloned_path_in(root_node: &'a TrieNodeODRc, owned_root: bool, path: &[u8], root_prefix_len: usize, root_key_start: usize, root_val: Option<&'a V>, alloc: A, tracker: Option>) -> Self { @@ -2647,11 +2650,11 @@ pub(crate) mod read_zipper_core { let (_key_len, focus_node) = parent.node_get_child(self.parent_key()).unwrap(); !focus_node.is_empty() && focus_node.refcount() > 1 } else { - match &self.root_node { - OwnedOrBorrowed::Owned(root) => !root.is_empty() && root.refcount() > 1, - OwnedOrBorrowed::Borrowed(root) => !root.is_empty() && root.refcount() > 1, - OwnedOrBorrowed::None => false, - } + let focus = match &self.root_node { + OwnedOrBorrowed::None => return false, + _ => self.focus_parent(), + }; + !focus.is_empty() && focus.refcount() > 1 } } } @@ -2820,6 +2823,42 @@ pub(crate) mod read_zipper_core { new_zipper.make_static_path() } + /// Like [Self::new_with_node_and_path_in] with an owned root, but the root is a private node holding + /// only the entry at `path`. A ZipperHead reader must not share a node that live writers point into + pub(crate) fn new_isolated_in(root_node: &'a TrieNodeODRc, path: &'path [u8], root_val: Option<&'a V>, alloc: A) -> Self { + let Some((&last, parent_path)) = path.split_last() else { + return Self::new_with_node_and_path_in(root_node, true, path, 0, 0, root_val, alloc) + }; + let (parent, key, _) = node_along_path(root_node, parent_path, None, false); + let mut entry_key = key.to_vec(); + entry_key.push(last); + let parent = parent.as_tagged(); + let mut root = TrieNodeODRc::new_in(crate::dense_byte_node::DenseByteNode::new_in(alloc.clone()), alloc.clone()); + let val = parent.node_get_val(&entry_key).cloned(); + let child = parent.get_node_at_key(&entry_key).into_option(); + let dangling = val.is_none() && child.is_none() && parent.node_contains_partial_key(&entry_key); + if let Some(val) = val { + if let Err(n) = root.make_mut().node_set_val(&[last], val) { root = n } + } + if let Some(child) = child { + if let Err(n) = root.make_mut().node_set_branch(&[last], child) { root = n } + } + if dangling { + if let Err(n) = root.make_mut().node_create_dangling(&[last]) { root = n } + } + //The root value is read from `root`, via `root_parent_key_start` + Self::new_with_node_and_path_internal_in(OwnedOrBorrowed::Owned(root), path, path.len() - 1, None, alloc) + } + /// Same as [Self::new_isolated_in], but with a `'static` path + pub(crate) fn new_isolated_cloned_path_in(root_node: &'a TrieNodeODRc, path: &[u8], root_val: Option<&'a V>, alloc: A) -> ReadZipperCore<'a, 'static, V, A> { + let mut new_zipper = ReadZipperCore::<'a, '_, V, A>::new_isolated_in(root_node, path, root_val, alloc); + new_zipper.prefix_buf = Vec::with_capacity(EXPECTED_PATH_LEN); + new_zipper.prefix_buf.extend(path); + new_zipper.origin_path = SliceOrLen::new_owned(path.len()); + new_zipper.ancestors = Vec::with_capacity(EXPECTED_DEPTH); + new_zipper.make_static_path() + } + /// Makes a version of `self` that has an allocated path buffer and a `'static`` path lifetime #[inline] pub(crate) fn make_static_path(mut self) -> ReadZipperCore<'a, 'static, V, A> { @@ -2902,7 +2941,12 @@ pub(crate) mod read_zipper_core { // we currently share the same implementation between `val()` and `get_val()` because the only difference is the return // lifetime, and the current ZipperHead implementation is actually ok with referencing the value in the root of the ZipperHead. // debug_assert!(self.root_node.is_borrowed()); - self.root_val + if self.root_val.is_some() || self.root_parent_key_start == usize::MAX || !self.root_node.is_owned() { + self.root_val + } else { + //SAFETY: see the note on this method + self.root_node.as_ref().as_tagged().node_get_val(self.root_node_key()).map(|v| unsafe{ &*(v as *const V) }) + } } } } @@ -3031,6 +3075,10 @@ pub(crate) mod read_zipper_core { if parent_key.len() == 0 { return self.root_node.as_ref() } + if self.ancestors.is_empty() { + //At the root, with the focus on a child of the root node + return self.root_node.as_ref().as_tagged().node_get_child(parent_key).unwrap().1 + } self.focus_parent_borrowed() } @@ -3253,8 +3301,10 @@ pub(crate) mod read_zipper_core { } else { if let Some((parent, _iter_tok, _prefix_offset)) = self.ancestors.last() { parent.node_contains_val(self.parent_key()) - } else { + } else if self.root_val.is_some() || self.root_parent_key_start == usize::MAX || !self.root_node.is_owned() { self.root_val.is_some() + } else { + self.root_node.as_ref().as_tagged().node_contains_val(self.root_node_key()) } } } @@ -3341,6 +3391,8 @@ pub(crate) mod read_zipper_core { if self.prefix_buf.len() > 0 { let key_start = if self.ancestors.len() > 1 { unsafe{ self.ancestors.get_unchecked(self.ancestors.len()-2) }.2 + } else if self.ancestors.is_empty() && self.root_parent_key_start != usize::MAX { + self.root_parent_key_start } else { self.root_key_start }; diff --git a/src/zipper_head.rs b/src/zipper_head.rs index 38aa0cac..6ef68575 100644 --- a/src/zipper_head.rs +++ b/src/zipper_head.rs @@ -209,7 +209,7 @@ impl<'trie, Z, V: 'trie + Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zip // logic makes sure conflicting paths aren't permitted, so we should not get aliased &mut borrows let root_node: &'trie TrieNodeODRc = unsafe{ core::mem::transmute(root_node) }; let root_val: Option<&'trie V> = root_val.map(|v| unsafe{ &*v.as_ptr() } ); - let new_zipper = ReadZipperTracked::new_with_node_and_path_in(root_node, true, path.as_ref(), path.len(), 0, root_val, z.alloc.clone(), zipper_tracker); + let new_zipper = ReadZipperTracked::new_isolated_in(root_node, path, root_val, z.alloc.clone(), zipper_tracker); Ok(new_zipper) }) } @@ -229,7 +229,7 @@ impl<'trie, Z, V: 'trie + Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zip #[cfg(not(debug_assertions))] let zipper_tracker = None; - ReadZipperTracked::new_with_node_and_path_in(root_node, true, path.as_ref(), path.len(), 0, root_val, z.alloc.clone(), zipper_tracker) + ReadZipperTracked::new_isolated_in(root_node, path, root_val, z.alloc.clone(), zipper_tracker) }) } fn read_zipper_at_path<'a, K: AsRef<[u8]>>(&'a self, path: K) -> Result, Conflict> where 'trie: 'a { @@ -242,7 +242,7 @@ impl<'trie, Z, V: 'trie + Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zip let root_node: &'trie TrieNodeODRc = unsafe{ core::mem::transmute(root_node) }; let root_val: Option<&'trie V> = root_val.map(|v| unsafe{ &*v.as_ptr() } ); - let new_zipper = ReadZipperTracked::new_with_node_and_cloned_path_in(root_node, true, path.as_ref(), path.len(), 0, root_val, z.alloc.clone(), Some(zipper_tracker)); + let new_zipper = ReadZipperTracked::new_isolated_cloned_path_in(root_node, path, root_val, z.alloc.clone(), Some(zipper_tracker)); Ok(new_zipper) }) } @@ -263,7 +263,7 @@ impl<'trie, Z, V: 'trie + Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zip #[cfg(not(debug_assertions))] let zipper_tracker = None; - ReadZipperTracked::new_with_node_and_cloned_path_in(root_node, true, path.as_ref(), path.len(), 0, root_val, z.alloc.clone(), zipper_tracker) + ReadZipperTracked::new_isolated_cloned_path_in(root_node, path, root_val, z.alloc.clone(), zipper_tracker) }) } fn write_zipper_at_exclusive_path<'a, K: AsRef<[u8]>>(&'a self, path: K) -> Result, Conflict> where 'trie: 'a { @@ -1564,6 +1564,28 @@ mod tests { assert_eq!(map.get_val_at(&[1u8, 2, 3]), Some(&7)); } + /// A reader must not hold a node that live writers point into + #[test] + fn head_reader_beside_live_writers() { + let mut map = PathMap::::new(); + map.set_val_at(&[0u8, 0], 1); + let zh = map.into_zipper_head(&[]); + { + let mut w1 = zh.write_zipper_at_exclusive_path(&[0x11u8]).unwrap(); + //Not in the trie, so it used to hold the root node, which the next writer then copied + let r0 = zh.read_zipper_at_path(&[0x22u8, 0, 0]).unwrap(); + let w0 = zh.write_zipper_at_exclusive_path(&[0u8]).unwrap(); + assert!(!r0.path_exists()); + drop(r0); + w1.set_val(5); + drop(w0); + drop(w1); + } + let map = zh.into_map(); + assert_eq!(map.get_val_at(&[0x11u8]), Some(&5)); + assert_eq!(map.get_val_at(&[0u8, 0]), Some(&1)); + } + /// `get_trie_ref`, `get_focus` and forks from a head's read zipper, which owns its root node #[test] fn head_read_zipper_trie_refs() { From bda1eac4e669695d310679d3427e7fe55030cb25 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 05:49:36 +0000 Subject: [PATCH 68/73] crash_fuzz: rerun signal crashes alone, trace more Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/bin/crash_fuzz.rs | 11 +++++- differential/src/crash.rs | 55 ++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/differential/src/bin/crash_fuzz.rs b/differential/src/bin/crash_fuzz.rs index ebfa2d7a..6dee5e02 100644 --- a/differential/src/bin/crash_fuzz.rs +++ b/differential/src/bin/crash_fuzz.rs @@ -416,8 +416,17 @@ fn supervise(args: &Args, source: Source) { None => eprintln!(" could not locate"), } } + for c in r.crashes.iter_mut().filter(|c| (c.kind == "signal" || c.kind == "abort") && c.idx != usize::MAX) { + //Threads share one heap, so memory corrupted by one input can abort another. Only + // blame an input that fails on its own. + let alone = run_child(args, c.idx, c.idx + 1, 1, &[], false); + if alone.completed { + eprintln!(" input {} runs clean alone; not attributing the {}", c.idx, c.kind); + c.kind = format!("{} (not reproduced alone)", c.kind); + } + } let progressed = !r.crashes.is_empty() || r.completed; - for c in r.crashes.iter().filter(|c| c.kind != "panic" && c.kind != "hang" && c.idx != usize::MAX) { + for c in r.crashes.iter().filter(|c| (c.kind == "signal" || c.kind == "abort") && c.idx != usize::MAX) { // The child saves what its hooks report; a signal or an abort leaves it to us. if let Some(dir) = &save { let _ = std::fs::create_dir_all(dir); diff --git a/differential/src/crash.rs b/differential/src/crash.rs index 13d70415..8ffd620b 100644 --- a/differential/src/crash.rs +++ b/differential/src/crash.rs @@ -191,6 +191,7 @@ fn path(d: &mut Dec) -> Option> { fn short_path(d: &mut Dec) -> Option> { let mut p = path(d)?; p.truncate(8); + note!(" short_path {}", hex_path(&p)); Some(p) } @@ -210,7 +211,9 @@ fn mask(d: &mut Dec) -> Option { } fn val(d: &mut Dec) -> Option { - Some(V::from_byte(d.u8()?)) + let b = d.u8()?; + note!(" val {b}"); + Some(V::from_byte(b)) } /// Result of a map-level lattice operation, as a map. @@ -390,7 +393,9 @@ where /// zipper, and iteration that hands them out. macro_rules! ro_extras { ($d:expr, $z:expr) => {{ - match $d.modn(4)? { + let __e = $d.modn(4)?; + note!("ro_extras {__e}"); + match __e { 0 => { let _ = $z.get_val().cloned(); } 1 => { let p = path($d)?; let _ = $z.get_val_at(&p).cloned(); } 2 => { let mut n = 0; while n < WALK && $z.to_next_get_val().is_some() { n += 1; } } @@ -403,7 +408,9 @@ macro_rules! ro_extras { /// focus borrowing, forks, trie refs, buffer management. macro_rules! sub_extras { ($d:expr, $steps:expr, $z:expr) => {{ - match $d.modn(12)? { + let __e = $d.modn(12)?; + note!("sub_extras {__e}"); + match __e { 0 => { let w = $z.witness(); let _ = $z.get_val_with_witness(&w).cloned(); } 1 => { let w = $z.witness(); @@ -457,7 +464,7 @@ where V: CrashValue, W: ZipperWriting + ZipperMoving + ZipperPath + ZipperValues, { - let src = |d: &mut Dec| -> Option<&PathMap> { Some(&srcs[d.modn(NMAPS)?]) }; + let src = |d: &mut Dec| -> Option<&PathMap> { let i = d.modn(NMAPS)?; note!(" src map {i}"); Some(&srcs[i]) }; let op = d.modn(40)?; note!("write {op} at {}", hex_path(z.path())); // Growth is refused below an oversized focus; everything else still runs. @@ -495,11 +502,12 @@ where let s = src(d)?.clone(); let p = short_path(d)?; let pr = d.boolean()?; + note!(" pr={pr}"); let mut sw = s.into_write_zipper(&p); let _ = z.join_into_take(&mut sw, pr); let _ = sw.into_map().val_count(); } - 20 => { let k = kpath_k(d, 6)?; let pr = d.boolean()?; let _ = z.join_k_path_into(k, pr); } + 20 => { let k = kpath_k(d, 6)?; let pr = d.boolean()?; note!(" k={k} pr={pr}"); let _ = z.join_k_path_into(k, pr); } 21 => { let k = d.modn(6)?; let pr = d.boolean()?; @@ -568,6 +576,19 @@ fn snapshot(maps: &[PathMap; NMAPS]) -> [PathMap; NMAPS] { // Episodes: one zipper kind, created, driven, dropped // --------------------------------------------------------------------------- +/// Traces the paths holding values in each map. +fn note_maps(maps: &[PathMap; NMAPS]) { + if *TRACE.get_or_init(|| std::env::var_os("CRASH_TRACE").is_some()) { + for (i, mp) in maps.iter().enumerate() { + let mut rz = mp.read_zipper(); + let mut ps = vec![]; + if rz.is_val() { ps.push("_".to_string()); } + while rz.to_next_val() { ps.push(hex_path(rz.path())); } + note!(" map {i}: {}", ps.join(" ")); + } + } +} + fn read_episode(d: &mut Dec, st: &mut State) -> Option<()> { let m = d.modn(NMAPS)?; let p = short_path(d)?; @@ -575,6 +596,7 @@ fn read_episode(d: &mut Dec, st: &mut State) -> Option<()> { let steps = &mut st.steps; let __kind = d.modn(12)?; note!("read episode {__kind} map {m} at {}", hex_path(&p)); + note_maps(&st.maps); match __kind { 0 => { let mut z = map.read_zipper_at_path(&p); @@ -637,16 +659,22 @@ fn read_episode(d: &mut Dec, st: &mut State) -> Option<()> { } 6 => { // Secondary factors are map-root zippers: see KNOWN_PRECONDITIONS. - let others: Vec> = (0..d.modn(4)?).map(|_| d.modn(NMAPS).map(|i| st.maps[i].clone())).collect::>()?; - let more = st.maps[d.modn(NMAPS)?].clone(); + let pick: Vec = (0..d.modn(4)?).map(|_| d.modn(NMAPS)).collect::>()?; + let others: Vec> = pick.iter().map(|&i| st.maps[i].clone()).collect(); + let more_i = d.modn(NMAPS)?; + note!(" product factors {pick:?}, more {more_i}"); + let more = st.maps[more_i].clone(); let steps = &mut st.steps; let mut z = ProductZipper::new(map.read_zipper_at_path(&p), others.iter().map(|o| o.read_zipper())); if d.boolean()? { + note!(" new_factors"); z.new_factors([more.read_zipper()]); } for _ in 0..d.modn(8)? { tick(steps)?; - match d.modn(4)? { + let __c = d.modn(4)?; + note!(" product op {__c} at {}", hex_path(z.path())); + match __c { 0 => { let _ = (z.focus_factor(), z.factor_count(), z.path_indices().len()); } 1 => { let w = z.witness(); let _ = z.get_val_with_witness(&w).cloned(); } 2 => { let _ = (z.is_shared(), z.shared_node_id(), z.origin_path().len()); } @@ -743,6 +771,7 @@ fn write_episode(d: &mut Dec, st: &mut State) -> Option<()> { let State { maps, steps } = st; let __kind = d.modn(7)?; note!("write episode {__kind} map {m} at {}", hex_path(&p)); + note_maps(maps); match __kind { 0 => { let mut z = maps[m].write_zipper_at_path(&p); @@ -815,16 +844,19 @@ where let mut w1 = zh.write_zipper_at_exclusive_path(&paths[1]).ok(); let mut r0 = zh.read_zipper_at_path(&paths[2]).ok(); let mut r1 = zh.read_zipper_at_borrowed_path(&paths[2]).ok(); + note!(" head zippers: w0 {} {}, w1 {} {}, r0/r1 {} {} {}", hex_path(&paths[0]), w0.is_some(), hex_path(&paths[1]), w1.is_some(), hex_path(&paths[2]), r0.is_some(), r1.is_some()); let n = d.modn(EPISODE_STEPS)?; for _ in 0..n { tick(steps)?; - match d.modn(7)? { + let __which = d.modn(7)?; + note!(" head step {__which}"); + match __which { 0 => if let Some(z) = w0.as_mut() { write_step(d, z, srcs)? }, 1 => if let Some(z) = w1.as_mut() { write_step(d, z, srcs)? }, 2 => if let Some(z) = r0.as_mut() { iter_step(d, z)? }, 3 => if let Some(z) = r1.as_mut() { if d.boolean()? { sub_extras!(d, steps, z) } else { iter_step(d, z)? } }, - 4 => { drop(w0.take()); let q = short_path(d)?; w0 = zh.write_zipper_at_exclusive_path(&q).ok(); } - 5 => { drop(r0.take()); let q = short_path(d)?; r0 = zh.read_zipper_at_path(&q).ok(); } + 4 => { drop(w0.take()); let q = short_path(d)?; w0 = zh.write_zipper_at_exclusive_path(&q).ok(); note!(" w0 = {} {}", hex_path(&q), w0.is_some()); } + 5 => { drop(r0.take()); let q = short_path(d)?; r0 = zh.read_zipper_at_path(&q).ok(); note!(" r0 = {} {}", hex_path(&q), r0.is_some()); } _ => { w1 = None; r1 = None; } } } @@ -1037,6 +1069,7 @@ fn seed(d: &mut Dec) -> Option> { for _ in 0..d.modn(12)? { let p = path(d)?; let v = val(d)?; + note!("seed map {m}: {} = {v:?}", hex_path(&p)); st.maps[m].set_val_at(&p, v); } } From 9b8d9ad61f77eff993c7c2fb77b33a004a40bc81 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 05:59:28 +0000 Subject: [PATCH 69/73] Fix ProductZipper sibling step failing at a factor root Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/product_zipper.rs | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/product_zipper.rs b/src/product_zipper.rs index 0636eaf9..75a6952f 100644 --- a/src/product_zipper.rs +++ b/src/product_zipper.rs @@ -290,7 +290,10 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper fn to_next_sibling_byte(&mut self) -> Option { //Stepping sideways leaves a factor entered at this depth, but at the root there's no sideways if self.depth() > 0 && self.factor_paths.last().cloned() == Some(self.depth()) { + //Take the factor's root off the core zipper too; a failed step enters it again below self.factor_paths.pop(); + self.z.deregularize(); + self.z.regularize(); } let moved = self.z.to_next_sibling_byte(); self.ensure_descend_next_factor(); @@ -299,7 +302,10 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper fn to_prev_sibling_byte(&mut self) -> Option { //Stepping sideways leaves a factor entered at this depth, but at the root there's no sideways if self.depth() > 0 && self.factor_paths.last().cloned() == Some(self.depth()) { + //Take the factor's root off the core zipper too; a failed step enters it again below self.factor_paths.pop(); + self.z.deregularize(); + self.z.regularize(); } let moved = self.z.to_prev_sibling_byte(); self.ensure_descend_next_factor(); @@ -2084,6 +2090,46 @@ mod tests { assert_eq!(z.path(), &[] as &[u8]); } + /// A failed sibling step at the root of a factor stays in that factor + #[test] + fn product_zipper_no_sibling_at_factor_root() { + let mut a = PathMap::::new(); + a.set_val_at(&[0u8, 0], 1); + let mut b = PathMap::::new(); + b.set_val_at(&[5u8], 2); + for prev in [false, true] { + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper()]); + assert!(z.to_next_val()); + assert_eq!(z.path(), &[0, 0]); + let moved = if prev { z.to_prev_sibling_byte() } else { z.to_next_sibling_byte() }; + assert_eq!(moved, None); + assert_eq!(z.child_count(), 1); + let _ = (z.is_shared(), z.shared_node_id()); + assert!(z.to_next_val()); + assert_eq!((z.path(), z.val()), (&[0u8, 0, 5][..], Some(&2))); + } + } + + /// Sibling steps out of a factor entered below an empty node, as a dropped head writer leaves + #[test] + fn product_zipper_k_path_past_empty_node() { + let mut a = PathMap::::new(); + for p in [&[0u8][..], &[0, 0, 0], &[1]] { a.set_val_at(p, 0); } + { + let zh = a.zipper_head(); + zh.write_zipper_at_exclusive_path(&[1u8, 0]).unwrap().set_val(0); + let _w = zh.write_zipper_at_exclusive_path(&[0u8, 0, 0, 0]).unwrap(); + } + let b = a.clone(); + let mut z = ProductZipper::new(a.read_zipper(), [b.read_zipper()]); + let mut paths = vec![]; + if z.descend_first_k_path(3) { + paths.push(z.path().to_vec()); + while paths.len() < 64 && z.to_next_k_path(3) { paths.push(z.path().to_vec()); } + } + assert!(paths.iter().all(|p| p.len() == 3), "{paths:?}"); + } + /// `is_shared` and `shared_node_id` across factor boundaries #[test] fn product_zipper_is_shared_across_factors() { From 460ff78c3b06ddf02f4199be2e01cd0818e9b2ac Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 06:40:11 +0000 Subject: [PATCH 70/73] Fix TrieRef::is_shared on an empty node Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/trie_ref.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/trie_ref.rs b/src/trie_ref.rs index 12bf0afe..3e781e8c 100644 --- a/src/trie_ref.rs +++ b/src/trie_ref.rs @@ -452,7 +452,7 @@ impl ZipperConcrete for TrieRefBor } fn is_shared(&self) -> bool { match self.focus_node { - Some(node) => self.node_key().is_empty() && node.refcount() > 1, + Some(node) => self.node_key().is_empty() && !node.is_empty() && node.refcount() > 1, None => false, } } @@ -779,7 +779,7 @@ impl ZipperConcrete for TrieRefOwn } fn is_shared(&self) -> bool { match &self.focus_node { - Some(node) => self.node_key().is_empty() && node.refcount() > 1, + Some(node) => self.node_key().is_empty() && !node.is_empty() && node.refcount() > 1, None => false } } @@ -1465,4 +1465,27 @@ mod tests { assert_eq!(rz.val_at(&[1u8]), None); assert_eq!(rz.val_at(&[7u8; 60]), None); } + + /// `is_shared` where the focus is the empty sentinel node + #[test] + fn trie_ref_is_shared_on_empty_node() { + let mut map = PathMap::::new(); + map.set_val_at(&[1u8], 1); + map.remove_val_at(&[1u8], false); + let empty = PathMap::::new(); + for m in [&map, &empty] { + for path in [&[][..], &[1u8][..]] { + let t = m.trie_ref_at_path(path); + let _ = (t.is_shared(), t.shared_node_id()); + } + } + let mut src = PathMap::::new(); + src.set_val_at(&[2u8, 3], 1); + { + let mut wz = src.write_zipper_at_path(&[2u8]); + wz.remove_branches(false); + } + let t = src.trie_ref_at_path(&[2u8]); + let _ = (t.is_shared(), t.shared_node_id()); + } } From 9e15a70372a5f5a85b78b7abae6e948077486919 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 06:53:56 +0000 Subject: [PATCH 71/73] Fix list node join of two empty children under one key Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- src/line_list_node.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/line_list_node.rs b/src/line_list_node.rs index 6b027ae9..3b309611 100644 --- a/src/line_list_node.rs +++ b/src/line_list_node.rs @@ -1394,7 +1394,11 @@ fn merge_guts<'a, V: Clone + Lattice + Send + Sync, A: Allocator, const ASLOT: u (true, true) => { //both are child nodes, so join them let a_child = unsafe{ a.child_in_slot::() }; let b_child = unsafe{ b.child_in_slot::() }; - return a_child.pjoin(b_child).map(|new_child| (a_key, ValOrChild::Child(new_child))) + return match a_child.pjoin(b_child) { + //Two empty children are both just the dangling path + AlgebraicResult::None => AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT), + joined => joined.map(|new_child| (a_key, ValOrChild::Child(new_child))), + } }, (false, false) => { //both are values, so join them let a_val = unsafe{ a.val_in_slot::() }; @@ -4125,4 +4129,20 @@ mod tests { into.write_zipper().join_map_into(cell.clone()); assert_eq!(into.iter().map(|(k, v)| (k, *v)).collect::>(), vals); } + + /// Joining two nodes whose children under the same key are both empty + #[test] + fn merge_empty_children_under_same_key() { + let node = || { + let mut n = LineListNode::::new_in(global_alloc()); + let child = LineListNode::::new_in(global_alloc()); + unsafe { n.set_child_0(&[0, 3], TrieNodeODRc::new_in(child, global_alloc())); } + n + }; + let (a, b) = (node(), node()); + match merge_list_nodes(&a, &b) { + Ok(AlgebraicResult::Identity(mask)) => assert_eq!(mask, SELF_IDENT | COUNTER_IDENT), + _ => panic!("expected an identity"), + } + } } From 6c4e2e57f47c197faf907d68f57d3f7a9dba889c Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 06:53:56 +0000 Subject: [PATCH 72/73] crash_fuzz: trace dangling paths, masks and paths Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/src/crash.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/differential/src/crash.rs b/differential/src/crash.rs index 8ffd620b..6652f80c 100644 --- a/differential/src/crash.rs +++ b/differential/src/crash.rs @@ -184,6 +184,7 @@ fn path(d: &mut Dec) -> Option> { _ => b, }); } + note!(" path {}", hex_path(&v)); Some(v) } @@ -207,6 +208,7 @@ fn mask(d: &mut Dec) -> Option { for _ in 0..n { m.set_bit(byte(d)?); } + note!(" mask {:?}", m.iter().collect::>()); Some(m) } @@ -474,7 +476,7 @@ where } match op { 0 => { let v = val(d)?; let _ = z.set_val(v); } - 1 => { let pr = d.boolean()?; let _ = z.remove_val(pr); } + 1 => { let pr = d.boolean()?; note!(" pr={pr}"); let _ = z.remove_val(pr); } 2 => { let v = val(d)?; if let Some(slot) = z.get_val_mut() { *slot = v; } } 3 => { let v = val(d)?; let _ = z.get_val_or_set_mut(v).clone(); } 4 => { let v = val(d)?; let _ = z.get_val_or_set_mut_with(|| v).clone(); } @@ -529,7 +531,7 @@ where } } 27 => { let m = mask(d)?; let pr = d.boolean()?; z.remove_unmasked_branches(m, pr); } - 28 => { let s = src(d)?; let p = short_path(d)?; let m = mask(d)?; let ru = d.boolean()?; z.graft_masked_branches(&s.read_zipper_at_path(&p), m, ru); } + 28 => { let s = src(d)?; let p = short_path(d)?; let m = mask(d)?; let ru = d.boolean()?; note!(" ru={ru}"); z.graft_masked_branches(&s.read_zipper_at_path(&p), m, ru); } 29 => { let s = src(d)?; let m = mask(d)?; @@ -583,7 +585,10 @@ fn note_maps(maps: &[PathMap; NMAPS]) { let mut rz = mp.read_zipper(); let mut ps = vec![]; if rz.is_val() { ps.push("_".to_string()); } - while rz.to_next_val() { ps.push(hex_path(rz.path())); } + //Values, and the ends of dangling paths marked `~` + while ps.len() < 64 && rz.to_next_step() { + if rz.is_val() { ps.push(hex_path(rz.path())) } else if rz.child_count() == 0 { ps.push(format!("{}~", hex_path(rz.path()))) } + } note!(" map {i}: {}", ps.join(" ")); } } From ee7546e7148874c9b68c67fd3658f6e8a3176783 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Thu, 17 Sep 2026 07:04:37 +0000 Subject: [PATCH 73/73] CRASH_FINDINGS: note the fuzz-fixes-v3 fixes Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019R2H8fnco29asY2v3TPbtF --- differential/CRASH_FINDINGS.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/differential/CRASH_FINDINGS.md b/differential/CRASH_FINDINGS.md index 32db7e93..fa1a56ba 100644 --- a/differential/CRASH_FINDINGS.md +++ b/differential/CRASH_FINDINGS.md @@ -2,8 +2,8 @@ Failures `crash_fuzz` found in `pathmap` on 2026-09-16, at commit `7c61024` on `fuzz-fixes-v2`. The op table is `differential/src/crash.rs`; see -`differential/src/bin/crash_fuzz.rs` for the runner. Nothing here has been -fixed. +`differential/src/bin/crash_fuzz.rs` for the runner. The sites below are as +found; see "Fixed on `fuzz-fixes-v3`" for where they stand now. Each finding is an input that makes the crate panic, fail a debug assertion, abort, segfault or hang **through its public API**, with every documented @@ -11,6 +11,27 @@ precondition met. Calls the crate documents as panicking, stubs, and failures already known from the differential work are steered around unless `--include-known` is passed; they are listed at the end. +## Fixed on `fuzz-fixes-v3` + +No site outside "Known failures" reproduces on `fuzz-fixes-v3` (fixes on +`fuzz-fixes-v2` and `v3`, each with a test that fails without it). These +turned up only once the earlier ones were gone: + +- A `ZipperHead` reader cloned an ancestor node, so the next exclusive writer + copied that node and left live writers pointing into the old copy + (use-after-free). Readers now own a private root holding only their entry. + This was behind findings 3, 4 and 8 and the "Lock is missing" panic in + `zipper_tracking.rs`. +- `PrefixZipper::fork_read_zipper` always forked from the prefix start. +- `ProductZipper` sibling steps that fail at a factor root lost the factor. +- `TrieRef::is_shared` read the refcount of the empty sentinel. +- Joins of two empty child nodes under one key: a dense-node debug assertion + and a list node left with two onward children. + +After the fixes, with the debug-assertion build and known failures steered +around: 0 failures in 4M inputs (seed 36) and in 8M inputs with `--maxlen +2000` (seed 39). With `--include-known`, only the known failures remain. + ## How the surveys were run | Survey | Build | Inputs | Seed | Time | Failures | Sites |