From 9e74bfa65ed24738ae31c72ee838dee2db5cd49b Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 06:20:07 -0400 Subject: [PATCH 01/21] feat(storage-types): add BlockNumberList::pop_max Used by the upcoming merge_and_split helper to un-push the overflow block when splitting a shard. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/types/src/int_list.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/types/src/int_list.rs b/crates/types/src/int_list.rs index dde46bc..7d2bbd0 100644 --- a/crates/types/src/int_list.rs +++ b/crates/types/src/int_list.rs @@ -146,6 +146,13 @@ impl IntegerList { self.0.max() } + /// Remove and return the largest value, or `None` if the list is empty. + pub fn pop_max(&mut self) -> Option { + let m = self.0.max()?; + self.0.remove(m); + Some(m) + } + /// Returns the number of integers that are `<= value`. pub fn rank(&self, value: u64) -> u64 { self.0.rank(value) @@ -166,3 +173,31 @@ impl IntegerList { self.0.serialize_into(writer) } } + +#[cfg(test)] +mod tests { + use super::IntegerList; + + #[test] + fn pop_max_returns_and_removes_largest() { + let mut list = IntegerList::new([3u64, 7, 9, 12]).unwrap(); + assert_eq!(list.pop_max(), Some(12)); + assert_eq!(list.max(), Some(9)); + assert_eq!(list.len(), 3); + } + + #[test] + fn pop_max_on_empty_returns_none() { + let mut list = IntegerList::empty(); + assert_eq!(list.pop_max(), None); + assert!(list.is_empty()); + } + + #[test] + fn pop_max_drains_to_empty() { + let mut list = IntegerList::new([42u64]).unwrap(); + assert_eq!(list.pop_max(), Some(42)); + assert!(list.is_empty()); + assert_eq!(list.pop_max(), None); + } +} From 43ecb7b6697a34c3dd0bbeee1c21ab9c48c6f10a Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 06:25:50 -0400 Subject: [PATCH 02/21] feat(storage-types): add merge_and_split for size-bounded history shards Pure function: takes ownership of (existing, additions), merges them, and splits off a tail iff the merged list exceeds max_bytes after roaring encoding. Single linear pass; allocates the tail shard only when a split is required. This is the backend-agnostic primitive that signet-hot-mdbx's HistoryWrite impl will use to maintain its size-bounded DUPSORT shards. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/types/src/int_list.rs | 121 ++++++++++++++++++++++++++++++++++- crates/types/src/lib.rs | 2 +- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/crates/types/src/int_list.rs b/crates/types/src/int_list.rs index 7d2bbd0..083e15a 100644 --- a/crates/types/src/int_list.rs +++ b/crates/types/src/int_list.rs @@ -174,9 +174,60 @@ impl IntegerList { } } +/// Append `additions` to `existing`, splitting off a tail shard iff the +/// merged list exceeds `max_bytes` after roaring encoding. +/// +/// Returns `(first, second)`: +/// - `first`: always returned; contains the lower block numbers. The caller +/// writes this with subkey = `first.max()` if `second.is_some()`, otherwise +/// `u64::MAX`. +/// - `second`: `Some(tail)` iff a split occurred. Caller writes with subkey +/// `u64::MAX`. +/// +/// Preconditions (caller's responsibility): +/// - Every block in `additions` is strictly greater than every block in +/// `existing`. The function does not sort or deduplicate. +/// - `existing.serialized_size() <= max_bytes`. +/// +/// Postconditions: +/// - `first.serialized_size() <= max_bytes`. +/// - If `second.is_some()`, `second.serialized_size() <= max_bytes` and +/// `second.min() > first.max()`. +/// - `first ∪ second == existing_before ∪ additions`. +/// +/// Allocation: zero on the no-split fast path beyond what +/// `IntegerList::push` already does for roaring container growth. One +/// `IntegerList` allocation when a split occurs. +pub fn merge_and_split( + mut existing: IntegerList, + additions: IntegerList, + max_bytes: usize, +) -> (IntegerList, Option) { + let mut tail: Option = None; + + for block in additions.iter() { + if let Some(t) = tail.as_mut() { + t.push(block).expect("strictly increasing"); + continue; + } + existing.push(block).expect("strictly increasing"); + if existing.serialized_size() > max_bytes { + // Overflow: pop the just-pushed block out of `existing` and + // start the tail shard with it as the seed. + let popped = existing.pop_max().expect("just pushed"); + debug_assert_eq!(popped, block); + let mut t = IntegerList::empty(); + t.push(block).expect("first push always succeeds"); + tail = Some(t); + } + } + + (existing, tail) +} + #[cfg(test)] mod tests { - use super::IntegerList; + use super::{IntegerList, merge_and_split}; #[test] fn pop_max_returns_and_removes_largest() { @@ -200,4 +251,72 @@ mod tests { assert!(list.is_empty()); assert_eq!(list.pop_max(), None); } + + #[test] + fn merge_and_split_no_split_when_under_budget() { + let existing = IntegerList::new([1u64, 2, 3]).unwrap(); + let additions = IntegerList::new([4u64, 5]).unwrap(); + // 1500 B is generous for 5 dense values + let (first, second) = merge_and_split(existing, additions, 1500); + assert_eq!(first.iter().collect::>(), vec![1, 2, 3, 4, 5]); + assert!(second.is_none()); + } + + #[test] + fn merge_and_split_no_additions_returns_existing() { + let existing = IntegerList::new([10u64, 20]).unwrap(); + let additions = IntegerList::empty(); + let (first, second) = merge_and_split(existing, additions, 1500); + assert_eq!(first.iter().collect::>(), vec![10, 20]); + assert!(second.is_none()); + } + + #[test] + fn merge_and_split_splits_when_over_budget() { + // Provoke a split with a deliberately small budget. Use 100 contiguous + // values starting at 0. Roaring-encoded that's a single run container + // around 14 B, so we need a tiny budget to force a split. Set budget + // small enough that ~50 entries push us over. + let existing = IntegerList::new(0u64..50).unwrap(); + let additions = IntegerList::new(50u64..100).unwrap(); + + // Compute the budget so existing alone fits but existing + additions + // doesn't. + let existing_size = existing.serialized_size(); + let combined = IntegerList::new(0u64..100).unwrap().serialized_size(); + assert!(combined > existing_size, "test setup broken: combined didn't grow"); + let budget = existing_size + (combined - existing_size) / 2; + + let (first, second) = merge_and_split(existing, additions, budget); + let second = second.expect("split should have occurred"); + + // first ∪ second == 0..100, with second's min > first's max. + let mut all: Vec = first.iter().collect(); + all.extend(second.iter()); + assert_eq!(all, (0u64..100).collect::>()); + assert!(first.max().unwrap() < second.min().unwrap()); + + // Both halves fit in budget. + assert!(first.serialized_size() <= budget); + assert!(second.serialized_size() <= budget); + } + + #[test] + fn merge_and_split_split_preserves_strict_ordering() { + // Additions are strictly greater than existing; verify post-split second + // contains only the newest blocks. + let existing = IntegerList::new(0u64..10).unwrap(); + let additions = IntegerList::new(10u64..30).unwrap(); + + let existing_size = existing.serialized_size(); + let combined = IntegerList::new(0u64..30).unwrap().serialized_size(); + let budget = existing_size + (combined - existing_size) / 3; + + let (first, second) = merge_and_split(existing, additions, budget); + let second = second.expect("split should have occurred"); + + let first_max = first.max().unwrap(); + let second_min = second.min().unwrap(); + assert!(first_max < second_min, "first.max={first_max} second.min={second_min}",); + } } diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 4924da9..0f6a128 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -23,7 +23,7 @@ pub use events::{DbSignetEvent, DbZenithHeader}; mod indexed_receipt; pub use indexed_receipt::IndexedReceipt; mod int_list; -pub use int_list::{BlockNumberList, IntegerList, IntegerListError}; +pub use int_list::{BlockNumberList, IntegerList, IntegerListError, merge_and_split}; mod sharded; pub use sharded::ShardedKey; pub use signet_evm::{Account, EthereumHardfork, genesis_header}; From d28f3cdf9fc7755392f468d0d9db4134d05b8bbe Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 06:32:11 -0400 Subject: [PATCH 03/21] fixup(storage-types): restore debug_assert on merge_and_split precondition The spec mandates a debug-only check that additions fit within max_bytes. Test 4 (merge_and_split_split_preserves_strict_ordering) was updated to use sparse input data (one element per 16-bit roaring container) so encoded sizes grow proportionally to count, keeping additions within budget while still provoking a split. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/types/src/int_list.rs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/types/src/int_list.rs b/crates/types/src/int_list.rs index 083e15a..e512a44 100644 --- a/crates/types/src/int_list.rs +++ b/crates/types/src/int_list.rs @@ -203,6 +203,10 @@ pub fn merge_and_split( additions: IntegerList, max_bytes: usize, ) -> (IntegerList, Option) { + debug_assert!( + additions.serialized_size() <= max_bytes, + "additions exceed a single shard's budget", + ); let mut tail: Option = None; for block in additions.iter() { @@ -303,14 +307,21 @@ mod tests { #[test] fn merge_and_split_split_preserves_strict_ordering() { - // Additions are strictly greater than existing; verify post-split second - // contains only the newest blocks. - let existing = IntegerList::new(0u64..10).unwrap(); - let additions = IntegerList::new(10u64..30).unwrap(); - + // Sparse blocks (one per distinct 16-bit container) make serialized + // size grow proportionally to count, so we can budget-tune to force + // a split while keeping additions within budget. + let existing_blocks: Vec = (0..50u64).map(|i| i * 0x1_0000).collect(); + let addition_blocks: Vec = (50..70u64).map(|i| i * 0x1_0000).collect(); + let combined_blocks: Vec = (0..70u64).map(|i| i * 0x1_0000).collect(); + + let existing = IntegerList::new(existing_blocks).unwrap(); + let additions = IntegerList::new(addition_blocks).unwrap(); + let combined_size = IntegerList::new(combined_blocks).unwrap().serialized_size(); + let additions_size = additions.serialized_size(); let existing_size = existing.serialized_size(); - let combined = IntegerList::new(0u64..30).unwrap().serialized_size(); - let budget = existing_size + (combined - existing_size) / 3; + // Budget that fits both existing and additions alone, but not combined. + let budget = existing_size.max(additions_size) + 16; + assert!(combined_size > budget, "test setup broken: combined fits in budget"); let (first, second) = merge_and_split(existing, additions, budget); let second = second.expect("split should have occurred"); From 7e4e02d7f89c0b3d61c37eeb64a67ff25ebb2ada Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 06:37:57 -0400 Subject: [PATCH 04/21] test(storage-types): pin BlockNumberList worst-case sizes against MDBX budget Two property-style tests assert roaring's encoded size stays under 1500 B for both the dense-pack (650 contiguous) and sparse-distinct-container (100 blocks across 100 16-bit chunks) worst cases, plus a round-trip test through merge_and_split at the realistic budget. These pin roaring's encoding behavior so a future crate update can't silently break MDBX's DUPSORT value cap. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/types/src/int_list.rs | 50 ++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/crates/types/src/int_list.rs b/crates/types/src/int_list.rs index e512a44..28eaa21 100644 --- a/crates/types/src/int_list.rs +++ b/crates/types/src/int_list.rs @@ -330,4 +330,54 @@ mod tests { let second_min = second.min().unwrap(); assert!(first_max < second_min, "first.max={first_max} second.min={second_min}",); } + + /// Dense-pack worst case: many contiguous blocks in a single 16-bit + /// container should encode efficiently (run-length or array). Even at + /// hundreds of contiguous blocks, we stay comfortably under 1500 B. + #[test] + fn worst_case_dense_pack_fits_in_dupsort_budget() { + let list = IntegerList::new(0u64..650).unwrap(); + let size = list.serialized_size(); + assert!(size <= 1500, "dense pack of 650 blocks encoded as {size} B, expected <= 1500"); + } + + /// Sparse worst case: 100 blocks each in a distinct 16-bit container. + /// Each container is array-encoded with a single element plus header. + /// This is the realistic worst case for a long-lived hot address touched + /// once per ~64k-block window. + #[test] + fn worst_case_sparse_distinct_containers_fits_in_dupsort_budget() { + let blocks: Vec = (0..100u64).map(|i| i * 0x1_0000).collect(); + let list = IntegerList::new(blocks).unwrap(); + let size = list.serialized_size(); + assert!(size <= 1500, "100 sparse blocks encoded as {size} B, expected <= 1500"); + } + + /// merge_and_split with the realistic budget produces shards each within + /// the budget, even for the worst-case sparse input. + #[test] + fn merge_and_split_at_realistic_budget_respects_per_shard_size() { + // Build 200 sparse blocks (200 distinct containers). Splitter should + // split this into ~2 shards each under 1500 B. + let blocks: Vec = (0..200u64).map(|i| i * 0x1_0000).collect(); + let half = blocks.len() / 2; + let existing = IntegerList::new(blocks[..half].iter().copied()).unwrap(); + let additions = IntegerList::new(blocks[half..].iter().copied()).unwrap(); + + assert!(existing.serialized_size() <= 1500); + assert!(additions.serialized_size() <= 1500); + + let (first, second) = merge_and_split(existing, additions, 1500); + assert!(first.serialized_size() <= 1500); + if let Some(second) = &second { + assert!(second.serialized_size() <= 1500); + } + + // Round-trip: union of (first, second) equals the original input. + let mut roundtrip: Vec = first.iter().collect(); + if let Some(s) = second { + roundtrip.extend(s.iter()); + } + assert_eq!(roundtrip, blocks); + } } From b627389a9feb03a6fcdbda080dc80a8b288c75a5 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 06:53:36 -0400 Subject: [PATCH 05/21] refactor(hot): rename legacy history traits in preparation for delamination HistoryRead -> LegacyHistoryRead UnsafeHistoryWrite -> LegacyUnsafeHistoryWrite HistoryWrite -> LegacyConsistentHistoryWrite Pure mechanical rename. The unprefixed names are reserved for the new logical traits introduced in the next commit, which will live in crate::db::history and expose only logical (no shard-leaking) operations. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot-mdbx/src/test_utils.rs | 2 +- crates/hot/README.md | 6 +++--- crates/hot/src/conformance/history.rs | 2 +- crates/hot/src/conformance/range.rs | 2 +- crates/hot/src/conformance/roundtrip.rs | 2 +- crates/hot/src/conformance/unwind.rs | 2 +- crates/hot/src/db/consistent.rs | 6 +++--- crates/hot/src/db/inconsistent.rs | 8 ++++---- crates/hot/src/db/mod.rs | 6 +++--- crates/hot/src/db/read.rs | 4 ++-- crates/hot/src/lib.rs | 14 +++++++------- crates/hot/src/model/revm.rs | 8 ++++---- crates/hot/src/model/traits.rs | 8 ++++---- crates/hot/tests/load_genesis.rs | 2 +- crates/storage/README.md | 2 +- crates/storage/src/lib.rs | 2 +- crates/storage/src/unified.rs | 2 +- crates/storage/tests/unified.rs | 4 +++- 18 files changed, 42 insertions(+), 40 deletions(-) diff --git a/crates/hot-mdbx/src/test_utils.rs b/crates/hot-mdbx/src/test_utils.rs index 91d93c5..46cd2f8 100644 --- a/crates/hot-mdbx/src/test_utils.rs +++ b/crates/hot-mdbx/src/test_utils.rs @@ -1227,7 +1227,7 @@ mod tests { }; use signet_hot::{ conformance::make_bundle_state, - db::{HistoryWrite, HotDbRead}, + db::{HotDbRead, LegacyConsistentHistoryWrite}, model::{DualTableTraverse, HotKvRead}, tables, }; diff --git a/crates/hot/README.md b/crates/hot/README.md index 807761f..b45aa97 100644 --- a/crates/hot/README.md +++ b/crates/hot/README.md @@ -8,7 +8,7 @@ with opinionated serialization and predefined tables for blockchain state. ## Usage ```rust,ignore -use signet_hot::{HotKv, HistoryRead, HistoryWrite}; +use signet_hot::{HotKv, LegacyHistoryRead, LegacyConsistentHistoryWrite}; fn example(db: &D) -> Result<(), signet_hot::db::HotKvError> { // Read operations @@ -32,10 +32,10 @@ For a concrete implementation, see the `signet-hot-mdbx` crate. HotKv ← Transaction factory ├─ reader() → HotKvRead ← Read-only transactions │ └─ HotDbRead ← Typed accessors (blanket impl) - │ └─ HistoryRead ← History queries (blanket impl) + │ └─ LegacyHistoryRead ← History queries (blanket impl) └─ writer() → HotKvWrite ← Read-write transactions └─ UnsafeDbWrite ← Low-level writes (blanket impl) - └─ HistoryWrite ← Safe chain operations (blanket impl) + └─ LegacyConsistentHistoryWrite ← Safe chain operations (blanket impl) ``` ## Serialization diff --git a/crates/hot/src/conformance/history.rs b/crates/hot/src/conformance/history.rs index af0250c..b73331c 100644 --- a/crates/hot/src/conformance/history.rs +++ b/crates/hot/src/conformance/history.rs @@ -1,7 +1,7 @@ //! History and change set tests for hot storage. use crate::{ - db::{HistoryRead, UnsafeDbWrite, UnsafeHistoryWrite}, + db::{LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, model::{HotKv, HotKvWrite}, tables, }; diff --git a/crates/hot/src/conformance/range.rs b/crates/hot/src/conformance/range.rs index 496795b..7fea95a 100644 --- a/crates/hot/src/conformance/range.rs +++ b/crates/hot/src/conformance/range.rs @@ -1,7 +1,7 @@ //! Clear/take range operations for single and dual-keyed tables. use crate::{ - db::{HistoryRead, HotDbRead, UnsafeDbWrite, UnsafeHistoryWrite}, + db::{HotDbRead, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, model::{HotKv, HotKvWrite}, tables, }; diff --git a/crates/hot/src/conformance/roundtrip.rs b/crates/hot/src/conformance/roundtrip.rs index cba2c76..3e25d24 100644 --- a/crates/hot/src/conformance/roundtrip.rs +++ b/crates/hot/src/conformance/roundtrip.rs @@ -1,7 +1,7 @@ //! Basic CRUD roundtrip tests for hot storage. use crate::{ - db::{HistoryRead, HotDbRead, UnsafeDbWrite, UnsafeHistoryWrite}, + db::{HotDbRead, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, model::{HotKv, HotKvRead}, tables, }; diff --git a/crates/hot/src/conformance/unwind.rs b/crates/hot/src/conformance/unwind.rs index 3471c6d..07b0d01 100644 --- a/crates/hot/src/conformance/unwind.rs +++ b/crates/hot/src/conformance/unwind.rs @@ -1,7 +1,7 @@ //! Unwind conformance test and helpers. use crate::{ - db::{HistoryWrite, UnsafeDbWrite}, + db::{LegacyConsistentHistoryWrite, UnsafeDbWrite}, model::{DualKeyValue, DualTableTraverse, HotKv, HotKvRead, KeyValue, TableTraverse}, tables::{self, DualKey, SingleKey}, }; diff --git a/crates/hot/src/db/consistent.rs b/crates/hot/src/db/consistent.rs index c180d65..4bb0a26 100644 --- a/crates/hot/src/db/consistent.rs +++ b/crates/hot/src/db/consistent.rs @@ -1,5 +1,5 @@ use crate::{ - db::{HistoryError, UnsafeDbWrite, UnsafeHistoryWrite}, + db::{HistoryError, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, tables, }; use ahash::AHashSet; @@ -16,7 +16,7 @@ const ADDRESS_MAX: Address = address!("0xfffffffffffffffffffffffffffffffffffffff /// Trait for database write operations on hot history tables. This trait /// maintains a consistent state of the database. -pub trait HistoryWrite: UnsafeDbWrite + UnsafeHistoryWrite { +pub trait LegacyConsistentHistoryWrite: UnsafeDbWrite + LegacyUnsafeHistoryWrite { /// Validate that a range of headers forms a valid chain extension. /// /// Headers must be in order and each must extend the previous. @@ -344,4 +344,4 @@ pub trait HistoryWrite: UnsafeDbWrite + UnsafeHistoryWrite { } } -impl HistoryWrite for T where T: UnsafeDbWrite + UnsafeHistoryWrite {} +impl LegacyConsistentHistoryWrite for T where T: UnsafeDbWrite + LegacyUnsafeHistoryWrite {} diff --git a/crates/hot/src/db/inconsistent.rs b/crates/hot/src/db/inconsistent.rs index 9a47a2b..8eb4302 100644 --- a/crates/hot/src/db/inconsistent.rs +++ b/crates/hot/src/db/inconsistent.rs @@ -1,5 +1,5 @@ use crate::{ - db::{HistoryError, HistoryRead}, + db::{HistoryError, LegacyHistoryRead}, model::HotKvWrite, tables, }; @@ -29,7 +29,7 @@ pub type BundleInit = /// inconsistent state if not used carefully. Users should prefer /// [`HotHistoryWrite`] or higher-level abstractions when possible. /// -/// [`HotHistoryWrite`]: crate::db::HistoryWrite +/// [`HotHistoryWrite`]: crate::db::LegacyConsistentHistoryWrite pub trait UnsafeDbWrite: HotKvWrite + super::sealed::Sealed { /// Write a block header. This will leave the DB in an inconsistent state /// until the corresponding header number is also written. Users should @@ -118,7 +118,7 @@ impl UnsafeDbWrite for T where T: HotKvWrite {} /// These tables maintain historical information about accounts and storage /// changes, and their contents can be used to reconstruct past states or /// roll back changes. -pub trait UnsafeHistoryWrite: UnsafeDbWrite + HistoryRead { +pub trait LegacyUnsafeHistoryWrite: UnsafeDbWrite + LegacyHistoryRead { /// Maintain a list of block numbers where an account was touched. /// /// Accounts are keyed @@ -514,7 +514,7 @@ pub trait UnsafeHistoryWrite: UnsafeDbWrite + HistoryRead { } } -impl UnsafeHistoryWrite for T where T: UnsafeDbWrite + HotKvWrite {} +impl LegacyUnsafeHistoryWrite for T where T: UnsafeDbWrite + HotKvWrite {} /// Append indices to a sharded history entry, handling shard splitting. /// diff --git a/crates/hot/src/db/mod.rs b/crates/hot/src/db/mod.rs index 522bdd4..9be3ae9 100644 --- a/crates/hot/src/db/mod.rs +++ b/crates/hot/src/db/mod.rs @@ -1,16 +1,16 @@ //! Primary access traits for hot storage backends. mod consistent; -pub use consistent::HistoryWrite; +pub use consistent::LegacyConsistentHistoryWrite; mod errors; pub use errors::{HistoryError, HistoryResult}; mod inconsistent; -pub use inconsistent::{BundleInit, UnsafeDbWrite, UnsafeHistoryWrite}; +pub use inconsistent::{BundleInit, LegacyUnsafeHistoryWrite, UnsafeDbWrite}; mod read; -pub use read::{HistoryRead, HotDbRead}; +pub use read::{HotDbRead, LegacyHistoryRead}; pub(crate) mod sealed { use crate::model::HotKvRead; diff --git a/crates/hot/src/db/read.rs b/crates/hot/src/db/read.rs index b6adf47..6fc8cd1 100644 --- a/crates/hot/src/db/read.rs +++ b/crates/hot/src/db/read.rs @@ -61,7 +61,7 @@ impl HotDbRead for T where T: HotKvRead {} /// /// Users should prefer this trait unless customizations are needed to the /// table set. -pub trait HistoryRead: HotDbRead { +pub trait LegacyHistoryRead: HotDbRead { /// Get the list of block numbers where an account was touched. /// Get the list of block numbers where an account was touched. fn get_account_history( @@ -368,4 +368,4 @@ pub trait HistoryRead: HotDbRead { } } -impl HistoryRead for T where T: HotDbRead {} +impl LegacyHistoryRead for T where T: HotDbRead {} diff --git a/crates/hot/src/lib.rs b/crates/hot/src/lib.rs index 6430967..5ea8bd3 100644 --- a/crates/hot/src/lib.rs +++ b/crates/hot/src/lib.rs @@ -7,7 +7,7 @@ //! # Quick Start //! //! ```ignore -//! use signet_hot::{HotKv, HistoryRead, HistoryWrite}; +//! use signet_hot::{HotKv, LegacyHistoryRead, LegacyConsistentHistoryWrite}; //! //! fn example(db: &D) -> Result<(), signet_hot::db::HotKvError> { //! // Read operations @@ -31,10 +31,10 @@ //! HotKv ← Transaction factory //! ├─ reader() → HotKvRead ← Read-only transactions //! │ └─ HotDbRead ← Typed accessors (blanket impl) -//! │ └─ HistoryRead ← History queries (blanket impl) +//! │ └─ LegacyHistoryRead ← History queries (blanket impl) //! └─ writer() → HotKvWrite ← Read-write transactions //! └─ UnsafeDbWrite ← Low-level writes (blanket impl) -//! └─ HistoryWrite ← Safe chain operations (blanket impl) +//! └─ LegacyConsistentHistoryWrite ← Safe chain operations (blanket impl) //! ``` //! //! ## Serialization @@ -56,7 +56,7 @@ //! These traits provide methods for common operations such as getting, //! setting, and deleting key-value pairs in hot storage tables. The raw //! key-value operations use byte slices for maximum flexibility. The -//! [`HistoryRead`] and [`HistoryWrite`] traits provide higher-level +//! [`LegacyHistoryRead`] and [`LegacyConsistentHistoryWrite`] traits provide higher-level //! abstractions that work with the predefined tables and their associated key //! and value types. //! @@ -73,8 +73,8 @@ //! See the [`Table`] trait documentation for more information on defining and //! using tables. //! -//! [`HistoryRead`]: db::HistoryRead -//! [`HistoryWrite`]: db::HistoryWrite +//! [`LegacyHistoryRead`]: db::LegacyHistoryRead +//! [`LegacyConsistentHistoryWrite`]: db::LegacyConsistentHistoryWrite //! [`HotKvRead`]: model::HotKvRead //! [`HotKvWrite`]: model::HotKvWrite //! [`HotKv`]: model::HotKv @@ -109,7 +109,7 @@ pub mod connect; pub use connect::HotConnect; pub mod db; -pub use db::{HistoryError, HistoryRead, HistoryWrite}; +pub use db::{HistoryError, LegacyConsistentHistoryWrite, LegacyHistoryRead}; pub mod model; pub use model::HotKv; diff --git a/crates/hot/src/model/revm.rs b/crates/hot/src/model/revm.rs index db91072..b1dcf72 100644 --- a/crates/hot/src/model/revm.rs +++ b/crates/hot/src/model/revm.rs @@ -1,5 +1,5 @@ use crate::{ - db::HistoryRead, + db::LegacyHistoryRead, model::{HotKvError, HotKvRead, HotKvWrite}, tables::{self, Bytecodes, DualKey, PlainAccountState, SingleKey, Table}, }; @@ -39,10 +39,10 @@ impl RevmRead { /// current state, and heights before the first block return the /// pre-state of the earliest change. Use /// [`HotKv::revm_reader_at_height`] which validates, or call - /// [`HistoryRead::check_height`] manually. + /// [`LegacyHistoryRead::check_height`] manually. /// /// [`HotKv::revm_reader_at_height`]: crate::model::HotKv::revm_reader_at_height - /// [`HistoryRead::check_height`]: crate::db::HistoryRead::check_height + /// [`LegacyHistoryRead::check_height`]: crate::db::LegacyHistoryRead::check_height pub const fn at_height(reader: T, height: u64) -> Self { Self { reader, height: Some(height) } } @@ -445,7 +445,7 @@ where mod tests { use super::*; use crate::{ - db::{HistoryRead, UnsafeDbWrite, UnsafeHistoryWrite}, + db::{LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, mem::MemKv, model::{HotKv, HotKvRead, HotKvWrite}, tables::{Bytecodes, PlainAccountState}, diff --git a/crates/hot/src/model/traits.rs b/crates/hot/src/model/traits.rs index 7d4ee1c..6a00c76 100644 --- a/crates/hot/src/model/traits.rs +++ b/crates/hot/src/model/traits.rs @@ -1,5 +1,5 @@ use crate::{ - db::HistoryRead, + db::LegacyHistoryRead, model::{ DualKeyTraverse, DualKeyTraverseMut, DualTableCursor, HotKvError, HotKvReadError, KvTraverse, KvTraverseMut, TableCursor, @@ -15,14 +15,14 @@ use std::borrow::Cow; /// This is the top-level trait for hot storage backends, providing /// transactional access through read-only and read-write transactions. /// -/// We recommend using [`HistoryRead`] and [`HistoryWrite`] for most use cases, +/// We recommend using [`LegacyHistoryRead`] and [`LegacyConsistentHistoryWrite`] for most use cases, /// as they provide higher-level abstractions over predefined tables. /// /// When implementing this trait, consult the [`model`] module documentation for /// details on the associated types and their requirements. /// -/// [`HistoryRead`]: crate::db::HistoryRead -/// [`HistoryWrite`]: crate::db::HistoryWrite +/// [`LegacyHistoryRead`]: crate::db::LegacyHistoryRead +/// [`LegacyConsistentHistoryWrite`]: crate::db::LegacyConsistentHistoryWrite /// [`model`]: crate::model #[auto_impl::auto_impl(&, Arc, Box)] pub trait HotKv { diff --git a/crates/hot/tests/load_genesis.rs b/crates/hot/tests/load_genesis.rs index 44d6167..decd72a 100644 --- a/crates/hot/tests/load_genesis.rs +++ b/crates/hot/tests/load_genesis.rs @@ -9,7 +9,7 @@ use alloy::{ }; use signet_hot::{ HotKv, - db::{HistoryRead, HistoryWrite, HotDbRead, UnsafeDbWrite}, + db::{HotDbRead, LegacyConsistentHistoryWrite, LegacyHistoryRead, UnsafeDbWrite}, mem::MemKv, }; use signet_storage_types::EthereumHardfork; diff --git a/crates/storage/README.md b/crates/storage/README.md index b1ffaa2..091958a 100644 --- a/crates/storage/README.md +++ b/crates/storage/README.md @@ -70,5 +70,5 @@ Cold dispatch errors indicate either: Key types are re-exported for convenience: - `ExecutedBlock`, `ExecutedBlockBuilder` - Block data structures -- `HotKv`, `HistoryRead`, `HistoryWrite` - Hot storage traits +- `HotKv`, `LegacyHistoryRead`, `LegacyConsistentHistoryWrite` - Hot storage traits - `ColdStorageHandle`, `ColdStorageError` - Cold storage types diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index c389e22..3f00ff7 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -87,7 +87,7 @@ pub use signet_cold::{ }; pub use signet_cold_mdbx::MdbxColdBackend; pub use signet_hot::{ - HistoryError, HistoryRead, HistoryWrite, HotKv, + HistoryError, HotKv, LegacyConsistentHistoryWrite, LegacyHistoryRead, model::{HotKvRead, RevmRead, RevmWrite}, }; pub use signet_hot_mdbx::{DatabaseArguments, DatabaseEnv}; diff --git a/crates/storage/src/unified.rs b/crates/storage/src/unified.rs index f525d12..9accc54 100644 --- a/crates/storage/src/unified.rs +++ b/crates/storage/src/unified.rs @@ -8,7 +8,7 @@ use crate::StorageResult; use alloy::primitives::BlockNumber; use signet_cold::{BlockData, ColdReceipt, ColdStorage, ColdStorageBackend, ColdStorageError}; use signet_hot::{ - HistoryRead, HistoryWrite, HotKv, + HotKv, LegacyConsistentHistoryWrite, LegacyHistoryRead, model::{HotKvReadError, HotKvWrite, RevmRead}, }; use signet_storage_types::{ExecutedBlock, SealedHeader}; diff --git a/crates/storage/tests/unified.rs b/crates/storage/tests/unified.rs index ed238be..0f900c4 100644 --- a/crates/storage/tests/unified.rs +++ b/crates/storage/tests/unified.rs @@ -5,7 +5,9 @@ use alloy::{ primitives::{Address, B256, Signature, TxKind, U256}, }; use signet_cold::{ColdStorage, HeaderSpecifier, mem::MemColdBackend}; -use signet_hot::{HistoryRead, HistoryWrite, HotKv, mem::MemKv, model::HotKvWrite}; +use signet_hot::{ + HotKv, LegacyConsistentHistoryWrite, LegacyHistoryRead, mem::MemKv, model::HotKvWrite, +}; use signet_storage::UnifiedStorage; use signet_storage_types::{ ExecutedBlock, ExecutedBlockBuilder, Receipt, RecoveredTx, SealedHeader, TransactionSigned, From 86425a233f10c9e9cb5b41734ab3146d5c7e2e95 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 06:58:56 -0400 Subject: [PATCH 06/21] feat(hot): introduce logical HistoryRead / HistoryWrite trait module New module crate::db::history defines: - HistoryRead: blanket-impled on HotKvRead, default-impl-only. - HistoryWrite: required per-backend, with default impls for bulk ops. No callers yet; this is the trait scaffolding. Per-backend HistoryWrite impls land in subsequent commits. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/src/db/history.rs | 281 +++++++++++++++++++++++++++++++++++ crates/hot/src/db/mod.rs | 3 + 2 files changed, 284 insertions(+) create mode 100644 crates/hot/src/db/history.rs diff --git a/crates/hot/src/db/history.rs b/crates/hot/src/db/history.rs new file mode 100644 index 0000000..8e3e324 --- /dev/null +++ b/crates/hot/src/db/history.rs @@ -0,0 +1,281 @@ +//! Logical history reads and writes. +//! +//! These traits replace the shard-leaking surface in `db::read` and +//! `db::inconsistent`. [`HistoryRead`] is blanket-impled on [`HotKvRead`] and +//! cannot be overridden — the KV-table layout is mandated by the abstraction. +//! [`HistoryWrite`] is required per-backend; each backend chooses its +//! splitting policy (MDBX uses [`signet_storage_types::merge_and_split`]; +//! MemKv writes a single dup entry per addr). + +use crate::{ + db::{HistoryError, HotDbRead, UnsafeDbWrite}, + model::HotKvRead, + tables, +}; +use ahash::AHashMap; +use alloy::primitives::{Address, BlockNumber, U256}; +use itertools::Itertools; +use signet_storage_types::{Account, BlockNumberList, ShardedKey}; +use std::ops::RangeInclusive; + +/// Logical reads against history + changeset tables. +/// +/// Default-impl-only. Backends cannot override — the blanket impl below +/// occupies the slot, and orphan rules prevent downstream impls. This is +/// structural enforcement of "the KV-table access pattern is mandated by +/// the abstraction". +pub trait HistoryRead: HotDbRead { + /// All block numbers where `addr` was touched. `None` if no history. + fn blocks_changed_account( + &self, + addr: &Address, + ) -> Result, Self::Error> { + let mut cursor = self.traverse_dual::()?; + let mut iter = cursor.iter_k2(addr)?; + let Some(first) = iter.next().transpose()? else { + return Ok(None); + }; + // first is (u64, BlockNumberList) — the shard key and its list + let (_, mut merged) = first; + for entry in iter { + let (_, list) = entry?; + merged + .append(list.iter()) + .map_err(|e| { + // Safety: we are iterating over history blocks which are + // supposed to be strictly increasing; if not, the DB is + // corrupt. We convert to the DB error type here. + // Note: Self::Error must come from some HotKvReadError impl + // but IntegerListError is not that type. We drop it since + // blocks_changed_account returns Self::Error, not + // HistoryError. The caller can use the HistoryWrite path for + // mutations. For reads, a corrupt DB is a storage error that + // this layer cannot express; we suppress it. + let _ = e; + }) + .ok(); + } + Ok(Some(merged)) + } + + /// All block numbers where `(addr, slot)` was touched. `None` if none. + fn blocks_changed_storage( + &self, + addr: &Address, + slot: &U256, + ) -> Result, Self::Error> { + let target = ShardedKey::new(*slot, 0u64); + let mut cursor = self.traverse_dual::()?; + let Some((found_addr, sk, list)) = cursor.next_dual_above(addr, &target)? else { + return Ok(None); + }; + if found_addr != *addr || sk.key != *slot { + return Ok(None); + } + let mut merged = list; + while let Some((next_addr, next_sk, next_list)) = cursor.read_next()? { + if next_addr != *addr || next_sk.key != *slot { + break; + } + merged.append(next_list.iter()).ok(); + } + Ok(Some(merged)) + } + + /// Smallest block `> height` where `addr` was touched. + /// + /// `None` means the account was not changed after `height`; the caller + /// should consult the current plain state. + fn block_account_changed_after( + &self, + addr: &Address, + height: u64, + ) -> Result, Self::Error> { + let Some(target) = height.checked_add(1) else { + return Ok(None); + }; + let mut cursor = self.traverse_dual::()?; + let Some((found_addr, _, list)) = cursor.next_dual_above(addr, &target)? else { + return Ok(None); + }; + if found_addr != *addr { + return Ok(None); + } + let rank = list.rank(height); + Ok(list.select(rank)) + } + + /// Smallest block `> height` where `(addr, slot)` was touched. + fn block_storage_changed_after( + &self, + addr: &Address, + slot: &U256, + height: u64, + ) -> Result, Self::Error> { + let Some(target_block) = height.checked_add(1) else { + return Ok(None); + }; + let target = ShardedKey::new(*slot, target_block); + let mut cursor = self.traverse_dual::()?; + let Some((found_addr, sk, list)) = cursor.next_dual_above(addr, &target)? else { + return Ok(None); + }; + if found_addr != *addr || sk.key != *slot { + return Ok(None); + } + let rank = list.rank(height); + Ok(list.select(rank)) + } + + /// Account pre-state recorded at `block`, or `None` if `addr` was not + /// changed in `block`. + fn get_account_change( + &self, + block: u64, + addr: &Address, + ) -> Result, Self::Error> { + self.get_dual::(&block, addr) + } + + /// Storage pre-state recorded at `block` for `(addr, slot)`. + fn get_storage_change( + &self, + block: u64, + addr: &Address, + slot: &U256, + ) -> Result, Self::Error> { + self.get_dual::(&(block, *addr), slot) + } + + /// Get account state, optionally at a specific historical block height. + /// + /// When `height` is `Some`, reconstructs the account state at that block + /// height by consulting history and change set tables. When `None`, returns + /// the current value from `PlainAccountState`. + fn get_account_at_height( + &self, + addr: &Address, + height: Option, + ) -> Result, Self::Error> { + let Some(h) = height else { + return self.get_account(addr); + }; + match self.block_account_changed_after(addr, h)? { + None => self.get_account(addr), + Some(first) => self.get_account_change(first, addr), + } + } + + /// Get storage slot value, optionally at a specific historical block height. + fn get_storage_at_height( + &self, + addr: &Address, + slot: &U256, + height: Option, + ) -> Result, Self::Error> { + let Some(h) = height else { + return self.get_storage(addr, slot); + }; + match self.block_storage_changed_after(addr, slot, h)? { + None => self.get_storage(addr, slot), + Some(first) => self.get_storage_change(first, addr, slot), + } + } +} + +impl HistoryRead for T where T: HotKvRead {} + +/// Logical writes against history + changeset tables. Required per backend. +/// +/// Backends that implement this trait choose their own shard-splitting policy. +/// The default `update_history_indices` bulk operation is expressed in terms of +/// the four required primitives and works for any backend. +pub trait HistoryWrite: UnsafeDbWrite + HistoryRead { + /// Merge `new_blocks` into `addr`'s account history. + /// + /// Preconditions: `new_blocks` is sorted ascending and every entry is + /// strictly greater than any block already stored for `addr`. + fn append_account_history( + &self, + addr: &Address, + new_blocks: &BlockNumberList, + ) -> Result<(), HistoryError>; + + /// Merge `new_blocks` into `(addr, slot)`'s storage history. + fn append_storage_history( + &self, + addr: &Address, + slot: &U256, + new_blocks: &BlockNumberList, + ) -> Result<(), HistoryError>; + + /// Remove all blocks `> above` from `addr`'s account history. + /// If nothing remains, delete the entry. + fn truncate_account_history_above( + &self, + addr: &Address, + above: u64, + ) -> Result<(), HistoryError>; + + /// Remove all blocks `> above` from `(addr, slot)`'s storage history. + fn truncate_storage_history_above( + &self, + addr: &Address, + slot: &U256, + above: u64, + ) -> Result<(), HistoryError>; + + // ---- default-impl bulk operations (in terms of the four required) ---- + + /// Build per-address block lists from changesets in `range` and call + /// [`Self::append_account_history`] / [`Self::append_storage_history`] per + /// entry. + fn update_history_indices( + &self, + range: RangeInclusive, + ) -> Result<(), HistoryError> { + // Account stage: collect (addr, block_number) pairs from changesets + let account_indices: AHashMap> = self + .traverse_dual::() + .map_err(HistoryError::Db)? + .iter_from(range.start(), &Address::ZERO) + .map_err(HistoryError::Db)? + .process_results(|iter| { + iter.take_while(|(num, _, _)| range.contains(num)) + .map(|(num, addr, _)| (addr, num)) + .into_group_map_by(|(addr, _)| *addr) + .into_iter() + .map(|(addr, pairs)| (addr, pairs.into_iter().map(|(_, n)| n).collect())) + .collect() + }) + .map_err(HistoryError::Db)?; + + for (addr, blocks) in account_indices { + let list = BlockNumberList::new_pre_sorted(blocks); + self.append_account_history(&addr, &list)?; + } + + // Storage stage: collect ((addr, slot), block_number) pairs from changesets + let storage_indices: AHashMap<(Address, U256), Vec> = self + .traverse_dual::() + .map_err(HistoryError::Db)? + .iter_from(&(*range.start(), Address::ZERO), &U256::ZERO) + .map_err(HistoryError::Db)? + .process_results(|iter| { + iter.take_while(|(num_addr, _, _)| range.contains(&num_addr.0)) + .map(|(num_addr, slot, _)| ((num_addr.1, slot), num_addr.0)) + .into_group_map_by(|(k, _)| *k) + .into_iter() + .map(|(k, pairs)| (k, pairs.into_iter().map(|(_, n)| n).collect())) + .collect() + }) + .map_err(HistoryError::Db)?; + + for ((addr, slot), blocks) in storage_indices { + let list = BlockNumberList::new_pre_sorted(blocks); + self.append_storage_history(&addr, &slot, &list)?; + } + + Ok(()) + } +} diff --git a/crates/hot/src/db/mod.rs b/crates/hot/src/db/mod.rs index 9be3ae9..64e6389 100644 --- a/crates/hot/src/db/mod.rs +++ b/crates/hot/src/db/mod.rs @@ -6,6 +6,9 @@ pub use consistent::LegacyConsistentHistoryWrite; mod errors; pub use errors::{HistoryError, HistoryResult}; +pub mod history; +pub use history::{HistoryRead, HistoryWrite}; + mod inconsistent; pub use inconsistent::{BundleInit, LegacyUnsafeHistoryWrite, UnsafeDbWrite}; From 66aa99ed6c794b15fa872c44c0a84b83196fb440 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 07:00:39 -0400 Subject: [PATCH 07/21] fixup(hot): panic on impossible-state in blocks_changed_* default impls Replace .ok() with .expect("history blocks strictly increasing") on the two BlockNumberList::append calls inside blocks_changed_account and blocks_changed_storage. The error case is only reachable on DB corruption; silently suppressing it would convert corruption into data loss in the read path. Loud panic is correct. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/src/db/history.rs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/crates/hot/src/db/history.rs b/crates/hot/src/db/history.rs index 8e3e324..b3a50b0 100644 --- a/crates/hot/src/db/history.rs +++ b/crates/hot/src/db/history.rs @@ -39,21 +39,7 @@ pub trait HistoryRead: HotDbRead { let (_, mut merged) = first; for entry in iter { let (_, list) = entry?; - merged - .append(list.iter()) - .map_err(|e| { - // Safety: we are iterating over history blocks which are - // supposed to be strictly increasing; if not, the DB is - // corrupt. We convert to the DB error type here. - // Note: Self::Error must come from some HotKvReadError impl - // but IntegerListError is not that type. We drop it since - // blocks_changed_account returns Self::Error, not - // HistoryError. The caller can use the HistoryWrite path for - // mutations. For reads, a corrupt DB is a storage error that - // this layer cannot express; we suppress it. - let _ = e; - }) - .ok(); + merged.append(list.iter()).expect("history blocks strictly increasing"); } Ok(Some(merged)) } @@ -77,7 +63,7 @@ pub trait HistoryRead: HotDbRead { if next_addr != *addr || next_sk.key != *slot { break; } - merged.append(next_list.iter()).ok(); + merged.append(next_list.iter()).expect("history blocks strictly increasing"); } Ok(Some(merged)) } From 9dc7b46d66caf1a6ba516edbf138c21aaea691f0 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 07:15:28 -0400 Subject: [PATCH 08/21] feat(hot): implement HistoryWrite for MemKv Single shard per (addr) at subkey u64::MAX, or per (addr, slot) at ShardedKey(slot, u64::MAX). MemKv has no per-value size budget, so no splitting is needed. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/src/mem.rs | 110 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/crates/hot/src/mem.rs b/crates/hot/src/mem.rs index d326f50..e807691 100644 --- a/crates/hot/src/mem.rs +++ b/crates/hot/src/mem.rs @@ -4,14 +4,18 @@ //! testing. use crate::{ + db::HistoryError, model::{ DualKeyItem, DualKeyTraverse, DualKeyTraverseMut, HotKv, HotKvError, HotKvRead, HotKvReadError, HotKvWrite, KvTraverse, KvTraverseMut, RawDualKeyItem, RawDualKeyValue, RawKeyValue, RawValue, }, ser::{DeserError, MAX_KEY_SIZE}, + tables, }; +use alloy::primitives::{Address, U256}; use bytes::Bytes; +use signet_storage_types::{BlockNumberList, ShardedKey}; use std::{ borrow::Cow, collections::BTreeMap, @@ -1439,6 +1443,83 @@ impl HotKvWrite for MemKvRwTx { } } +impl crate::db::history::HistoryWrite for MemKvRwTx { + fn append_account_history( + &self, + addr: &Address, + new_blocks: &BlockNumberList, + ) -> Result<(), HistoryError> { + // Load the existing tail (if any), append new blocks, write back at + // u64::MAX. MemKv has no size budget, so there's always one shard. + let mut merged = self + .get_dual::(addr, &u64::MAX) + .map_err(HistoryError::Db)? + .unwrap_or_default(); + merged.append(new_blocks.iter()).map_err(HistoryError::IntList)?; + self.queue_put_dual::(addr, &u64::MAX, &merged) + .map_err(HistoryError::Db) + } + + fn append_storage_history( + &self, + addr: &Address, + slot: &U256, + new_blocks: &BlockNumberList, + ) -> Result<(), HistoryError> { + let tail_key = ShardedKey::new(*slot, u64::MAX); + let mut merged = self + .get_dual::(addr, &tail_key) + .map_err(HistoryError::Db)? + .unwrap_or_default(); + merged.append(new_blocks.iter()).map_err(HistoryError::IntList)?; + self.queue_put_dual::(addr, &tail_key, &merged) + .map_err(HistoryError::Db) + } + + fn truncate_account_history_above( + &self, + addr: &Address, + above: u64, + ) -> Result<(), HistoryError> { + // Single shard at u64::MAX. Load, filter, write back or delete. + let Some(existing) = + self.get_dual::(addr, &u64::MAX).map_err(HistoryError::Db)? + else { + return Ok(()); + }; + let kept = BlockNumberList::new_pre_sorted(existing.iter().take_while(|&b| b <= above)); + self.queue_delete_dual::(addr, &u64::MAX) + .map_err(HistoryError::Db)?; + if !kept.is_empty() { + self.queue_put_dual::(addr, &u64::MAX, &kept) + .map_err(HistoryError::Db)?; + } + Ok(()) + } + + fn truncate_storage_history_above( + &self, + addr: &Address, + slot: &U256, + above: u64, + ) -> Result<(), HistoryError> { + let tail_key = ShardedKey::new(*slot, u64::MAX); + let Some(existing) = + self.get_dual::(addr, &tail_key).map_err(HistoryError::Db)? + else { + return Ok(()); + }; + let kept = BlockNumberList::new_pre_sorted(existing.iter().take_while(|&b| b <= above)); + self.queue_delete_dual::(addr, &tail_key) + .map_err(HistoryError::Db)?; + if !kept.is_empty() { + self.queue_put_dual::(addr, &tail_key, &kept) + .map_err(HistoryError::Db)?; + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -2547,4 +2628,33 @@ mod tests { assert!(reader.get_dual::(&3u64, &1000u32).unwrap().is_some()); } } + + #[test] + fn memkv_history_write_round_trips() { + use crate::db::{HistoryRead, HistoryWrite}; + + let mem = MemKv::default(); + let addr = Address::from_slice(&[0x1; 20]); + let slot = U256::from(42u64); + + let writer = mem.writer().unwrap(); + writer + .append_account_history(&addr, &BlockNumberList::new([5u64, 10, 15]).unwrap()) + .unwrap(); + writer + .append_storage_history(&addr, &slot, &BlockNumberList::new([7u64, 11]).unwrap()) + .unwrap(); + writer.raw_commit().unwrap(); + + let reader = mem.reader().unwrap(); + let acct_blocks = reader.blocks_changed_account(&addr).unwrap().unwrap(); + assert_eq!(acct_blocks.iter().collect::>(), vec![5, 10, 15]); + + let stor_blocks = reader.blocks_changed_storage(&addr, &slot).unwrap().unwrap(); + assert_eq!(stor_blocks.iter().collect::>(), vec![7, 11]); + + // block_account_changed_after + assert_eq!(reader.block_account_changed_after(&addr, 6).unwrap(), Some(10)); + assert_eq!(reader.block_account_changed_after(&addr, 15).unwrap(), None); + } } From 28f34eacbae23d4970ef692179e44a655af914a4 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 07:22:20 -0400 Subject: [PATCH 09/21] feat(hot-mdbx): implement HistoryWrite using merge_and_split MDBX backend's HistoryWrite uses signet_storage_types::merge_and_split with MAX_SHARD_BYTES = 1500, derived from MDBX's DUPSORT value cap minus key2 and per-node overhead. Load-merge-rewrite for appends; backwards-walk delete-and-re-key-tail for truncates. The u64::MAX-tail subkey convention is preserved structurally: appends always write the tail at u64::MAX; truncates re-key the surviving prefix to u64::MAX if the original tail was deleted. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot-mdbx/Cargo.toml | 2 +- crates/hot-mdbx/src/test_utils.rs | 41 ++++++ crates/hot-mdbx/src/tx.rs | 217 ++++++++++++++++++++++++++++++ 3 files changed, 259 insertions(+), 1 deletion(-) diff --git a/crates/hot-mdbx/Cargo.toml b/crates/hot-mdbx/Cargo.toml index 1ba9f3c..f5d1210 100644 --- a/crates/hot-mdbx/Cargo.toml +++ b/crates/hot-mdbx/Cargo.toml @@ -22,6 +22,7 @@ page_size.workspace = true parking_lot.workspace = true signet-hot.workspace = true signet-libmdbx.workspace = true +signet-storage-types.workspace = true sysinfo = "0.37.2" tempfile = { workspace = true, optional = true } thiserror.workspace = true @@ -31,7 +32,6 @@ trevm.workspace = true [dev-dependencies] serial_test = "3.3.1" signet-hot = { workspace = true, features = ["test-utils"] } -signet-storage-types.workspace = true tempfile.workspace = true [features] diff --git a/crates/hot-mdbx/src/test_utils.rs b/crates/hot-mdbx/src/test_utils.rs index 46cd2f8..a39028b 100644 --- a/crates/hot-mdbx/src/test_utils.rs +++ b/crates/hot-mdbx/src/test_utils.rs @@ -2004,4 +2004,45 @@ mod tests { assert_eq!(fsi_headers, FixedSizeInfo::None); } } + + /// Smoke test: append + truncate via the new `HistoryWrite` trait. + #[test] + #[serial] + fn mdbx_history_write_round_trips() { + use signet_hot::db::{HistoryRead, HistoryWrite}; + use signet_storage_types::BlockNumberList; + + run_test(|db| { + let addr = Address::from_slice(&[0x1; 20]); + let slot = U256::from(42u64); + + let writer: Tx = db.writer().unwrap(); + writer + .append_account_history(&addr, &BlockNumberList::new([5u64, 10, 15]).unwrap()) + .unwrap(); + writer + .append_storage_history(&addr, &slot, &BlockNumberList::new([7u64, 11]).unwrap()) + .unwrap(); + writer.commit().unwrap(); + + let reader: Tx = db.reader().unwrap(); + let acct_blocks = reader.blocks_changed_account(&addr).unwrap().unwrap(); + assert_eq!(acct_blocks.iter().collect::>(), vec![5, 10, 15]); + + let stor_blocks = reader.blocks_changed_storage(&addr, &slot).unwrap().unwrap(); + assert_eq!(stor_blocks.iter().collect::>(), vec![7, 11]); + + assert_eq!(reader.block_account_changed_after(&addr, 7).unwrap(), Some(10)); + assert_eq!(reader.block_account_changed_after(&addr, 15).unwrap(), None); + + // Truncate above block 10 — should remove block 15 from account history. + let writer: Tx = db.writer().unwrap(); + writer.truncate_account_history_above(&addr, 10).unwrap(); + writer.commit().unwrap(); + + let reader: Tx = db.reader().unwrap(); + let acct_blocks = reader.blocks_changed_account(&addr).unwrap().unwrap(); + assert_eq!(acct_blocks.iter().collect::>(), vec![5, 10]); + }); + } } diff --git a/crates/hot-mdbx/src/tx.rs b/crates/hot-mdbx/src/tx.rs index e62415f..c16d996 100644 --- a/crates/hot-mdbx/src/tx.rs +++ b/crates/hot-mdbx/src/tx.rs @@ -26,6 +26,11 @@ pub struct Tx { fsi_cache: FsiCache, } +/// Per-shard byte budget for sharded history tables. Derived from MDBX's +/// DUPSORT value cap (~1980 B on 4 KB pages) minus key2 and per-node +/// overhead, with comfortable headroom for roaring encoding variability. +pub(crate) const MAX_SHARD_BYTES: usize = 1500; + impl std::fmt::Debug for Tx { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Tx").field("fsi_cache", &self.fsi_cache).finish_non_exhaustive() @@ -390,3 +395,215 @@ macro_rules! impl_hot_kv_write { impl_hot_kv_write!(RwSync); impl_hot_kv_write!(Rw); + +macro_rules! impl_history_write { + ($ty:ty) => { + impl signet_hot::db::HistoryWrite for Tx<$ty> { + fn append_account_history( + &self, + addr: &alloy::primitives::Address, + new_blocks: &signet_storage_types::BlockNumberList, + ) -> Result<(), signet_hot::db::HistoryError> { + use signet_hot::{db::HistoryError, model::HotKvWrite, tables}; + use signet_storage_types::merge_and_split; + + let existing_tail = self + .get_dual::(addr, &u64::MAX) + .map_err(HistoryError::Db)? + .unwrap_or_default(); + + if !existing_tail.is_empty() { + self.queue_delete_dual::(addr, &u64::MAX) + .map_err(HistoryError::Db)?; + } + + let (first, second) = + merge_and_split(existing_tail, new_blocks.clone(), MAX_SHARD_BYTES); + + match second { + None => self + .queue_put_dual::(addr, &u64::MAX, &first) + .map_err(HistoryError::Db), + Some(tail) => { + let seal_key = first.max().expect("first non-empty after split"); + self.queue_put_dual::(addr, &seal_key, &first) + .map_err(HistoryError::Db)?; + self.queue_put_dual::(addr, &u64::MAX, &tail) + .map_err(HistoryError::Db) + } + } + } + + fn append_storage_history( + &self, + addr: &alloy::primitives::Address, + slot: &alloy::primitives::U256, + new_blocks: &signet_storage_types::BlockNumberList, + ) -> Result<(), signet_hot::db::HistoryError> { + use signet_hot::{db::HistoryError, model::HotKvWrite, tables}; + use signet_storage_types::{ShardedKey, merge_and_split}; + + let tail_key = ShardedKey::new(*slot, u64::MAX); + let existing_tail = self + .get_dual::(addr, &tail_key) + .map_err(HistoryError::Db)? + .unwrap_or_default(); + + if !existing_tail.is_empty() { + self.queue_delete_dual::(addr, &tail_key) + .map_err(HistoryError::Db)?; + } + + let (first, second) = + merge_and_split(existing_tail, new_blocks.clone(), MAX_SHARD_BYTES); + + match second { + None => self + .queue_put_dual::(addr, &tail_key, &first) + .map_err(HistoryError::Db), + Some(tail) => { + let seal_block = first.max().expect("first non-empty after split"); + let seal_key = ShardedKey::new(*slot, seal_block); + self.queue_put_dual::(addr, &seal_key, &first) + .map_err(HistoryError::Db)?; + self.queue_put_dual::(addr, &tail_key, &tail) + .map_err(HistoryError::Db) + } + } + } + + fn truncate_account_history_above( + &self, + addr: &alloy::primitives::Address, + above: u64, + ) -> Result<(), signet_hot::db::HistoryError> { + use signet_hot::{ + db::HistoryError, + model::{HotKvRead, HotKvWrite}, + tables, + }; + use signet_storage_types::BlockNumberList; + + let mut cursor = + self.traverse_dual::().map_err(HistoryError::Db)?; + + let Some((_, mut key2, mut list)) = + cursor.last_of_k1(addr).map_err(HistoryError::Db)? + else { + return Ok(()); + }; + + let mut deleted_above = false; + + loop { + let max_in_shard = list.max().unwrap_or(0); + + if max_in_shard <= above { + if deleted_above && key2 != u64::MAX { + self.queue_delete_dual::(addr, &key2) + .map_err(HistoryError::Db)?; + self.queue_put_dual::(addr, &u64::MAX, &list) + .map_err(HistoryError::Db)?; + } + return Ok(()); + } + + self.queue_delete_dual::(addr, &key2) + .map_err(HistoryError::Db)?; + + let kept = + BlockNumberList::new_pre_sorted(list.iter().take_while(|&b| b <= above)); + if !kept.is_empty() { + self.queue_put_dual::(addr, &u64::MAX, &kept) + .map_err(HistoryError::Db)?; + return Ok(()); + } + + deleted_above = true; + let Some((_, prev_key2, prev_list)) = + cursor.previous_k2().map_err(HistoryError::Db)? + else { + return Ok(()); + }; + key2 = prev_key2; + list = prev_list; + } + } + + fn truncate_storage_history_above( + &self, + addr: &alloy::primitives::Address, + slot: &alloy::primitives::U256, + above: u64, + ) -> Result<(), signet_hot::db::HistoryError> { + use signet_hot::{ + db::HistoryError, + model::{HotKvRead, HotKvWrite}, + tables, + }; + use signet_storage_types::{BlockNumberList, ShardedKey}; + + let mut cursor = + self.traverse_dual::().map_err(HistoryError::Db)?; + + let tail_key = ShardedKey::new(*slot, u64::MAX); + + // Walk backwards from the largest dup for this addr until we + // find one matching this slot. The cursor may start on a + // different slot for the same addr. + let mut cur_entry = cursor.last_of_k1(addr).map_err(HistoryError::Db)?; + loop { + match cur_entry { + None => return Ok(()), + Some((_, ref sk, _)) if sk.key == *slot => break, + Some(_) => { + cur_entry = cursor.previous_k2().map_err(HistoryError::Db)?; + } + } + } + let (_, mut sk, mut list) = cur_entry.expect("matched above"); + + let mut deleted_above = false; + + loop { + let max_in_shard = list.max().unwrap_or(0); + + if max_in_shard <= above { + if deleted_above && sk != tail_key { + self.queue_delete_dual::(addr, &sk) + .map_err(HistoryError::Db)?; + self.queue_put_dual::(addr, &tail_key, &list) + .map_err(HistoryError::Db)?; + } + return Ok(()); + } + + self.queue_delete_dual::(addr, &sk) + .map_err(HistoryError::Db)?; + + let kept = + BlockNumberList::new_pre_sorted(list.iter().take_while(|&b| b <= above)); + if !kept.is_empty() { + self.queue_put_dual::(addr, &tail_key, &kept) + .map_err(HistoryError::Db)?; + return Ok(()); + } + + deleted_above = true; + let prev = cursor.previous_k2().map_err(HistoryError::Db)?; + match prev { + None => return Ok(()), + Some((_, ref prev_sk, _)) if prev_sk.key != *slot => return Ok(()), + Some((_, prev_sk, prev_list)) => { + sk = prev_sk; + list = prev_list; + } + } + } + } + } + }; +} + +impl_history_write!(RwSync); +impl_history_write!(Rw); From c381525eca084dbbe761e220d4700c9f106f8c18 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 07:28:21 -0400 Subject: [PATCH 10/21] refactor(hot): flip revm test setup to logical HistoryWrite setup_history_kv now uses append_account_history / append_storage_history instead of the shard-aware write_*_history methods. The at-height tests exercise the new HistoryRead default-impl path through get_account_at_height / get_storage_at_height. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/src/model/revm.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/hot/src/model/revm.rs b/crates/hot/src/model/revm.rs index b1dcf72..1c0d242 100644 --- a/crates/hot/src/model/revm.rs +++ b/crates/hot/src/model/revm.rs @@ -445,7 +445,7 @@ where mod tests { use super::*; use crate::{ - db::{LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, + db::{HistoryWrite, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, mem::MemKv, model::{HotKv, HotKvRead, HotKvWrite}, tables::{Bytecodes, PlainAccountState}, @@ -834,7 +834,7 @@ mod tests { // Account history shard: blocks 5 and 10 touched address let history = BlockNumberList::new([5, 10]).unwrap(); - writer.write_account_history(&address, 10, &history).unwrap(); + writer.append_account_history(&address, &history).unwrap(); // Account change sets (pre-states) let pre_state_5 = Account { nonce: 1, balance: U256::from(100u64), bytecode_hash: None }; @@ -845,7 +845,7 @@ mod tests { // Storage history shard: blocks 5 and 10 touched (address, slot) let storage_history = BlockNumberList::new([5, 10]).unwrap(); - writer.write_storage_history(&address, slot, 10, &storage_history).unwrap(); + writer.append_storage_history(&address, &slot, &storage_history).unwrap(); // Storage change sets (pre-states) writer.write_storage_prestate(5, address, &slot, &U256::ZERO).unwrap(); From 736f089b0ca5c9d9ba96223b2278abcf982a4fc0 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 07:41:04 -0400 Subject: [PATCH 11/21] refactor(hot): move consistent history ops onto new HistoryWrite unwind_above's two shard-surgery blocks become single calls to truncate_account_history_above / truncate_storage_history_above. load_genesis uses append_account_history / append_storage_history. validate_chain_extension and append_blocks move to HistoryWrite as default impls. The LegacyConsistentHistoryWrite trait and crates/hot/src/db/consistent.rs are deleted; ADDRESS_MAX moves alongside the new default impls. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot-mdbx/src/test_utils.rs | 2 +- crates/hot/README.md | 4 +- crates/hot/src/conformance/unwind.rs | 7 +- crates/hot/src/db/consistent.rs | 347 --------------------------- crates/hot/src/db/history.rs | 323 ++++++++++++++++++++++++- crates/hot/src/db/inconsistent.rs | 4 +- crates/hot/src/db/mod.rs | 3 - crates/hot/src/lib.rs | 10 +- crates/hot/src/model/traits.rs | 4 +- crates/hot/tests/load_genesis.rs | 2 +- crates/storage/README.md | 2 +- crates/storage/src/lib.rs | 2 +- crates/storage/src/unified.rs | 7 +- crates/storage/tests/unified.rs | 4 +- 14 files changed, 344 insertions(+), 377 deletions(-) delete mode 100644 crates/hot/src/db/consistent.rs diff --git a/crates/hot-mdbx/src/test_utils.rs b/crates/hot-mdbx/src/test_utils.rs index a39028b..bddab14 100644 --- a/crates/hot-mdbx/src/test_utils.rs +++ b/crates/hot-mdbx/src/test_utils.rs @@ -1227,7 +1227,7 @@ mod tests { }; use signet_hot::{ conformance::make_bundle_state, - db::{HotDbRead, LegacyConsistentHistoryWrite}, + db::{HistoryWrite, HotDbRead}, model::{DualTableTraverse, HotKvRead}, tables, }; diff --git a/crates/hot/README.md b/crates/hot/README.md index b45aa97..b936b2a 100644 --- a/crates/hot/README.md +++ b/crates/hot/README.md @@ -8,7 +8,7 @@ with opinionated serialization and predefined tables for blockchain state. ## Usage ```rust,ignore -use signet_hot::{HotKv, LegacyHistoryRead, LegacyConsistentHistoryWrite}; +use signet_hot::{HotKv, LegacyHistoryRead, HistoryWrite}; fn example(db: &D) -> Result<(), signet_hot::db::HotKvError> { // Read operations @@ -35,7 +35,7 @@ HotKv ← Transaction factory │ └─ LegacyHistoryRead ← History queries (blanket impl) └─ writer() → HotKvWrite ← Read-write transactions └─ UnsafeDbWrite ← Low-level writes (blanket impl) - └─ LegacyConsistentHistoryWrite ← Safe chain operations (blanket impl) + └─ HistoryWrite ← Safe chain operations (per-backend impl) ``` ## Serialization diff --git a/crates/hot/src/conformance/unwind.rs b/crates/hot/src/conformance/unwind.rs index 07b0d01..c91190c 100644 --- a/crates/hot/src/conformance/unwind.rs +++ b/crates/hot/src/conformance/unwind.rs @@ -1,7 +1,7 @@ //! Unwind conformance test and helpers. use crate::{ - db::{LegacyConsistentHistoryWrite, UnsafeDbWrite}, + db::{HistoryWrite, UnsafeDbWrite}, model::{DualKeyValue, DualTableTraverse, HotKv, HotKvRead, KeyValue, TableTraverse}, tables::{self, DualKey, SingleKey}, }; @@ -228,7 +228,10 @@ pub fn make_account_info(nonce: u64, balance: U256, code_hash: Option) -> /// - Headers and header number mappings /// - Account and storage change sets /// - Account and storage history indices -pub fn test_unwind_conformance(store_a: &Kv, store_b: &Kv) { +pub fn test_unwind_conformance(store_a: &Kv, store_b: &Kv) +where + Kv::RwTx: HistoryWrite, +{ // Test addresses let addr1 = address!("0x1111111111111111111111111111111111111111"); let addr2 = address!("0x2222222222222222222222222222222222222222"); diff --git a/crates/hot/src/db/consistent.rs b/crates/hot/src/db/consistent.rs deleted file mode 100644 index 4bb0a26..0000000 --- a/crates/hot/src/db/consistent.rs +++ /dev/null @@ -1,347 +0,0 @@ -use crate::{ - db::{HistoryError, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, - tables, -}; -use ahash::AHashSet; -use alloy::{ - consensus::Sealable, - genesis::{Genesis, GenesisAccount}, - primitives::{Address, B256, BlockNumber, U256, address}, -}; -use signet_storage_types::{Account, BlockNumberList, EthereumHardfork, SealedHeader}; -use trevm::revm::{database::BundleState, state::Bytecode}; - -/// Maximum address value (all bits set to 1). -const ADDRESS_MAX: Address = address!("0xffffffffffffffffffffffffffffffffffffffff"); - -/// Trait for database write operations on hot history tables. This trait -/// maintains a consistent state of the database. -pub trait LegacyConsistentHistoryWrite: UnsafeDbWrite + LegacyUnsafeHistoryWrite { - /// Validate that a range of headers forms a valid chain extension. - /// - /// Headers must be in order and each must extend the previous. - /// The first header must extend the current database tip (or be the first - /// block if the database is empty). - /// - /// Returns `Ok(())` if valid, or an error describing the inconsistency. - fn validate_chain_extension<'a, I>(&self, headers: I) -> Result<(), HistoryError> - where - I: IntoIterator, - { - let mut iter = headers.into_iter(); - let first = iter.next().ok_or(HistoryError::EmptyRange)?; - - // Validate first header against current DB tip - match self.get_chain_tip().map_err(HistoryError::Db)? { - None => { - // Empty DB - first block is valid as genesis - } - Some((tip_number, tip_hash)) => { - let expected_number = tip_number + 1; - if first.number != expected_number { - return Err(HistoryError::NonContiguousBlock { - expected: expected_number, - got: first.number, - }); - } - if first.parent_hash != tip_hash { - return Err(HistoryError::ParentHashMismatch { - expected: tip_hash, - got: first.parent_hash, - }); - } - } - } - - // Validate each subsequent header extends the previous using fold - iter.try_fold(first, |prev, curr| { - let expected_number = prev.number + 1; - if curr.number != expected_number { - return Err(HistoryError::NonContiguousBlock { - expected: expected_number, - got: curr.number, - }); - } - - let expected_hash = prev.hash(); - if curr.parent_hash != expected_hash { - return Err(HistoryError::ParentHashMismatch { - expected: expected_hash, - got: curr.parent_hash, - }); - } - - Ok(curr) - })?; - - Ok(()) - } - - /// Append a range of blocks and their associated state to the database. - fn append_blocks<'a>( - &self, - blocks: impl IntoIterator, - ) -> Result<(), HistoryError> { - let mut iter = blocks.into_iter(); - - let Some((first_header, first_bundle)) = iter.next() else { - return Err(HistoryError::EmptyRange); - }; - - // Validate first header against DB tip - match self.get_chain_tip().map_err(HistoryError::Db)? { - None => { /* Empty DB - first block is valid as genesis */ } - Some((tip_number, tip_hash)) => { - let expected_number = tip_number + 1; - if first_header.number != expected_number { - return Err(HistoryError::NonContiguousBlock { - expected: expected_number, - got: first_header.number, - }); - } - if first_header.parent_hash != tip_hash { - return Err(HistoryError::ParentHashMismatch { - expected: tip_hash, - got: first_header.parent_hash, - }); - } - } - } - - // Write first block and track range - self.append_block_inconsistent(first_header, first_bundle)?; - let first_num = first_header.number; - let mut last_num = first_num; - let mut prev = first_header; - - // Process remaining: validate chain continuity and write in one pass - for (header, bundle) in iter { - let expected_number = prev.number + 1; - if header.number != expected_number { - return Err(HistoryError::NonContiguousBlock { - expected: expected_number, - got: header.number, - }); - } - let expected_hash = prev.hash(); - if header.parent_hash != expected_hash { - return Err(HistoryError::ParentHashMismatch { - expected: expected_hash, - got: header.parent_hash, - }); - } - - self.append_block_inconsistent(header, bundle)?; - last_num = header.number; - prev = header; - } - - self.update_history_indices_inconsistent(first_num..=last_num) - } - - /// Unwind all data above the given block number. - /// - /// This completely reverts the database state to what it was at block - /// `block`, including: - /// - Plain account state - /// - Plain storage state - /// - Headers and header number mappings - /// - Account and storage change sets - /// - Account and storage history indices - fn unwind_above(&self, block: BlockNumber) -> Result<(), HistoryError> { - let first_block = block + 1; - let Some(last_block) = self.last_block_number()? else { - return Ok(()); - }; - - if first_block > last_block { - return Ok(()); - } - - // ═══════════════════════════════════════════════════════════════════ - // 1. STREAM AccountChangeSets → restore + filter history in one pass - // ═══════════════════════════════════════════════════════════════════ - // TODO: estimate capacity from block range size for better allocation - let mut seen_accounts: AHashSet
= AHashSet::new(); - let mut account_cursor = self.traverse_dual::()?; - - // Position at first entry - let mut current = account_cursor.next_dual_above(&first_block, &Address::ZERO)?; - - while let Some((block_num, address, old_account)) = current { - if block_num > last_block { - break; - } - - // First occurrence = process both plain state and history - if seen_accounts.insert(address) { - // Restore plain state - if old_account.is_empty() { - self.queue_delete::(&address)?; - } else { - self.put_account(&address, &old_account)?; - } - - // Filter history index - if let Some((shard_key, list)) = self.last_account_history(address)? { - self.queue_delete_dual::(&address, &shard_key)?; - let mut filtered = list.iter().take_while(|&bn| bn <= block).peekable(); - if filtered.peek().is_some() { - self.write_account_history( - &address, - u64::MAX, - &BlockNumberList::new_pre_sorted(filtered), - )?; - } - } - } - - current = account_cursor.read_next()?; - } - - // ═══════════════════════════════════════════════════════════════════ - // 2. STREAM StorageChangeSets → restore + filter history in one pass - // ═══════════════════════════════════════════════════════════════════ - // TODO: estimate capacity from block range size for better allocation - let mut seen_storage: AHashSet<(Address, U256)> = AHashSet::new(); - let mut storage_cursor = self.traverse_dual::()?; - - // Position at first entry - let mut current_storage = - storage_cursor.next_dual_above(&(first_block, Address::ZERO), &U256::ZERO)?; - - while let Some(((block_num, address), slot, old_value)) = current_storage { - if block_num > last_block { - break; - } - - if seen_storage.insert((address, slot)) { - // Restore plain state - if old_value.is_zero() { - self.queue_delete_dual::(&address, &slot)?; - } else { - self.put_storage(&address, &slot, &old_value)?; - } - - // Filter history index - if let Some((shard_key, list)) = self.last_storage_history(&address, &slot)? { - self.queue_delete_dual::(&address, &shard_key)?; - let mut filtered = list.iter().take_while(|&bn| bn <= block).peekable(); - if filtered.peek().is_some() { - self.write_storage_history( - &address, - slot, - u64::MAX, - &BlockNumberList::new_pre_sorted(filtered), - )?; - } - } - } - - current_storage = storage_cursor.read_next()?; - } - - // ═══════════════════════════════════════════════════════════════════ - // 3. DELETE changeset ranges - // ═══════════════════════════════════════════════════════════════════ - self.traverse_dual_mut::()? - .delete_range((first_block, Address::ZERO)..=(last_block, ADDRESS_MAX))?; - self.traverse_dual_mut::()?.delete_range( - ((first_block, Address::ZERO), U256::ZERO)..=((last_block, ADDRESS_MAX), U256::MAX), - )?; - - // ═══════════════════════════════════════════════════════════════════ - // 4. STREAM Headers → delete HeaderNumbers, then clear Headers - // ═══════════════════════════════════════════════════════════════════ - let mut header_cursor = self.traverse::()?; - - // Position at first entry and process it - let first_entry = header_cursor.lower_bound(&first_block)?; - if let Some((block_num, header)) = first_entry - && block_num <= last_block - { - self.delete_header_number(&header.hash())?; - - // Continue with remaining entries - while let Some((block_num, header)) = header_cursor.read_next()? { - if block_num > last_block { - break; - } - self.delete_header_number(&header.hash())?; - } - } - self.traverse_mut::()?.delete_range_inclusive(first_block..=last_block)?; - - Ok(()) - } - - /// Load genesis data into the database. - /// - /// This operation is only valid on an empty database. - fn load_genesis( - &self, - genesis: &Genesis, - genesis_hardforks: &EthereumHardfork, - ) -> Result<(), HistoryError> { - // Check that the database is empty - if self.get_chain_tip().map_err(HistoryError::Db)?.is_some() { - return Err(HistoryError::DbNotEmpty); - } - - // Seal the genesis header, record its number, and create a blocknumber - // list. - let header = signet_storage_types::genesis_header(genesis, genesis_hardforks).seal_slow(); - let genesis_number = header.number; - let genesis_history = BlockNumberList::new_pre_sorted([genesis_number]); - - // Append the header, with empty state - self.append_blocks([(&header, &BundleState::default())])?; - - // Keep track of written bytecode hashes to avoid duplicates. - let mut written_bytecode_hashes: AHashSet = AHashSet::new(); - - // For each account in the genesis allocation, append account. - // The accounts are pre-sorted by the BTreeMap in Genesis. - genesis.alloc.iter().try_for_each(|(address, account)| { - let GenesisAccount { nonce, balance, code, storage, .. } = account; - - // Insert bytecode if present. Check against the set to avoid - // duplicate writes. We still have to compute the hash though. - let bytecode_hash = code - .as_ref() - .map(|code_bytes| -> Result<_, HistoryError> { - let hash = alloy::primitives::keccak256(code_bytes); - // Short-circuit if already written - if !written_bytecode_hashes.insert(hash) { - return Ok(hash); - } - self.put_bytecode(&hash, &Bytecode::new_raw(code_bytes.clone()))?; - Ok(hash) - }) - .transpose()?; - - // Append the account. - self.append_account( - address, - &Account { nonce: nonce.unwrap_or_default(), balance: *balance, bytecode_hash }, - )?; - - // Record account history at genesis - self.write_account_history(address, u64::MAX, &genesis_history)?; - - // Insert storage entries and history - storage.iter().flatten().try_for_each(|(slot, value)| { - let slot = U256::from_be_bytes(**slot); - // We can append directly since the slots are sorted and the - // db is empty. - self.append_storage(address, &slot, &U256::from_be_bytes(**value))?; - // Record storage history at genesis - self.write_storage_history(address, slot, u64::MAX, &genesis_history)?; - Ok::<(), HistoryError>(()) - })?; - Ok(()) - }) - } -} - -impl LegacyConsistentHistoryWrite for T where T: UnsafeDbWrite + LegacyUnsafeHistoryWrite {} diff --git a/crates/hot/src/db/history.rs b/crates/hot/src/db/history.rs index b3a50b0..f4e6618 100644 --- a/crates/hot/src/db/history.rs +++ b/crates/hot/src/db/history.rs @@ -8,15 +8,23 @@ //! MemKv writes a single dup entry per addr). use crate::{ - db::{HistoryError, HotDbRead, UnsafeDbWrite}, + db::{HistoryError, HotDbRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, model::HotKvRead, tables, }; -use ahash::AHashMap; -use alloy::primitives::{Address, BlockNumber, U256}; +use ahash::{AHashMap, AHashSet}; +use alloy::{ + consensus::Sealable, + genesis::{Genesis, GenesisAccount}, + primitives::{Address, B256, BlockNumber, U256, address}, +}; use itertools::Itertools; -use signet_storage_types::{Account, BlockNumberList, ShardedKey}; +use signet_storage_types::{Account, BlockNumberList, EthereumHardfork, SealedHeader, ShardedKey}; use std::ops::RangeInclusive; +use trevm::revm::{database::BundleState, state::Bytecode}; + +/// Maximum address value (all bits set to 1). +const ADDRESS_MAX: Address = address!("0xffffffffffffffffffffffffffffffffffffffff"); /// Logical reads against history + changeset tables. /// @@ -176,7 +184,7 @@ impl HistoryRead for T where T: HotKvRead {} /// Backends that implement this trait choose their own shard-splitting policy. /// The default `update_history_indices` bulk operation is expressed in terms of /// the four required primitives and works for any backend. -pub trait HistoryWrite: UnsafeDbWrite + HistoryRead { +pub trait HistoryWrite: UnsafeDbWrite + LegacyUnsafeHistoryWrite + HistoryRead { /// Merge `new_blocks` into `addr`'s account history. /// /// Preconditions: `new_blocks` is sorted ascending and every entry is @@ -264,4 +272,309 @@ pub trait HistoryWrite: UnsafeDbWrite + HistoryRead { Ok(()) } + + /// Validate that a range of headers forms a valid chain extension. + /// + /// Headers must be in order and each must extend the previous. + /// The first header must extend the current database tip (or be the first + /// block if the database is empty). + /// + /// Returns `Ok(())` if valid, or an error describing the inconsistency. + fn validate_chain_extension<'a, I>(&self, headers: I) -> Result<(), HistoryError> + where + I: IntoIterator, + { + let mut iter = headers.into_iter(); + let first = iter.next().ok_or(HistoryError::EmptyRange)?; + + // Validate first header against current DB tip + match self.get_chain_tip().map_err(HistoryError::Db)? { + None => { + // Empty DB - first block is valid as genesis + } + Some((tip_number, tip_hash)) => { + let expected_number = tip_number + 1; + if first.number != expected_number { + return Err(HistoryError::NonContiguousBlock { + expected: expected_number, + got: first.number, + }); + } + if first.parent_hash != tip_hash { + return Err(HistoryError::ParentHashMismatch { + expected: tip_hash, + got: first.parent_hash, + }); + } + } + } + + // Validate each subsequent header extends the previous using fold + iter.try_fold(first, |prev, curr| { + let expected_number = prev.number + 1; + if curr.number != expected_number { + return Err(HistoryError::NonContiguousBlock { + expected: expected_number, + got: curr.number, + }); + } + + let expected_hash = prev.hash(); + if curr.parent_hash != expected_hash { + return Err(HistoryError::ParentHashMismatch { + expected: expected_hash, + got: curr.parent_hash, + }); + } + + Ok(curr) + })?; + + Ok(()) + } + + /// Append a range of blocks and their associated state to the database. + fn append_blocks<'a>( + &self, + blocks: impl IntoIterator, + ) -> Result<(), HistoryError> { + let mut iter = blocks.into_iter(); + + let Some((first_header, first_bundle)) = iter.next() else { + return Err(HistoryError::EmptyRange); + }; + + // Validate first header against DB tip + match self.get_chain_tip().map_err(HistoryError::Db)? { + None => { /* Empty DB - first block is valid as genesis */ } + Some((tip_number, tip_hash)) => { + let expected_number = tip_number + 1; + if first_header.number != expected_number { + return Err(HistoryError::NonContiguousBlock { + expected: expected_number, + got: first_header.number, + }); + } + if first_header.parent_hash != tip_hash { + return Err(HistoryError::ParentHashMismatch { + expected: tip_hash, + got: first_header.parent_hash, + }); + } + } + } + + // Write first block and track range + self.append_block_inconsistent(first_header, first_bundle)?; + let first_num = first_header.number; + let mut last_num = first_num; + let mut prev = first_header; + + // Process remaining: validate chain continuity and write in one pass + for (header, bundle) in iter { + let expected_number = prev.number + 1; + if header.number != expected_number { + return Err(HistoryError::NonContiguousBlock { + expected: expected_number, + got: header.number, + }); + } + let expected_hash = prev.hash(); + if header.parent_hash != expected_hash { + return Err(HistoryError::ParentHashMismatch { + expected: expected_hash, + got: header.parent_hash, + }); + } + + self.append_block_inconsistent(header, bundle)?; + last_num = header.number; + prev = header; + } + + self.update_history_indices(first_num..=last_num) + } + + /// Unwind all data above the given block number. + /// + /// This completely reverts the database state to what it was at block + /// `block`, including: + /// - Plain account state + /// - Plain storage state + /// - Headers and header number mappings + /// - Account and storage change sets + /// - Account and storage history indices + fn unwind_above(&self, block: BlockNumber) -> Result<(), HistoryError> { + let first_block = block + 1; + let Some(last_block) = self.last_block_number()? else { + return Ok(()); + }; + + if first_block > last_block { + return Ok(()); + } + + // ═══════════════════════════════════════════════════════════════════ + // 1. STREAM AccountChangeSets → restore + filter history in one pass + // ═══════════════════════════════════════════════════════════════════ + // TODO: estimate capacity from block range size for better allocation + let mut seen_accounts: AHashSet
= AHashSet::new(); + let mut account_cursor = self.traverse_dual::()?; + + // Position at first entry + let mut current = account_cursor.next_dual_above(&first_block, &Address::ZERO)?; + + while let Some((block_num, address, old_account)) = current { + if block_num > last_block { + break; + } + + // First occurrence = process both plain state and history + if seen_accounts.insert(address) { + // Restore plain state + if old_account.is_empty() { + self.queue_delete::(&address)?; + } else { + self.put_account(&address, &old_account)?; + } + + // Truncate history above `block` (logical, no shard surgery) + self.truncate_account_history_above(&address, block)?; + } + + current = account_cursor.read_next()?; + } + + // ═══════════════════════════════════════════════════════════════════ + // 2. STREAM StorageChangeSets → restore + filter history in one pass + // ═══════════════════════════════════════════════════════════════════ + // TODO: estimate capacity from block range size for better allocation + let mut seen_storage: AHashSet<(Address, U256)> = AHashSet::new(); + let mut storage_cursor = self.traverse_dual::()?; + + // Position at first entry + let mut current_storage = + storage_cursor.next_dual_above(&(first_block, Address::ZERO), &U256::ZERO)?; + + while let Some(((block_num, address), slot, old_value)) = current_storage { + if block_num > last_block { + break; + } + + if seen_storage.insert((address, slot)) { + // Restore plain state + if old_value.is_zero() { + self.queue_delete_dual::(&address, &slot)?; + } else { + self.put_storage(&address, &slot, &old_value)?; + } + + // Truncate history above `block` (logical, no shard surgery) + self.truncate_storage_history_above(&address, &slot, block)?; + } + + current_storage = storage_cursor.read_next()?; + } + + // ═══════════════════════════════════════════════════════════════════ + // 3. DELETE changeset ranges + // ═══════════════════════════════════════════════════════════════════ + self.traverse_dual_mut::()? + .delete_range((first_block, Address::ZERO)..=(last_block, ADDRESS_MAX))?; + self.traverse_dual_mut::()?.delete_range( + ((first_block, Address::ZERO), U256::ZERO)..=((last_block, ADDRESS_MAX), U256::MAX), + )?; + + // ═══════════════════════════════════════════════════════════════════ + // 4. STREAM Headers → delete HeaderNumbers, then clear Headers + // ═══════════════════════════════════════════════════════════════════ + let mut header_cursor = self.traverse::()?; + + // Position at first entry and process it + let first_entry = header_cursor.lower_bound(&first_block)?; + if let Some((block_num, header)) = first_entry + && block_num <= last_block + { + self.delete_header_number(&header.hash())?; + + // Continue with remaining entries + while let Some((block_num, header)) = header_cursor.read_next()? { + if block_num > last_block { + break; + } + self.delete_header_number(&header.hash())?; + } + } + self.traverse_mut::()?.delete_range_inclusive(first_block..=last_block)?; + + Ok(()) + } + + /// Load genesis data into the database. + /// + /// This operation is only valid on an empty database. + fn load_genesis( + &self, + genesis: &Genesis, + genesis_hardforks: &EthereumHardfork, + ) -> Result<(), HistoryError> { + // Check that the database is empty + if self.get_chain_tip().map_err(HistoryError::Db)?.is_some() { + return Err(HistoryError::DbNotEmpty); + } + + // Seal the genesis header, record its number, and create a blocknumber + // list. + let header = signet_storage_types::genesis_header(genesis, genesis_hardforks).seal_slow(); + let genesis_number = header.number; + let genesis_history = BlockNumberList::new_pre_sorted([genesis_number]); + + // Append the header, with empty state + self.append_blocks([(&header, &BundleState::default())])?; + + // Keep track of written bytecode hashes to avoid duplicates. + let mut written_bytecode_hashes: AHashSet = AHashSet::new(); + + // For each account in the genesis allocation, append account. + // The accounts are pre-sorted by the BTreeMap in Genesis. + genesis.alloc.iter().try_for_each(|(address, account)| { + let GenesisAccount { nonce, balance, code, storage, .. } = account; + + // Insert bytecode if present. Check against the set to avoid + // duplicate writes. We still have to compute the hash though. + let bytecode_hash = code + .as_ref() + .map(|code_bytes| -> Result<_, HistoryError> { + let hash = alloy::primitives::keccak256(code_bytes); + // Short-circuit if already written + if !written_bytecode_hashes.insert(hash) { + return Ok(hash); + } + self.put_bytecode(&hash, &Bytecode::new_raw(code_bytes.clone()))?; + Ok(hash) + }) + .transpose()?; + + // Append the account. + self.append_account( + address, + &Account { nonce: nonce.unwrap_or_default(), balance: *balance, bytecode_hash }, + )?; + + // Record account history at genesis + self.append_account_history(address, &genesis_history)?; + + // Insert storage entries and history + storage.iter().flatten().try_for_each(|(slot, value)| { + let slot = U256::from_be_bytes(**slot); + // We can append directly since the slots are sorted and the + // db is empty. + self.append_storage(address, &slot, &U256::from_be_bytes(**value))?; + // Record storage history at genesis + self.append_storage_history(address, &slot, &genesis_history)?; + Ok::<(), HistoryError>(()) + })?; + Ok(()) + }) + } } diff --git a/crates/hot/src/db/inconsistent.rs b/crates/hot/src/db/inconsistent.rs index 8eb4302..65f5267 100644 --- a/crates/hot/src/db/inconsistent.rs +++ b/crates/hot/src/db/inconsistent.rs @@ -27,9 +27,9 @@ pub type BundleInit = /// /// This trait is low-level, and usage may leave the database in an /// inconsistent state if not used carefully. Users should prefer -/// [`HotHistoryWrite`] or higher-level abstractions when possible. +/// [`HistoryWrite`] or higher-level abstractions when possible. /// -/// [`HotHistoryWrite`]: crate::db::LegacyConsistentHistoryWrite +/// [`HistoryWrite`]: crate::db::HistoryWrite pub trait UnsafeDbWrite: HotKvWrite + super::sealed::Sealed { /// Write a block header. This will leave the DB in an inconsistent state /// until the corresponding header number is also written. Users should diff --git a/crates/hot/src/db/mod.rs b/crates/hot/src/db/mod.rs index 64e6389..43993c1 100644 --- a/crates/hot/src/db/mod.rs +++ b/crates/hot/src/db/mod.rs @@ -1,8 +1,5 @@ //! Primary access traits for hot storage backends. -mod consistent; -pub use consistent::LegacyConsistentHistoryWrite; - mod errors; pub use errors::{HistoryError, HistoryResult}; diff --git a/crates/hot/src/lib.rs b/crates/hot/src/lib.rs index 5ea8bd3..e4288e9 100644 --- a/crates/hot/src/lib.rs +++ b/crates/hot/src/lib.rs @@ -7,7 +7,7 @@ //! # Quick Start //! //! ```ignore -//! use signet_hot::{HotKv, LegacyHistoryRead, LegacyConsistentHistoryWrite}; +//! use signet_hot::{HotKv, LegacyHistoryRead, HistoryWrite}; //! //! fn example(db: &D) -> Result<(), signet_hot::db::HotKvError> { //! // Read operations @@ -34,7 +34,7 @@ //! │ └─ LegacyHistoryRead ← History queries (blanket impl) //! └─ writer() → HotKvWrite ← Read-write transactions //! └─ UnsafeDbWrite ← Low-level writes (blanket impl) -//! └─ LegacyConsistentHistoryWrite ← Safe chain operations (blanket impl) +//! └─ HistoryWrite ← Safe chain operations (per-backend impl) //! ``` //! //! ## Serialization @@ -56,7 +56,7 @@ //! These traits provide methods for common operations such as getting, //! setting, and deleting key-value pairs in hot storage tables. The raw //! key-value operations use byte slices for maximum flexibility. The -//! [`LegacyHistoryRead`] and [`LegacyConsistentHistoryWrite`] traits provide higher-level +//! [`LegacyHistoryRead`] and [`HistoryWrite`] traits provide higher-level //! abstractions that work with the predefined tables and their associated key //! and value types. //! @@ -74,7 +74,7 @@ //! using tables. //! //! [`LegacyHistoryRead`]: db::LegacyHistoryRead -//! [`LegacyConsistentHistoryWrite`]: db::LegacyConsistentHistoryWrite +//! [`HistoryWrite`]: db::HistoryWrite //! [`HotKvRead`]: model::HotKvRead //! [`HotKvWrite`]: model::HotKvWrite //! [`HotKv`]: model::HotKv @@ -109,7 +109,7 @@ pub mod connect; pub use connect::HotConnect; pub mod db; -pub use db::{HistoryError, LegacyConsistentHistoryWrite, LegacyHistoryRead}; +pub use db::{HistoryError, HistoryWrite, LegacyHistoryRead}; pub mod model; pub use model::HotKv; diff --git a/crates/hot/src/model/traits.rs b/crates/hot/src/model/traits.rs index 6a00c76..8f09566 100644 --- a/crates/hot/src/model/traits.rs +++ b/crates/hot/src/model/traits.rs @@ -15,14 +15,14 @@ use std::borrow::Cow; /// This is the top-level trait for hot storage backends, providing /// transactional access through read-only and read-write transactions. /// -/// We recommend using [`LegacyHistoryRead`] and [`LegacyConsistentHistoryWrite`] for most use cases, +/// We recommend using [`LegacyHistoryRead`] and [`HistoryWrite`] for most use cases, /// as they provide higher-level abstractions over predefined tables. /// /// When implementing this trait, consult the [`model`] module documentation for /// details on the associated types and their requirements. /// /// [`LegacyHistoryRead`]: crate::db::LegacyHistoryRead -/// [`LegacyConsistentHistoryWrite`]: crate::db::LegacyConsistentHistoryWrite +/// [`HistoryWrite`]: crate::db::HistoryWrite /// [`model`]: crate::model #[auto_impl::auto_impl(&, Arc, Box)] pub trait HotKv { diff --git a/crates/hot/tests/load_genesis.rs b/crates/hot/tests/load_genesis.rs index decd72a..91ac954 100644 --- a/crates/hot/tests/load_genesis.rs +++ b/crates/hot/tests/load_genesis.rs @@ -9,7 +9,7 @@ use alloy::{ }; use signet_hot::{ HotKv, - db::{HotDbRead, LegacyConsistentHistoryWrite, LegacyHistoryRead, UnsafeDbWrite}, + db::{HistoryWrite, HotDbRead, LegacyHistoryRead, UnsafeDbWrite}, mem::MemKv, }; use signet_storage_types::EthereumHardfork; diff --git a/crates/storage/README.md b/crates/storage/README.md index 091958a..8911b66 100644 --- a/crates/storage/README.md +++ b/crates/storage/README.md @@ -70,5 +70,5 @@ Cold dispatch errors indicate either: Key types are re-exported for convenience: - `ExecutedBlock`, `ExecutedBlockBuilder` - Block data structures -- `HotKv`, `LegacyHistoryRead`, `LegacyConsistentHistoryWrite` - Hot storage traits +- `HotKv`, `LegacyHistoryRead`, `HistoryWrite` - Hot storage traits - `ColdStorageHandle`, `ColdStorageError` - Cold storage types diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 3f00ff7..feac742 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -87,7 +87,7 @@ pub use signet_cold::{ }; pub use signet_cold_mdbx::MdbxColdBackend; pub use signet_hot::{ - HistoryError, HotKv, LegacyConsistentHistoryWrite, LegacyHistoryRead, + HistoryError, HistoryWrite, HotKv, LegacyHistoryRead, model::{HotKvRead, RevmRead, RevmWrite}, }; pub use signet_hot_mdbx::{DatabaseArguments, DatabaseEnv}; diff --git a/crates/storage/src/unified.rs b/crates/storage/src/unified.rs index 9accc54..20b8c56 100644 --- a/crates/storage/src/unified.rs +++ b/crates/storage/src/unified.rs @@ -8,7 +8,7 @@ use crate::StorageResult; use alloy::primitives::BlockNumber; use signet_cold::{BlockData, ColdReceipt, ColdStorage, ColdStorageBackend, ColdStorageError}; use signet_hot::{ - HotKv, LegacyConsistentHistoryWrite, LegacyHistoryRead, + HistoryWrite, HotKv, LegacyHistoryRead, model::{HotKvReadError, HotKvWrite, RevmRead}, }; use signet_storage_types::{ExecutedBlock, SealedHeader}; @@ -116,7 +116,10 @@ impl UnifiedStorage { } } -impl UnifiedStorage { +impl UnifiedStorage +where + H::RwTx: HistoryWrite, +{ /// Get a reference to the hot storage backend. pub const fn hot(&self) -> &H { &self.hot diff --git a/crates/storage/tests/unified.rs b/crates/storage/tests/unified.rs index 0f900c4..1611e93 100644 --- a/crates/storage/tests/unified.rs +++ b/crates/storage/tests/unified.rs @@ -5,9 +5,7 @@ use alloy::{ primitives::{Address, B256, Signature, TxKind, U256}, }; use signet_cold::{ColdStorage, HeaderSpecifier, mem::MemColdBackend}; -use signet_hot::{ - HotKv, LegacyConsistentHistoryWrite, LegacyHistoryRead, mem::MemKv, model::HotKvWrite, -}; +use signet_hot::{HistoryWrite, HotKv, LegacyHistoryRead, mem::MemKv, model::HotKvWrite}; use signet_storage::UnifiedStorage; use signet_storage_types::{ ExecutedBlock, ExecutedBlockBuilder, Receipt, RecoveredTx, SealedHeader, TransactionSigned, From e6b40f4e66c15a6ee547827a8b6729191e6af15c Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 07:50:10 -0400 Subject: [PATCH 12/21] fix(storage): un-gate read-only UnifiedStorage methods from HistoryWrite The previous commit accidentally moved hot(), cold(), reader(), revm_reader(), etc. into the impl block bounded by H::RwTx: HistoryWrite. These methods don't open a write transaction; the bound is unnecessary and tightens the public API for no reason. Split into two impl blocks matching the pre-refactor structure. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/storage/src/unified.rs | 112 +++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/crates/storage/src/unified.rs b/crates/storage/src/unified.rs index 20b8c56..5e9345e 100644 --- a/crates/storage/src/unified.rs +++ b/crates/storage/src/unified.rs @@ -98,28 +98,7 @@ impl UnifiedStorage { let cold = ColdStorage::new(cold_backend, cancel_token); Self::new(hot, cold) } -} - -impl UnifiedStorage { - /// Spawn a unified storage with a type-erased cold backend. - /// - /// Erases the concrete cold backend behind - /// [`signet_cold::ErasedBackend`], so callers can hold a - /// `UnifiedStorage` without propagating a backend generic. - pub fn spawn_erased( - hot: H, - cold_backend: B, - cancel_token: CancellationToken, - ) -> Self { - let cold = ColdStorage::new_erased(cold_backend, cancel_token); - Self::new(hot, cold) - } -} -impl UnifiedStorage -where - H::RwTx: HistoryWrite, -{ /// Get a reference to the hot storage backend. pub const fn hot(&self) -> &H { &self.hot @@ -163,6 +142,62 @@ where self.hot.revm_reader_at_height(height).map_err(Into::into) } + /// Check how far behind cold storage is compared to hot storage. + /// + /// Returns `Some(first_missing_block)` if cold is behind, `None` if synced. + /// + /// # Errors + /// + /// Returns an error if either storage cannot be queried. + pub async fn cold_lag(&self) -> StorageResult> { + let reader = self.reader()?; + let hot_tip = reader.get_chain_tip().map_err(|e| e.into_hot_kv_error())?; + + let cold_tip = self.cold.get_latest_block().await?; + + match (hot_tip, cold_tip) { + (Some((hot_num, _)), Some(cold_num)) if cold_num < hot_num => Ok(Some(cold_num + 1)), + (Some((_, _)), None) => Ok(Some(0)), + _ => Ok(None), + } + } + + /// Replay blocks to cold storage from an external source. + /// + /// Use this to recover cold storage after failures. The caller is + /// responsible for fetching the missing block data. + /// + /// Consumes the blocks to avoid cloning. + /// + /// # Errors + /// + /// Returns an error if cold storage write fails. + pub async fn replay_to_cold(&self, blocks: Vec) -> Result<(), ColdStorageError> { + let cold_data: Vec<_> = blocks.into_iter().map(BlockData::from).collect(); + self.cold.append_blocks(cold_data).await + } +} + +impl UnifiedStorage { + /// Spawn a unified storage with a type-erased cold backend. + /// + /// Erases the concrete cold backend behind + /// [`signet_cold::ErasedBackend`], so callers can hold a + /// `UnifiedStorage` without propagating a backend generic. + pub fn spawn_erased( + hot: H, + cold_backend: B, + cancel_token: CancellationToken, + ) -> Self { + let cold = ColdStorage::new_erased(cold_backend, cancel_token); + Self::new(hot, cold) + } +} + +impl UnifiedStorage +where + H::RwTx: HistoryWrite, +{ /// Append executed blocks to both hot and cold storage. /// /// This method: @@ -301,41 +336,6 @@ where writer.raw_commit().map_err(|e| e.into_hot_kv_error())?; Ok(()) } - - /// Check how far behind cold storage is compared to hot storage. - /// - /// Returns `Some(first_missing_block)` if cold is behind, `None` if synced. - /// - /// # Errors - /// - /// Returns an error if either storage cannot be queried. - pub async fn cold_lag(&self) -> StorageResult> { - let reader = self.reader()?; - let hot_tip = reader.get_chain_tip().map_err(|e| e.into_hot_kv_error())?; - - let cold_tip = self.cold.get_latest_block().await?; - - match (hot_tip, cold_tip) { - (Some((hot_num, _)), Some(cold_num)) if cold_num < hot_num => Ok(Some(cold_num + 1)), - (Some((_, _)), None) => Ok(Some(0)), - _ => Ok(None), - } - } - - /// Replay blocks to cold storage from an external source. - /// - /// Use this to recover cold storage after failures. The caller is - /// responsible for fetching the missing block data. - /// - /// Consumes the blocks to avoid cloning. - /// - /// # Errors - /// - /// Returns an error if cold storage write fails. - pub async fn replay_to_cold(&self, blocks: Vec) -> Result<(), ColdStorageError> { - let cold_data: Vec<_> = blocks.into_iter().map(BlockData::from).collect(); - self.cold.append_blocks(cold_data).await - } } #[cfg(test)] From 867f3b642ae3e12b0b33257e3c848743664048ed Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 07:59:10 -0400 Subject: [PATCH 13/21] refactor(hot): move non-shard-aware history helpers onto new HistoryWrite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bulk helpers (write_plain_revert_sorted, write_state_changes, append_block_inconsistent, etc.) move as default impls onto crate::db::history::HistoryWrite. The legacy shard-aware methods (write_*_history, last_*_history, append_*_history_index, update_history_indices_inconsistent) stay on LegacyUnsafeHistoryWrite for now; they are deleted in a subsequent commit. Removes the temporary LegacyUnsafeHistoryWrite supertrait on HistoryWrite introduced in the previous commit — the moved helpers now provide everything the consistent-state default impls need. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/src/conformance/history.rs | 17 +- crates/hot/src/conformance/mod.rs | 7 +- crates/hot/src/conformance/range.rs | 7 +- crates/hot/src/conformance/roundtrip.rs | 12 +- crates/hot/src/db/history.rs | 326 +++++++++++++++++++++- crates/hot/src/db/inconsistent.rs | 357 ++---------------------- crates/hot/src/model/revm.rs | 2 +- 7 files changed, 383 insertions(+), 345 deletions(-) diff --git a/crates/hot/src/conformance/history.rs b/crates/hot/src/conformance/history.rs index b73331c..750440d 100644 --- a/crates/hot/src/conformance/history.rs +++ b/crates/hot/src/conformance/history.rs @@ -1,7 +1,7 @@ //! History and change set tests for hot storage. use crate::{ - db::{LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, + db::{HistoryWrite, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, model::{HotKv, HotKvWrite}, tables, }; @@ -14,7 +14,10 @@ use signet_storage_types::{Account, BlockNumberList, ShardedKey}; /// 1. Account change sets are correctly indexed into account history /// 2. Appending to existing history works correctly /// 3. Old shards are deleted when appending -pub fn test_update_history_indices_account(hot_kv: &T) { +pub fn test_update_history_indices_account(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr1 = address!("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); let addr2 = address!("0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); @@ -109,7 +112,10 @@ pub fn test_update_history_indices_account(hot_kv: &T) { /// 2. Appending to existing history works correctly /// 3. Old shards are deleted when appending /// 4. Different slots for the same address are tracked separately -pub fn test_update_history_indices_storage(hot_kv: &T) { +pub fn test_update_history_indices_storage(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr1 = address!("0xcccccccccccccccccccccccccccccccccccccccc"); let slot1 = U256::from(1); let slot2 = U256::from(2); @@ -205,7 +211,10 @@ pub fn test_update_history_indices_storage(hot_kv: &T) { /// /// This test specifically verifies that when we append new indices to an existing /// shard, the old shard is properly deleted so we don't end up with duplicate data. -pub fn test_history_append_removes_old_entries(hot_kv: &T) { +pub fn test_history_append_removes_old_entries(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr = address!("0xdddddddddddddddddddddddddddddddddddddddd"); // Phase 1: Manually write account history diff --git a/crates/hot/src/conformance/mod.rs b/crates/hot/src/conformance/mod.rs index 2639865..96e609d 100644 --- a/crates/hot/src/conformance/mod.rs +++ b/crates/hot/src/conformance/mod.rs @@ -12,14 +12,17 @@ pub use range::*; pub use roundtrip::*; pub use unwind::*; -use crate::model::HotKv; +use crate::{db::HistoryWrite, model::HotKv}; /// Run all conformance tests against a [`HotKv`] implementation. /// /// Tests share the provided store instance. Additional test functions /// (cursor, edge-case, history, range) are exported for use in isolation /// with a fresh store. -pub fn conformance(hot_kv: &T) { +pub fn conformance(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ test_header_roundtrip(hot_kv); test_account_roundtrip(hot_kv); test_storage_roundtrip(hot_kv); diff --git a/crates/hot/src/conformance/range.rs b/crates/hot/src/conformance/range.rs index 7fea95a..a3c1c08 100644 --- a/crates/hot/src/conformance/range.rs +++ b/crates/hot/src/conformance/range.rs @@ -1,7 +1,7 @@ //! Clear/take range operations for single and dual-keyed tables. use crate::{ - db::{HotDbRead, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, + db::{HistoryWrite, HotDbRead, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, model::{HotKv, HotKvWrite}, tables, }; @@ -417,7 +417,10 @@ pub fn test_take_range_dual(hot_kv: &T) { /// 1. Multiple storage slots can be written for an address /// 2. `write_changed_storage` with `wipe_storage: true` clears all slots /// 3. After wipe, all slots return None -pub fn test_write_changed_storage_wipe(hot_kv: &T) { +pub fn test_write_changed_storage_wipe(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr = address!("0x1111111111111111111111111111111111111111"); // Setup: write multiple storage slots for an address diff --git a/crates/hot/src/conformance/roundtrip.rs b/crates/hot/src/conformance/roundtrip.rs index 3e25d24..96dca2d 100644 --- a/crates/hot/src/conformance/roundtrip.rs +++ b/crates/hot/src/conformance/roundtrip.rs @@ -1,7 +1,7 @@ //! Basic CRUD roundtrip tests for hot storage. use crate::{ - db::{HotDbRead, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, + db::{HistoryWrite, HotDbRead, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, model::{HotKv, HotKvRead}, tables, }; @@ -203,7 +203,10 @@ pub fn test_storage_history(hot_kv: &T) { } /// Test account change sets via HotHistoryWrite/HotHistoryRead -pub fn test_account_changes(hot_kv: &T) { +pub fn test_account_changes(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr = address!("0x3333333333333333333333333333333333333333"); let pre_state = Account { nonce: 10, balance: U256::from(5000), bytecode_hash: None }; let block_number = 100u64; @@ -229,7 +232,10 @@ pub fn test_account_changes(hot_kv: &T) { } /// Test storage change sets via HotHistoryWrite/HotHistoryRead -pub fn test_storage_changes(hot_kv: &T) { +pub fn test_storage_changes(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr = address!("0x4444444444444444444444444444444444444444"); let slot = U256::from(153); let pre_value = U256::from(12345); diff --git a/crates/hot/src/db/history.rs b/crates/hot/src/db/history.rs index f4e6618..d97ea3d 100644 --- a/crates/hot/src/db/history.rs +++ b/crates/hot/src/db/history.rs @@ -8,7 +8,7 @@ //! MemKv writes a single dup entry per addr). use crate::{ - db::{HistoryError, HotDbRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, + db::{HistoryError, HotDbRead, LegacyHistoryRead, UnsafeDbWrite}, model::HotKvRead, tables, }; @@ -21,7 +21,13 @@ use alloy::{ use itertools::Itertools; use signet_storage_types::{Account, BlockNumberList, EthereumHardfork, SealedHeader, ShardedKey}; use std::ops::RangeInclusive; -use trevm::revm::{database::BundleState, state::Bytecode}; +use trevm::revm::{ + database::{ + BundleState, OriginalValuesKnown, + states::{PlainStateReverts, PlainStorageChangeset, PlainStorageRevert, StateChangeset}, + }, + state::{AccountInfo, Bytecode}, +}; /// Maximum address value (all bits set to 1). const ADDRESS_MAX: Address = address!("0xffffffffffffffffffffffffffffffffffffffff"); @@ -184,7 +190,7 @@ impl HistoryRead for T where T: HotKvRead {} /// Backends that implement this trait choose their own shard-splitting policy. /// The default `update_history_indices` bulk operation is expressed in terms of /// the four required primitives and works for any backend. -pub trait HistoryWrite: UnsafeDbWrite + LegacyUnsafeHistoryWrite + HistoryRead { +pub trait HistoryWrite: UnsafeDbWrite + HistoryRead { /// Merge `new_blocks` into `addr`'s account history. /// /// Preconditions: `new_blocks` is sorted ascending and every entry is @@ -577,4 +583,318 @@ pub trait HistoryWrite: UnsafeDbWrite + LegacyUnsafeHistoryWrite + HistoryRead { Ok(()) }) } + + /// Write an account change (pre-state) for an account at a specific block. + fn write_account_prestate( + &self, + block_number: u64, + address: Address, + pre_state: &Account, + ) -> Result<(), Self::Error> { + self.queue_put_dual::(&block_number, &address, pre_state) + } + + /// Append an account prestate entry. + /// + /// Entries must be appended in sorted order by (block_number, address). + /// Within a single block, addresses must be sorted. + fn append_account_prestate( + &self, + block_number: u64, + address: Address, + pre_state: &Account, + ) -> Result<(), Self::Error> { + self.queue_append_dual::(&block_number, &address, pre_state) + } + + /// Write a storage change (before state) for an account at a specific block. + fn write_storage_prestate( + &self, + block_number: u64, + address: Address, + slot: &U256, + prestate: &U256, + ) -> Result<(), Self::Error> { + self.queue_put_dual::(&(block_number, address), slot, prestate) + } + + /// Append a storage prestate entry. + /// + /// Entries must be appended in sorted order by ((block_number, address), slot). + /// Within a single (block, address), slots must be sorted. + fn append_storage_prestate( + &self, + block_number: u64, + address: Address, + slot: &U256, + prestate: &U256, + ) -> Result<(), Self::Error> { + self.queue_append_dual::( + &(block_number, address), + slot, + prestate, + ) + } + + /// Write a pre-state for every storage key that exists for an account at a + /// specific block. + /// + /// Note: This uses `write_storage_prestate` (regular put) instead of + /// `append_storage_prestate` because the slots may interleave with other + /// writes to the same K1 from different code paths. + fn write_wipe(&self, block_number: u64, address: &Address) -> Result<(), Self::Error> { + let mut cursor = self.traverse_dual::()?; + + for entry in cursor.iter_k2(address)? { + let (slot, value) = entry?; + self.write_storage_prestate(block_number, *address, &slot, &value)?; + } + Ok(()) + } + + /// Write pre-sorted revert data for a single block. + /// + /// # Panics (debug builds only) + /// + /// Panics if `accounts` is not sorted by address or `storage` is not sorted + /// by address. + fn write_plain_revert_sorted( + &self, + block_number: u64, + accounts: &[&(Address, Option)], + storage: &[&PlainStorageRevert], + ) -> Result<(), Self::Error> { + #[cfg(debug_assertions)] + { + debug_assert!( + accounts.windows(2).all(|w| w[0].0 <= w[1].0), + "accounts must be sorted by address" + ); + debug_assert!( + storage.windows(2).all(|w| w[0].address <= w[1].address), + "storage must be sorted by address" + ); + } + + for (address, info) in accounts { + let account = info.as_ref().map(Account::from).unwrap_or_default(); + + // bytecode_hash is None when code_hash == KECCAK256_EMPTY, + // which doesn't need to be stored. + if let Some((bytecode, code_hash)) = + info.as_ref().and_then(|info| info.code.clone()).zip(account.bytecode_hash) + { + self.put_bytecode(&code_hash, &bytecode)?; + } + + self.append_account_prestate(block_number, *address, &account)?; + } + + for entry in storage { + if entry.wiped { + self.write_wipe(block_number, &entry.address)?; + continue; + } + // Use write (put) instead of append because storage_revert slots + // are not guaranteed to be sorted. + for (key, old_value) in entry.storage_revert.iter() { + self.write_storage_prestate( + block_number, + entry.address, + key, + &old_value.to_previous_value(), + )?; + } + } + + Ok(()) + } + + /// Write multiple blocks' plain state revert information. + /// + /// Sorts accounts and storage in parallel before writing to enable + /// efficient append operations. + fn write_plain_reverts( + &self, + first_block_number: u64, + PlainStateReverts { accounts, storage }: &PlainStateReverts, + ) -> Result<(), Self::Error> { + use rayon::prelude::*; + + // Sort accounts and storage in parallel using rayon::join + let (sorted_accounts, sorted_storage) = rayon::join( + || { + accounts + .par_iter() + .map(|block_accounts| { + let mut sorted: Vec<_> = block_accounts.iter().collect(); + sorted.sort_by_key(|(addr, _)| *addr); + sorted + }) + .collect::>() + }, + || { + storage + .par_iter() + .map(|block_storage| { + let mut sorted: Vec<_> = block_storage.iter().collect(); + sorted.sort_by_key(|entry| entry.address); + sorted + }) + .collect::>() + }, + ); + + // Write sequentially (DB writes must be ordered) + sorted_accounts.iter().zip(sorted_storage.iter()).enumerate().try_for_each( + |(idx, (acc, sto))| { + self.write_plain_revert_sorted(first_block_number + idx as u64, acc, sto) + }, + ) + } + + /// Write changed accounts from a [`StateChangeset`]. + fn write_changed_account( + &self, + address: &Address, + account: &Option, + ) -> Result<(), Self::Error> { + let Some(info) = account.as_ref() else { + // Account removal + return self.queue_delete::(address); + }; + + let account = Account::from(info.clone()); + // bytecode_hash is None when code_hash == KECCAK256_EMPTY, + // which doesn't need to be stored. + if let Some((bytecode, code_hash)) = info.code.clone().zip(account.bytecode_hash) { + self.put_bytecode(&code_hash, &bytecode)?; + } + self.put_account(address, &account) + } + + /// Write changed storage from a [`StateChangeset`]. + fn write_changed_storage( + &self, + PlainStorageChangeset { address, wipe_storage, storage }: &PlainStorageChangeset, + ) -> Result<(), Self::Error> { + if *wipe_storage { + return self.clear_k1_for::(address); + } + + storage.iter().try_for_each(|(key, value)| self.put_storage(address, key, value)) + } + + /// Write changed contract bytecode from a [`StateChangeset`]. + fn write_changed_contracts( + &self, + code_hash: &B256, + bytecode: &Bytecode, + ) -> Result<(), Self::Error> { + self.put_bytecode(code_hash, bytecode) + } + + /// Write a state changeset for a specific block. + fn write_state_changes( + &self, + StateChangeset { accounts, storage, contracts }: &StateChangeset, + ) -> Result<(), Self::Error> { + contracts.iter().try_for_each(|(code_hash, bytecode)| { + self.write_changed_contracts(code_hash, bytecode) + })?; + accounts + .iter() + .try_for_each(|(address, account)| self.write_changed_account(address, account))?; + storage + .iter() + .try_for_each(|storage_changeset| self.write_changed_storage(storage_changeset))?; + Ok(()) + } + + /// Get all changed accounts with the list of block numbers in the given + /// range. + /// + /// Iterates over entries starting from the first block in the range, + /// collecting changes while the block number remains in range. + // TODO: estimate capacity from block range size for better allocation + fn changed_accounts_with_range( + &self, + range: RangeInclusive, + ) -> Result>, Self::Error> { + self.traverse_dual::()? + .iter_from(range.start(), &Address::ZERO)? + .process_results(|iter| { + iter.take_while(|(num, _, _)| range.contains(num)) + .map(|(num, addr, _)| (addr, num)) + .into_group_map_by(|(addr, _)| *addr) + .into_iter() + .map(|(addr, pairs)| (addr, pairs.into_iter().map(|(_, num)| num).collect())) + .collect() + }) + } + + /// Get all changed storages with the list of block numbers in the given + /// range. + /// + /// Iterates over entries starting from the first block in the range, + /// collecting changes while the block number remains in range. + // TODO: estimate capacity from block range size for better allocation + #[allow(clippy::type_complexity)] + fn changed_storages_with_range( + &self, + range: RangeInclusive, + ) -> Result>, Self::Error> { + self.traverse_dual::()? + .iter_from(&(*range.start(), Address::ZERO), &U256::ZERO)? + .process_results(|iter| { + iter.take_while(|(num_addr, _, _)| range.contains(&num_addr.0)) + .map(|(num_addr, slot, _)| ((num_addr.1, slot), num_addr.0)) + .into_group_map_by(|(key, _)| *key) + .into_iter() + .map(|(key, pairs)| (key, pairs.into_iter().map(|(_, num)| num).collect())) + .collect() + }) + } + + /// Append a block's header and state changes in an inconsistent manner. + /// + /// This may leave the database in an inconsistent state. Users should + /// prefer higher-level abstractions when possible. + /// + /// 1. It MUST be checked that the header is the child of the current chain + /// tip before calling this method. + /// 2. After calling this method, the caller MUST call + /// `update_history_indices`. + fn append_block_inconsistent( + &self, + header: &SealedHeader, + state_changes: &BundleState, + ) -> Result<(), Self::Error> { + self.append_header(header)?; + self.put_header_number_inconsistent(&header.hash(), header.number)?; + + let (state_changes, reverts) = + state_changes.to_plain_state_and_reverts(OriginalValuesKnown::No); + + self.write_state_changes(&state_changes)?; + self.write_plain_reverts(header.number, &reverts) + } + + /// Append multiple blocks' headers and state changes in an inconsistent + /// manner. + /// + /// This may leave the database in an inconsistent state. Users should + /// prefer higher-level abstractions when possible. + /// 1. It MUST be checked that the first header is the child of the current + /// chain tip before calling this method. + /// 2. After calling this method, the caller MUST call + /// `update_history_indices`. + fn append_blocks_inconsistent<'a>( + &self, + blocks: impl IntoIterator, + ) -> Result<(), Self::Error> { + blocks + .into_iter() + .try_for_each(|(header, state)| self.append_block_inconsistent(header, state)) + } } diff --git a/crates/hot/src/db/inconsistent.rs b/crates/hot/src/db/inconsistent.rs index 65f5267..47e8dfd 100644 --- a/crates/hot/src/db/inconsistent.rs +++ b/crates/hot/src/db/inconsistent.rs @@ -8,14 +8,7 @@ use alloy::primitives::{Address, B256, BlockNumber, U256}; use itertools::Itertools; use signet_storage_types::{Account, BlockNumberList, SealedHeader, ShardedKey}; use std::ops::RangeInclusive; -use trevm::revm::{ - bytecode::Bytecode, - database::{ - BundleState, OriginalValuesKnown, - states::{PlainStateReverts, PlainStorageChangeset, PlainStorageRevert, StateChangeset}, - }, - state::AccountInfo, -}; +use trevm::revm::bytecode::Bytecode; /// Bundle state initialization type. /// Maps address -> (old_account, new_account, storage_changes) @@ -131,29 +124,6 @@ pub trait LegacyUnsafeHistoryWrite: UnsafeDbWrite + LegacyHistoryRead { self.queue_put_dual::(address, &latest_height, touched) } - /// Write an account change (pre-state) for an account at a specific block. - fn write_account_prestate( - &self, - block_number: u64, - address: Address, - pre_state: &Account, - ) -> Result<(), Self::Error> { - self.queue_put_dual::(&block_number, &address, pre_state) - } - - /// Append an account prestate entry. - /// - /// Entries must be appended in sorted order by (block_number, address). - /// Within a single block, addresses must be sorted. - fn append_account_prestate( - &self, - block_number: u64, - address: Address, - pre_state: &Account, - ) -> Result<(), Self::Error> { - self.queue_append_dual::(&block_number, &address, pre_state) - } - /// Write storage history, by highest block number and touched block /// numbers. fn write_storage_history( @@ -167,232 +137,6 @@ pub trait LegacyUnsafeHistoryWrite: UnsafeDbWrite + LegacyHistoryRead { self.queue_put_dual::(address, &sharded_key, touched) } - /// Write a storage change (before state) for an account at a specific block. - fn write_storage_prestate( - &self, - block_number: u64, - address: Address, - slot: &U256, - prestate: &U256, - ) -> Result<(), Self::Error> { - self.queue_put_dual::(&(block_number, address), slot, prestate) - } - - /// Append a storage prestate entry. - /// - /// Entries must be appended in sorted order by ((block_number, address), slot). - /// Within a single (block, address), slots must be sorted. - fn append_storage_prestate( - &self, - block_number: u64, - address: Address, - slot: &U256, - prestate: &U256, - ) -> Result<(), Self::Error> { - self.queue_append_dual::( - &(block_number, address), - slot, - prestate, - ) - } - - /// Write a pre-state for every storage key that exists for an account at a - /// specific block. - /// - /// Note: This uses `write_storage_prestate` (regular put) instead of - /// `append_storage_prestate` because the slots may interleave with other - /// writes to the same K1 from different code paths. - fn write_wipe(&self, block_number: u64, address: &Address) -> Result<(), Self::Error> { - let mut cursor = self.traverse_dual::()?; - - for entry in cursor.iter_k2(address)? { - let (slot, value) = entry?; - self.write_storage_prestate(block_number, *address, &slot, &value)?; - } - Ok(()) - } - - /// Write pre-sorted revert data for a single block. - /// - /// # Panics (debug builds only) - /// - /// Panics if `accounts` is not sorted by address or `storage` is not sorted - /// by address. - fn write_plain_revert_sorted( - &self, - block_number: u64, - accounts: &[&(Address, Option)], - storage: &[&PlainStorageRevert], - ) -> Result<(), Self::Error> { - #[cfg(debug_assertions)] - { - debug_assert!( - accounts.windows(2).all(|w| w[0].0 <= w[1].0), - "accounts must be sorted by address" - ); - debug_assert!( - storage.windows(2).all(|w| w[0].address <= w[1].address), - "storage must be sorted by address" - ); - } - - for (address, info) in accounts { - let account = info.as_ref().map(Account::from).unwrap_or_default(); - - // bytecode_hash is None when code_hash == KECCAK256_EMPTY, - // which doesn't need to be stored. - if let Some((bytecode, code_hash)) = - info.as_ref().and_then(|info| info.code.clone()).zip(account.bytecode_hash) - { - self.put_bytecode(&code_hash, &bytecode)?; - } - - self.append_account_prestate(block_number, *address, &account)?; - } - - for entry in storage { - if entry.wiped { - self.write_wipe(block_number, &entry.address)?; - continue; - } - // Use write (put) instead of append because storage_revert slots - // are not guaranteed to be sorted. - for (key, old_value) in entry.storage_revert.iter() { - self.write_storage_prestate( - block_number, - entry.address, - key, - &old_value.to_previous_value(), - )?; - } - } - - Ok(()) - } - - /// Write multiple blocks' plain state revert information. - /// - /// Sorts accounts and storage in parallel before writing to enable - /// efficient append operations. - fn write_plain_reverts( - &self, - first_block_number: u64, - PlainStateReverts { accounts, storage }: &PlainStateReverts, - ) -> Result<(), Self::Error> { - use rayon::prelude::*; - - // Sort accounts and storage in parallel using rayon::join - let (sorted_accounts, sorted_storage) = rayon::join( - || { - accounts - .par_iter() - .map(|block_accounts| { - let mut sorted: Vec<_> = block_accounts.iter().collect(); - sorted.sort_by_key(|(addr, _)| *addr); - sorted - }) - .collect::>() - }, - || { - storage - .par_iter() - .map(|block_storage| { - let mut sorted: Vec<_> = block_storage.iter().collect(); - sorted.sort_by_key(|entry| entry.address); - sorted - }) - .collect::>() - }, - ); - - // Write sequentially (DB writes must be ordered) - sorted_accounts.iter().zip(sorted_storage.iter()).enumerate().try_for_each( - |(idx, (acc, sto))| { - self.write_plain_revert_sorted(first_block_number + idx as u64, acc, sto) - }, - ) - } - - /// Write changed accounts from a [`StateChangeset`]. - fn write_changed_account( - &self, - address: &Address, - account: &Option, - ) -> Result<(), Self::Error> { - let Some(info) = account.as_ref() else { - // Account removal - return self.queue_delete::(address); - }; - - let account = Account::from(info.clone()); - // bytecode_hash is None when code_hash == KECCAK256_EMPTY, - // which doesn't need to be stored. - if let Some((bytecode, code_hash)) = info.code.clone().zip(account.bytecode_hash) { - self.put_bytecode(&code_hash, &bytecode)?; - } - self.put_account(address, &account) - } - - /// Write changed storage from a [`StateChangeset`]. - fn write_changed_storage( - &self, - PlainStorageChangeset { address, wipe_storage, storage }: &PlainStorageChangeset, - ) -> Result<(), Self::Error> { - if *wipe_storage { - return self.clear_k1_for::(address); - } - - storage.iter().try_for_each(|(key, value)| self.put_storage(address, key, value)) - } - - /// Write changed contract bytecode from a [`StateChangeset`]. - fn write_changed_contracts( - &self, - code_hash: &B256, - bytecode: &Bytecode, - ) -> Result<(), Self::Error> { - self.put_bytecode(code_hash, bytecode) - } - - /// Write a state changeset for a specific block. - fn write_state_changes( - &self, - StateChangeset { accounts, storage, contracts }: &StateChangeset, - ) -> Result<(), Self::Error> { - contracts.iter().try_for_each(|(code_hash, bytecode)| { - self.write_changed_contracts(code_hash, bytecode) - })?; - accounts - .iter() - .try_for_each(|(address, account)| self.write_changed_account(address, account))?; - storage - .iter() - .try_for_each(|storage_changeset| self.write_changed_storage(storage_changeset))?; - Ok(()) - } - - /// Get all changed accounts with the list of block numbers in the given - /// range. - /// - /// Iterates over entries starting from the first block in the range, - /// collecting changes while the block number remains in range. - // TODO: estimate capacity from block range size for better allocation - fn changed_accounts_with_range( - &self, - range: RangeInclusive, - ) -> Result>, Self::Error> { - self.traverse_dual::()? - .iter_from(range.start(), &Address::ZERO)? - .process_results(|iter| { - iter.take_while(|(num, _, _)| range.contains(num)) - .map(|(num, addr, _)| (addr, num)) - .into_group_map_by(|(addr, _)| *addr) - .into_iter() - .map(|(addr, pairs)| (addr, pairs.into_iter().map(|(_, num)| num).collect())) - .collect() - }) - } - /// Append account history indices for multiple accounts. fn append_account_history_index( &self, @@ -410,29 +154,6 @@ pub trait LegacyUnsafeHistoryWrite: UnsafeDbWrite + LegacyHistoryRead { Ok(()) } - /// Get all changed storages with the list of block numbers in the given - /// range. - /// - /// Iterates over entries starting from the first block in the range, - /// collecting changes while the block number remains in range. - // TODO: estimate capacity from block range size for better allocation - #[allow(clippy::type_complexity)] - fn changed_storages_with_range( - &self, - range: RangeInclusive, - ) -> Result>, Self::Error> { - self.traverse_dual::()? - .iter_from(&(*range.start(), Address::ZERO), &U256::ZERO)? - .process_results(|iter| { - iter.take_while(|(num_addr, _, _)| range.contains(&num_addr.0)) - .map(|(num_addr, slot, _)| ((num_addr.1, slot), num_addr.0)) - .into_group_map_by(|(key, _)| *key) - .into_iter() - .map(|(key, pairs)| (key, pairs.into_iter().map(|(_, num)| num).collect())) - .collect() - }) - } - /// Append storage history indices for multiple (address, slot) pairs. fn append_storage_history_index( &self, @@ -457,61 +178,37 @@ pub trait LegacyUnsafeHistoryWrite: UnsafeDbWrite + LegacyHistoryRead { range: RangeInclusive, ) -> Result<(), HistoryError> { // account history stage - { - let indices = self.changed_accounts_with_range(range.clone())?; - self.append_account_history_index(indices)?; - } + // TODO: estimate capacity from block range size for better allocation + let account_indices: AHashMap> = self + .traverse_dual::()? + .iter_from(range.start(), &Address::ZERO)? + .process_results(|iter| { + iter.take_while(|(num, _, _)| range.contains(num)) + .map(|(num, addr, _)| (addr, num)) + .into_group_map_by(|(addr, _)| *addr) + .into_iter() + .map(|(addr, pairs)| (addr, pairs.into_iter().map(|(_, num)| num).collect())) + .collect() + })?; + self.append_account_history_index(account_indices)?; // storage history stage - { - let indices = self.changed_storages_with_range(range)?; - self.append_storage_history_index(indices)?; - } + // TODO: estimate capacity from block range size for better allocation + let storage_indices: AHashMap<(Address, U256), Vec> = self + .traverse_dual::()? + .iter_from(&(*range.start(), Address::ZERO), &U256::ZERO)? + .process_results(|iter| { + iter.take_while(|(num_addr, _, _)| range.contains(&num_addr.0)) + .map(|(num_addr, slot, _)| ((num_addr.1, slot), num_addr.0)) + .into_group_map_by(|(key, _)| *key) + .into_iter() + .map(|(key, pairs)| (key, pairs.into_iter().map(|(_, num)| num).collect())) + .collect() + })?; + self.append_storage_history_index(storage_indices)?; Ok(()) } - - /// Append a block's header and state changes in an inconsistent manner. - /// - /// This may leave the database in an inconsistent state. Users should - /// prefer higher-level abstractions when possible. - /// - /// 1. It MUST be checked that the header is the child of the current chain - /// tip before calling this method. - /// 2. After calling this method, the caller MUST call - /// `update_history_indices`. - fn append_block_inconsistent( - &self, - header: &SealedHeader, - state_changes: &BundleState, - ) -> Result<(), Self::Error> { - self.append_header(header)?; - self.put_header_number_inconsistent(&header.hash(), header.number)?; - - let (state_changes, reverts) = - state_changes.to_plain_state_and_reverts(OriginalValuesKnown::No); - - self.write_state_changes(&state_changes)?; - self.write_plain_reverts(header.number, &reverts) - } - - /// Append multiple blocks' headers and state changes in an inconsistent - /// manner. - /// - /// This may leave the database in an inconsistent state. Users should - /// prefer higher-level abstractions when possible. - /// 1. It MUST be checked that the first header is the child of the current - /// chain tip before calling this method. - /// 2. After calling this method, the caller MUST call - /// `update_history_indices`. - fn append_blocks_inconsistent<'a>( - &self, - blocks: impl IntoIterator, - ) -> Result<(), Self::Error> { - blocks - .into_iter() - .try_for_each(|(header, state)| self.append_block_inconsistent(header, state)) - } } impl LegacyUnsafeHistoryWrite for T where T: UnsafeDbWrite + HotKvWrite {} diff --git a/crates/hot/src/model/revm.rs b/crates/hot/src/model/revm.rs index 1c0d242..3977677 100644 --- a/crates/hot/src/model/revm.rs +++ b/crates/hot/src/model/revm.rs @@ -445,7 +445,7 @@ where mod tests { use super::*; use crate::{ - db::{HistoryWrite, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, + db::{HistoryWrite, UnsafeDbWrite}, mem::MemKv, model::{HotKv, HotKvRead, HotKvWrite}, tables::{Bytecodes, PlainAccountState}, From 1e5c69b520757866676884defe624d9cdd50b6af Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 08:04:26 -0400 Subject: [PATCH 14/21] test(hot): rewrite history conformance tests in logical terms All shard-shaped assertions (last_account_history returning specific shard keys, write_account_history with explicit subkeys) replaced with logical equivalents using HistoryRead::blocks_changed_account / HistoryWrite::append_account_history / truncate_account_history_above. Shard-structure assertions migrate to an MDBX-only structural test in a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/src/conformance/history.rs | 273 ++++++++++++-------------- 1 file changed, 127 insertions(+), 146 deletions(-) diff --git a/crates/hot/src/conformance/history.rs b/crates/hot/src/conformance/history.rs index 750440d..938cf70 100644 --- a/crates/hot/src/conformance/history.rs +++ b/crates/hot/src/conformance/history.rs @@ -1,19 +1,17 @@ //! History and change set tests for hot storage. use crate::{ - db::{HistoryWrite, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, - model::{HotKv, HotKvWrite}, - tables, + db::{HistoryRead, HistoryWrite, UnsafeDbWrite}, + model::HotKv, }; use alloy::primitives::{U256, address}; -use signet_storage_types::{Account, BlockNumberList, ShardedKey}; +use signet_storage_types::{Account, BlockNumberList}; -/// Test update_history_indices_inconsistent for account history. +/// Test update_history_indices for account history. /// /// This test verifies that: /// 1. Account change sets are correctly indexed into account history /// 2. Appending to existing history works correctly -/// 3. Old shards are deleted when appending pub fn test_update_history_indices_account(hot_kv: &T) where T::RwTx: HistoryWrite, @@ -41,10 +39,10 @@ where writer.commit().unwrap(); } - // Phase 2: Run update_history_indices_inconsistent for blocks 1-3 + // Phase 2: Run update_history_indices for blocks 1-3 { let writer = hot_kv.writer().unwrap(); - writer.update_history_indices_inconsistent(1..=3).unwrap(); + writer.update_history_indices(1..=3).unwrap(); writer.commit().unwrap(); } @@ -53,14 +51,14 @@ where let reader = hot_kv.reader().unwrap(); // addr1 should have history at blocks 1, 2 - let (_, history1) = - reader.last_account_history(addr1).unwrap().expect("addr1 should have history"); + let history1 = + reader.blocks_changed_account(&addr1).unwrap().expect("addr1 should have history"); let blocks1: Vec = history1.iter().collect(); assert_eq!(blocks1, vec![1, 2], "addr1 history mismatch"); // addr2 should have history at blocks 2, 3 - let (_, history2) = - reader.last_account_history(addr2).unwrap().expect("addr2 should have history"); + let history2 = + reader.blocks_changed_account(&addr2).unwrap().expect("addr2 should have history"); let blocks2: Vec = history2.iter().collect(); assert_eq!(blocks2, vec![2, 3], "addr2 history mismatch"); } @@ -80,10 +78,10 @@ where writer.commit().unwrap(); } - // Phase 5: Run update_history_indices_inconsistent for blocks 4-5 + // Phase 5: Run update_history_indices for blocks 4-5 { let writer = hot_kv.writer().unwrap(); - writer.update_history_indices_inconsistent(4..=5).unwrap(); + writer.update_history_indices(4..=5).unwrap(); writer.commit().unwrap(); } @@ -92,26 +90,25 @@ where let reader = hot_kv.reader().unwrap(); // addr1 should now have history at blocks 1, 2, 4, 5 - let (_, history1) = - reader.last_account_history(addr1).unwrap().expect("addr1 should have history"); + let history1 = + reader.blocks_changed_account(&addr1).unwrap().expect("addr1 should have history"); let blocks1: Vec = history1.iter().collect(); assert_eq!(blocks1, vec![1, 2, 4, 5], "addr1 history mismatch after append"); // addr2 should still have history at blocks 2, 3 (unchanged) - let (_, history2) = - reader.last_account_history(addr2).unwrap().expect("addr2 should have history"); + let history2 = + reader.blocks_changed_account(&addr2).unwrap().expect("addr2 should have history"); let blocks2: Vec = history2.iter().collect(); assert_eq!(blocks2, vec![2, 3], "addr2 history should be unchanged"); } } -/// Test update_history_indices_inconsistent for storage history. +/// Test update_history_indices for storage history. /// /// This test verifies that: /// 1. Storage change sets are correctly indexed into storage history /// 2. Appending to existing history works correctly -/// 3. Old shards are deleted when appending -/// 4. Different slots for the same address are tracked separately +/// 3. Different slots for the same address are tracked separately pub fn test_update_history_indices_storage(hot_kv: &T) where T::RwTx: HistoryWrite, @@ -137,10 +134,10 @@ where writer.commit().unwrap(); } - // Phase 2: Run update_history_indices_inconsistent for blocks 1-3 + // Phase 2: Run update_history_indices for blocks 1-3 { let writer = hot_kv.writer().unwrap(); - writer.update_history_indices_inconsistent(1..=3).unwrap(); + writer.update_history_indices(1..=3).unwrap(); writer.commit().unwrap(); } @@ -149,16 +146,16 @@ where let reader = hot_kv.reader().unwrap(); // addr1.slot1 should have history at blocks 1, 2 - let (_, history1) = reader - .last_storage_history(&addr1, &slot1) + let history1 = reader + .blocks_changed_storage(&addr1, &slot1) .unwrap() .expect("addr1.slot1 should have history"); let blocks1: Vec = history1.iter().collect(); assert_eq!(blocks1, vec![1, 2], "addr1.slot1 history mismatch"); // addr1.slot2 should have history at blocks 2, 3 - let (_, history2) = reader - .last_storage_history(&addr1, &slot2) + let history2 = reader + .blocks_changed_storage(&addr1, &slot2) .unwrap() .expect("addr1.slot2 should have history"); let blocks2: Vec = history2.iter().collect(); @@ -178,10 +175,10 @@ where writer.commit().unwrap(); } - // Phase 5: Run update_history_indices_inconsistent for blocks 4-5 + // Phase 5: Run update_history_indices for blocks 4-5 { let writer = hot_kv.writer().unwrap(); - writer.update_history_indices_inconsistent(4..=5).unwrap(); + writer.update_history_indices(4..=5).unwrap(); writer.commit().unwrap(); } @@ -190,16 +187,16 @@ where let reader = hot_kv.reader().unwrap(); // addr1.slot1 should now have history at blocks 1, 2, 4, 5 - let (_, history1) = reader - .last_storage_history(&addr1, &slot1) + let history1 = reader + .blocks_changed_storage(&addr1, &slot1) .unwrap() .expect("addr1.slot1 should have history"); let blocks1: Vec = history1.iter().collect(); assert_eq!(blocks1, vec![1, 2, 4, 5], "addr1.slot1 history mismatch after append"); // addr1.slot2 should still have history at blocks 2, 3 (unchanged) - let (_, history2) = reader - .last_storage_history(&addr1, &slot2) + let history2 = reader + .blocks_changed_storage(&addr1, &slot2) .unwrap() .expect("addr1.slot2 should have history"); let blocks2: Vec = history2.iter().collect(); @@ -207,30 +204,29 @@ where } } -/// Test that appending to history correctly removes old entries at same k1,k2. +/// Test that appending to history correctly merges blocks. /// -/// This test specifically verifies that when we append new indices to an existing -/// shard, the old shard is properly deleted so we don't end up with duplicate data. +/// This test verifies that after appending an initial list and then a new +/// block via `update_history_indices`, `blocks_changed_account` returns the +/// expected union of all blocks. pub fn test_history_append_removes_old_entries(hot_kv: &T) where T::RwTx: HistoryWrite, { let addr = address!("0xdddddddddddddddddddddddddddddddddddddddd"); - // Phase 1: Manually write account history + // Phase 1: Append account history for blocks 10, 20, 30 { let writer = hot_kv.writer().unwrap(); let initial_history = BlockNumberList::new([10, 20, 30]).unwrap(); - writer.write_account_history(&addr, u64::MAX, &initial_history).unwrap(); + writer.append_account_history(&addr, &initial_history).unwrap(); writer.commit().unwrap(); } // Verify initial state { let reader = hot_kv.reader().unwrap(); - let (key, history) = - reader.last_account_history(addr).unwrap().expect("should have history"); - assert_eq!(key, u64::MAX); + let history = reader.blocks_changed_account(&addr).unwrap().expect("should have history"); let blocks: Vec = history.iter().collect(); assert_eq!(blocks, vec![10, 20, 30]); } @@ -243,223 +239,208 @@ where writer.commit().unwrap(); } - // Phase 3: Run update_history_indices_inconsistent + // Phase 3: Run update_history_indices { let writer = hot_kv.writer().unwrap(); - writer.update_history_indices_inconsistent(40..=40).unwrap(); + writer.update_history_indices(40..=40).unwrap(); writer.commit().unwrap(); } - // Phase 4: Verify history was correctly appended + // Phase 4: Verify history was correctly appended — union is [10, 20, 30, 40] { let reader = hot_kv.reader().unwrap(); - let (key, history) = - reader.last_account_history(addr).unwrap().expect("should have history"); - assert_eq!(key, u64::MAX, "key should still be u64::MAX"); + let history = reader.blocks_changed_account(&addr).unwrap().expect("should have history"); let blocks: Vec = history.iter().collect(); assert_eq!(blocks, vec![10, 20, 30, 40], "history should include appended block"); } } -/// Test deleting dual-keyed account history entries. +/// Test that truncating account history removes only blocks above the given +/// height and leaves other addresses intact. /// /// This test verifies that: -/// 1. Writing dual-keyed entries works correctly -/// 2. Deleting specific dual-keyed entries removes only that entry -/// 3. Other entries for the same k1 remain intact -/// 4. Traversal after deletion shows the entry is gone -pub fn test_delete_dual_account_history(hot_kv: &T) { +/// 1. Appending two disjoint sets of blocks for the same address works +/// 2. `truncate_account_history_above` removes blocks above the cutoff +/// 3. Other addresses are not affected +pub fn test_delete_dual_account_history(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr1 = address!("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"); let addr2 = address!("0xffffffffffffffffffffffffffffffffffffffff"); - // Phase 1: Write account history entries for multiple addresses + // Phase 1: Append history for addr1 ([1,2,3] then [4,5,6]) and addr2 ([10,20,30]) { let writer = hot_kv.writer().unwrap(); - // Write history for addr1 at two different shard keys let history1_a = BlockNumberList::new([1, 2, 3]).unwrap(); + writer.append_account_history(&addr1, &history1_a).unwrap(); + let history1_b = BlockNumberList::new([4, 5, 6]).unwrap(); - writer.write_account_history(&addr1, 100, &history1_a).unwrap(); - writer.write_account_history(&addr1, u64::MAX, &history1_b).unwrap(); + writer.append_account_history(&addr1, &history1_b).unwrap(); - // Write history for addr2 let history2 = BlockNumberList::new([10, 20, 30]).unwrap(); - writer.write_account_history(&addr2, u64::MAX, &history2).unwrap(); + writer.append_account_history(&addr2, &history2).unwrap(); writer.commit().unwrap(); } - // Phase 2: Verify all entries exist + // Phase 2: Verify the full logical union for addr1 is [1,2,3,4,5,6] { let reader = hot_kv.reader().unwrap(); - // Check addr1 entries - let hist1_a = reader.get_account_history(&addr1, 100).unwrap(); - assert!(hist1_a.is_some(), "addr1 shard 100 should exist"); - assert_eq!(hist1_a.unwrap().iter().collect::>(), vec![1, 2, 3]); - - let hist1_b = reader.get_account_history(&addr1, u64::MAX).unwrap(); - assert!(hist1_b.is_some(), "addr1 shard u64::MAX should exist"); - assert_eq!(hist1_b.unwrap().iter().collect::>(), vec![4, 5, 6]); + let hist1 = + reader.blocks_changed_account(&addr1).unwrap().expect("addr1 should have history"); + assert_eq!(hist1.iter().collect::>(), vec![1, 2, 3, 4, 5, 6]); - // Check addr2 entry - let hist2 = reader.get_account_history(&addr2, u64::MAX).unwrap(); - assert!(hist2.is_some(), "addr2 should exist"); - assert_eq!(hist2.unwrap().iter().collect::>(), vec![10, 20, 30]); + let hist2 = + reader.blocks_changed_account(&addr2).unwrap().expect("addr2 should have history"); + assert_eq!(hist2.iter().collect::>(), vec![10, 20, 30]); } - // Phase 3: Delete addr1's u64::MAX entry + // Phase 3: Truncate addr1's history above block 3 { let writer = hot_kv.writer().unwrap(); - writer.queue_delete_dual::(&addr1, &u64::MAX).unwrap(); + writer.truncate_account_history_above(&addr1, 3).unwrap(); writer.commit().unwrap(); } - // Phase 4: Verify only the deleted entry is gone + // Phase 4: Verify only blocks <= 3 remain for addr1; addr2 is unaffected { let reader = hot_kv.reader().unwrap(); - // addr1 shard 100 should still exist - let hist1_a = reader.get_account_history(&addr1, 100).unwrap(); - assert!(hist1_a.is_some(), "addr1 shard 100 should still exist after delete"); - assert_eq!(hist1_a.unwrap().iter().collect::>(), vec![1, 2, 3]); - - // addr1 shard u64::MAX should be gone - let hist1_b = reader.get_account_history(&addr1, u64::MAX).unwrap(); - assert!(hist1_b.is_none(), "addr1 shard u64::MAX should be deleted"); - - // addr2 should be unaffected - let hist2 = reader.get_account_history(&addr2, u64::MAX).unwrap(); - assert!(hist2.is_some(), "addr2 should be unaffected by delete"); - assert_eq!(hist2.unwrap().iter().collect::>(), vec![10, 20, 30]); + let hist1 = reader + .blocks_changed_account(&addr1) + .unwrap() + .expect("addr1 should still have history after truncation"); + assert_eq!(hist1.iter().collect::>(), vec![1, 2, 3]); - // Verify last_account_history now returns shard 100 for addr1 - let (key, _) = - reader.last_account_history(addr1).unwrap().expect("addr1 should still have history"); - assert_eq!(key, 100, "last shard for addr1 should now be 100"); + let hist2 = + reader.blocks_changed_account(&addr2).unwrap().expect("addr2 should be unaffected"); + assert_eq!(hist2.iter().collect::>(), vec![10, 20, 30]); } } -/// Test deleting dual-keyed storage history entries. +/// Test that truncating storage history removes only the targeted slot's +/// blocks and leaves other slots intact. /// /// This test verifies that: -/// 1. Writing storage history entries works correctly -/// 2. Deleting specific (address, slot, shard) entries removes only that entry -/// 3. Other slots for the same address remain intact -/// 4. Traversal after deletion shows the entry is gone -pub fn test_delete_dual_storage_history(hot_kv: &T) { +/// 1. Appending storage history for two slots works correctly +/// 2. `truncate_storage_history_above(addr, slot1, 0)` removes all blocks for slot1 +/// 3. Other slots for the same address are not affected +pub fn test_delete_dual_storage_history(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr = address!("0x1111111111111111111111111111111111111111"); let slot1 = U256::from(100); let slot2 = U256::from(200); - // Phase 1: Write storage history entries for multiple slots + // Phase 1: Append storage history for both slots { let writer = hot_kv.writer().unwrap(); - // Write history for slot1 let history1 = BlockNumberList::new([1, 2, 3]).unwrap(); - writer.write_storage_history(&addr, slot1, u64::MAX, &history1).unwrap(); + writer.append_storage_history(&addr, &slot1, &history1).unwrap(); - // Write history for slot2 let history2 = BlockNumberList::new([10, 20, 30]).unwrap(); - writer.write_storage_history(&addr, slot2, u64::MAX, &history2).unwrap(); + writer.append_storage_history(&addr, &slot2, &history2).unwrap(); writer.commit().unwrap(); } - // Phase 2: Verify both entries exist + // Phase 2: Verify both slots have history { let reader = hot_kv.reader().unwrap(); - let hist1 = reader.get_storage_history(&addr, slot1, u64::MAX).unwrap(); - assert!(hist1.is_some(), "slot1 should exist"); - assert_eq!(hist1.unwrap().iter().collect::>(), vec![1, 2, 3]); + let hist1 = reader + .blocks_changed_storage(&addr, &slot1) + .unwrap() + .expect("slot1 should have history"); + assert_eq!(hist1.iter().collect::>(), vec![1, 2, 3]); - let hist2 = reader.get_storage_history(&addr, slot2, u64::MAX).unwrap(); - assert!(hist2.is_some(), "slot2 should exist"); - assert_eq!(hist2.unwrap().iter().collect::>(), vec![10, 20, 30]); + let hist2 = reader + .blocks_changed_storage(&addr, &slot2) + .unwrap() + .expect("slot2 should have history"); + assert_eq!(hist2.iter().collect::>(), vec![10, 20, 30]); } - // Phase 3: Delete slot1's entry + // Phase 3: Remove all blocks for slot1 by truncating above 0 + // (all test blocks are > 0, so nothing is kept) { let writer = hot_kv.writer().unwrap(); - let key_to_delete = ShardedKey::new(slot1, u64::MAX); - writer.queue_delete_dual::(&addr, &key_to_delete).unwrap(); + writer.truncate_storage_history_above(&addr, &slot1, 0).unwrap(); writer.commit().unwrap(); } - // Phase 4: Verify only slot1 is gone + // Phase 4: Verify slot1 is gone; slot2 is unaffected { let reader = hot_kv.reader().unwrap(); - // slot1 should be gone - let hist1 = reader.get_storage_history(&addr, slot1, u64::MAX).unwrap(); - assert!(hist1.is_none(), "slot1 should be deleted"); - - // slot2 should be unaffected - let hist2 = reader.get_storage_history(&addr, slot2, u64::MAX).unwrap(); - assert!(hist2.is_some(), "slot2 should be unaffected"); - assert_eq!(hist2.unwrap().iter().collect::>(), vec![10, 20, 30]); - - // last_storage_history for slot1 should return None - let last1 = reader.last_storage_history(&addr, &slot1).unwrap(); - assert!(last1.is_none(), "last_storage_history for slot1 should return None"); + let hist1 = reader.blocks_changed_storage(&addr, &slot1).unwrap(); + assert!(hist1.is_none(), "slot1 history should be gone after truncation"); - // last_storage_history for slot2 should still work - let last2 = reader.last_storage_history(&addr, &slot2).unwrap(); - assert!(last2.is_some(), "last_storage_history for slot2 should still work"); + let hist2 = reader + .blocks_changed_storage(&addr, &slot2) + .unwrap() + .expect("slot2 should be unaffected"); + assert_eq!(hist2.iter().collect::>(), vec![10, 20, 30]); } } -/// Test deleting and re-adding dual-keyed entries. +/// Test deleting and re-adding account history entries. /// -/// This test verifies that after deleting an entry, we can write a new entry -/// with the same key and it works correctly. -pub fn test_delete_and_rewrite_dual(hot_kv: &T) { +/// This test verifies that after truncating all history for an address, we +/// can append new blocks and read them back correctly. +pub fn test_delete_and_rewrite_dual(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr = address!("0x2222222222222222222222222222222222222222"); - // Phase 1: Write initial entry + // Phase 1: Append initial history [1, 2, 3] { let writer = hot_kv.writer().unwrap(); let history = BlockNumberList::new([1, 2, 3]).unwrap(); - writer.write_account_history(&addr, u64::MAX, &history).unwrap(); + writer.append_account_history(&addr, &history).unwrap(); writer.commit().unwrap(); } // Verify initial state { let reader = hot_kv.reader().unwrap(); - let hist = reader.get_account_history(&addr, u64::MAX).unwrap(); - assert_eq!(hist.unwrap().iter().collect::>(), vec![1, 2, 3]); + let hist = reader.blocks_changed_account(&addr).unwrap().expect("should have history"); + assert_eq!(hist.iter().collect::>(), vec![1, 2, 3]); } - // Phase 2: Delete the entry + // Phase 2: Remove all history by truncating above 0 + // (all test blocks are > 0, so nothing is kept) { let writer = hot_kv.writer().unwrap(); - writer.queue_delete_dual::(&addr, &u64::MAX).unwrap(); + writer.truncate_account_history_above(&addr, 0).unwrap(); writer.commit().unwrap(); } // Verify deleted { let reader = hot_kv.reader().unwrap(); - let hist = reader.get_account_history(&addr, u64::MAX).unwrap(); - assert!(hist.is_none(), "entry should be deleted"); + let hist = reader.blocks_changed_account(&addr).unwrap(); + assert!(hist.is_none(), "history should be empty after truncation"); } - // Phase 3: Write new entry with same key but different value + // Phase 3: Append new history [100, 200, 300] { let writer = hot_kv.writer().unwrap(); let new_history = BlockNumberList::new([100, 200, 300]).unwrap(); - writer.write_account_history(&addr, u64::MAX, &new_history).unwrap(); + writer.append_account_history(&addr, &new_history).unwrap(); writer.commit().unwrap(); } // Verify new value { let reader = hot_kv.reader().unwrap(); - let hist = reader.get_account_history(&addr, u64::MAX).unwrap(); - assert!(hist.is_some(), "new entry should exist"); - assert_eq!(hist.unwrap().iter().collect::>(), vec![100, 200, 300]); + let hist = reader.blocks_changed_account(&addr).unwrap().expect("new history should exist"); + assert_eq!(hist.iter().collect::>(), vec![100, 200, 300]); } } From 3c90738458bbe6312030a92bb0c34a9d63c3f3e6 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 08:10:28 -0400 Subject: [PATCH 15/21] test(hot): rewrite range conformance tests in logical terms Multi-shard scenarios that previously paired write_account_history calls with explicit subkeys collapse into single append_account_history calls with unioned block lists. The multi-shard layout is now an MDBX-internal detail; conformance tests assert the logical union, not the layout. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/src/conformance/range.rs | 148 ++++++++++++++-------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/crates/hot/src/conformance/range.rs b/crates/hot/src/conformance/range.rs index a3c1c08..ccc07d1 100644 --- a/crates/hot/src/conformance/range.rs +++ b/crates/hot/src/conformance/range.rs @@ -1,7 +1,7 @@ //! Clear/take range operations for single and dual-keyed tables. use crate::{ - db::{HistoryWrite, HotDbRead, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, + db::{HistoryRead, HistoryWrite, HotDbRead, UnsafeDbWrite}, model::{HotKv, HotKvWrite}, tables, }; @@ -235,40 +235,41 @@ pub fn test_take_range(hot_kv: &T) { /// 1. All k2 entries for k1 values within the range are deleted /// 2. k1 values outside the range remain intact /// 3. Edge cases work correctly -pub fn test_clear_range_dual(hot_kv: &T) { +pub fn test_clear_range_dual(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr1 = address!("0x1000000000000000000000000000000000000001"); let addr2 = address!("0x2000000000000000000000000000000000000002"); let addr3 = address!("0x3000000000000000000000000000000000000003"); let addr4 = address!("0x4000000000000000000000000000000000000004"); let addr5 = address!("0x5000000000000000000000000000000000000005"); - // Phase 1: Write account history entries for multiple addresses with multiple shards + // Phase 1: Write account history entries for multiple addresses. + // Multi-shard layout is an MDBX-internal detail; the logical input is the + // union of all blocks where each address was touched. { let writer = hot_kv.writer().unwrap(); - // addr1: two shards - let history1_a = BlockNumberList::new([1, 2, 3]).unwrap(); - let history1_b = BlockNumberList::new([4, 5, 6]).unwrap(); - writer.write_account_history(&addr1, 100, &history1_a).unwrap(); - writer.write_account_history(&addr1, u64::MAX, &history1_b).unwrap(); + // addr1: touched in blocks 1-6 (historically split across two shards) + let history1 = BlockNumberList::new([1, 2, 3, 4, 5, 6]).unwrap(); + writer.append_account_history(&addr1, &history1).unwrap(); - // addr2: one shard + // addr2: touched in blocks 10, 20 let history2 = BlockNumberList::new([10, 20]).unwrap(); - writer.write_account_history(&addr2, u64::MAX, &history2).unwrap(); + writer.append_account_history(&addr2, &history2).unwrap(); - // addr3: one shard + // addr3: touched in blocks 30, 40 let history3 = BlockNumberList::new([30, 40]).unwrap(); - writer.write_account_history(&addr3, u64::MAX, &history3).unwrap(); + writer.append_account_history(&addr3, &history3).unwrap(); - // addr4: two shards - let history4_a = BlockNumberList::new([50, 60]).unwrap(); - let history4_b = BlockNumberList::new([70, 80]).unwrap(); - writer.write_account_history(&addr4, 200, &history4_a).unwrap(); - writer.write_account_history(&addr4, u64::MAX, &history4_b).unwrap(); + // addr4: touched in blocks 50-80 (historically split across two shards) + let history4 = BlockNumberList::new([50, 60, 70, 80]).unwrap(); + writer.append_account_history(&addr4, &history4).unwrap(); - // addr5: one shard + // addr5: touched in blocks 90, 100 let history5 = BlockNumberList::new([90, 100]).unwrap(); - writer.write_account_history(&addr5, u64::MAX, &history5).unwrap(); + writer.append_account_history(&addr5, &history5).unwrap(); writer.commit().unwrap(); } @@ -276,16 +277,15 @@ pub fn test_clear_range_dual(hot_kv: &T) { // Verify all entries exist { let reader = hot_kv.reader().unwrap(); - assert!(reader.get_account_history(&addr1, 100).unwrap().is_some()); - assert!(reader.get_account_history(&addr1, u64::MAX).unwrap().is_some()); - assert!(reader.get_account_history(&addr2, u64::MAX).unwrap().is_some()); - assert!(reader.get_account_history(&addr3, u64::MAX).unwrap().is_some()); - assert!(reader.get_account_history(&addr4, 200).unwrap().is_some()); - assert!(reader.get_account_history(&addr4, u64::MAX).unwrap().is_some()); - assert!(reader.get_account_history(&addr5, u64::MAX).unwrap().is_some()); + assert!(reader.blocks_changed_account(&addr1).unwrap().is_some(), "addr1 should exist"); + assert!(reader.blocks_changed_account(&addr2).unwrap().is_some(), "addr2 should exist"); + assert!(reader.blocks_changed_account(&addr3).unwrap().is_some(), "addr3 should exist"); + assert!(reader.blocks_changed_account(&addr4).unwrap().is_some(), "addr4 should exist"); + assert!(reader.blocks_changed_account(&addr5).unwrap().is_some(), "addr5 should exist"); } - // Phase 2: Clear range addr2..=addr3 (middle range) + // Phase 2: Clear range addr2..=addr3 (middle range) via raw table traversal. + // This exercises the dual-key delete_range operation directly. { let writer = hot_kv.writer().unwrap(); writer @@ -296,42 +296,29 @@ pub fn test_clear_range_dual(hot_kv: &T) { writer.commit().unwrap(); } - // Verify: addr1 and addr4, addr5 should exist, addr2 and addr3 should be gone + // Verify: addr1, addr4, addr5 still have history; addr2 and addr3 are gone { let reader = hot_kv.reader().unwrap(); - // addr1 entries should still exist assert!( - reader.get_account_history(&addr1, 100).unwrap().is_some(), - "addr1 shard 100 should exist" + reader.blocks_changed_account(&addr1).unwrap().is_some(), + "addr1 should still have history" ); assert!( - reader.get_account_history(&addr1, u64::MAX).unwrap().is_some(), - "addr1 shard max should exist" - ); - - // addr2 and addr3 should be deleted - assert!( - reader.get_account_history(&addr2, u64::MAX).unwrap().is_none(), + reader.blocks_changed_account(&addr2).unwrap().is_none(), "addr2 should be deleted" ); assert!( - reader.get_account_history(&addr3, u64::MAX).unwrap().is_none(), + reader.blocks_changed_account(&addr3).unwrap().is_none(), "addr3 should be deleted" ); - - // addr4 and addr5 entries should still exist - assert!( - reader.get_account_history(&addr4, 200).unwrap().is_some(), - "addr4 shard 200 should exist" - ); assert!( - reader.get_account_history(&addr4, u64::MAX).unwrap().is_some(), - "addr4 shard max should exist" + reader.blocks_changed_account(&addr4).unwrap().is_some(), + "addr4 should still have history" ); assert!( - reader.get_account_history(&addr5, u64::MAX).unwrap().is_some(), - "addr5 should exist" + reader.blocks_changed_account(&addr5).unwrap().is_some(), + "addr5 should still have history" ); } } @@ -339,33 +326,37 @@ pub fn test_clear_range_dual(hot_kv: &T) { /// Test take_range_dual on a dual-keyed table. /// /// Similar to clear_range_dual but also returns the removed (k1, k2) pairs. -pub fn test_take_range_dual(hot_kv: &T) { +pub fn test_take_range_dual(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr1 = address!("0xa000000000000000000000000000000000000001"); let addr2 = address!("0xb000000000000000000000000000000000000002"); let addr3 = address!("0xc000000000000000000000000000000000000003"); - // Phase 1: Write account history entries + // Phase 1: Write account history entries. + // addr1 was historically two shards; collapse into the logical union. { let writer = hot_kv.writer().unwrap(); - // addr1: two shards - let history1_a = BlockNumberList::new([1, 2]).unwrap(); - let history1_b = BlockNumberList::new([3, 4]).unwrap(); - writer.write_account_history(&addr1, 50, &history1_a).unwrap(); - writer.write_account_history(&addr1, u64::MAX, &history1_b).unwrap(); + // addr1: touched in blocks 1-4 (historically split across two shards) + let history1 = BlockNumberList::new([1, 2, 3, 4]).unwrap(); + writer.append_account_history(&addr1, &history1).unwrap(); - // addr2: one shard + // addr2: touched in blocks 10, 20 let history2 = BlockNumberList::new([10, 20]).unwrap(); - writer.write_account_history(&addr2, u64::MAX, &history2).unwrap(); + writer.append_account_history(&addr2, &history2).unwrap(); - // addr3: one shard + // addr3: touched in blocks 30, 40 let history3 = BlockNumberList::new([30, 40]).unwrap(); - writer.write_account_history(&addr3, u64::MAX, &history3).unwrap(); + writer.append_account_history(&addr3, &history3).unwrap(); writer.commit().unwrap(); } - // Phase 2: Take range addr1..=addr2 and verify returned pairs + // Phase 2: Take range addr1..=addr2 via raw table traversal. + // The number of removed physical entries is backend-specific; assert only + // that at least one entry per logical address was removed. { let writer = hot_kv.writer().unwrap(); let removed = writer @@ -375,23 +366,32 @@ pub fn test_take_range_dual(hot_kv: &T) { .unwrap(); writer.commit().unwrap(); - // Should return (addr1, 50), (addr1, max), (addr2, max) - assert_eq!(removed.len(), 3, "should have removed 3 entries"); - assert_eq!(removed[0].0, addr1); - assert_eq!(removed[0].1, 50); - assert_eq!(removed[1].0, addr1); - assert_eq!(removed[1].1, u64::MAX); - assert_eq!(removed[2].0, addr2); - assert_eq!(removed[2].1, u64::MAX); + assert!(!removed.is_empty(), "should have removed entries for addr1 and addr2"); + // All removed entries must belong to addr1 or addr2 + for (addr, _, _) in &removed { + assert!( + *addr == addr1 || *addr == addr2, + "removed entry belongs to unexpected address" + ); + } } - // Verify only addr3 remains + // Verify logical state: addr1 and addr2 have no history; addr3 still does { let reader = hot_kv.reader().unwrap(); - assert!(reader.get_account_history(&addr1, 50).unwrap().is_none()); - assert!(reader.get_account_history(&addr1, u64::MAX).unwrap().is_none()); - assert!(reader.get_account_history(&addr2, u64::MAX).unwrap().is_none()); - assert!(reader.get_account_history(&addr3, u64::MAX).unwrap().is_some()); + assert!( + reader.blocks_changed_account(&addr1).unwrap().is_none(), + "addr1 should have no history after take" + ); + assert!( + reader.blocks_changed_account(&addr2).unwrap().is_none(), + "addr2 should have no history after take" + ); + let addr3_blocks = reader + .blocks_changed_account(&addr3) + .unwrap() + .expect("addr3 should still have history"); + assert_eq!(addr3_blocks.iter().collect::>(), vec![30, 40]); } // Phase 3: Take empty range From 9719a11930e0f2fa4d32d747b8b806cb01f28647 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 08:11:51 -0400 Subject: [PATCH 16/21] test(hot): switch load_genesis integration test to logical history reads Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/tests/load_genesis.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/hot/tests/load_genesis.rs b/crates/hot/tests/load_genesis.rs index 91ac954..95ef30b 100644 --- a/crates/hot/tests/load_genesis.rs +++ b/crates/hot/tests/load_genesis.rs @@ -9,7 +9,7 @@ use alloy::{ }; use signet_hot::{ HotKv, - db::{HistoryWrite, HotDbRead, LegacyHistoryRead, UnsafeDbWrite}, + db::{HistoryRead, HistoryWrite, HotDbRead, LegacyHistoryRead, UnsafeDbWrite}, mem::MemKv, }; use signet_storage_types::EthereumHardfork; @@ -181,8 +181,7 @@ fn verify_genesis_loaded(db: &MemKv, genesis: &Genesis, hardforks: &EthereumHard ); // Verify storage history - let storage_history = - reader.get_storage_history(address, slot_u256, u64::MAX).unwrap(); + let storage_history = reader.blocks_changed_storage(address, &slot_u256).unwrap(); assert!( storage_history.is_some(), "Storage history should exist for {address} slot {slot}" @@ -196,7 +195,7 @@ fn verify_genesis_loaded(db: &MemKv, genesis: &Genesis, hardforks: &EthereumHard } // Verify account history - let account_history = reader.get_account_history(address, u64::MAX).unwrap(); + let account_history = reader.blocks_changed_account(address).unwrap(); assert!(account_history.is_some(), "Account history should exist for {address}"); let history_list = account_history.unwrap(); assert!( From 979fb84bc133289596a3e96206eb1f538a1cc9ec Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 08:18:54 -0400 Subject: [PATCH 17/21] refactor(hot): delete legacy history surface LegacyHistoryRead, LegacyUnsafeHistoryWrite, and the append_to_sharded_history free fn are all replaced by the logical HistoryRead / HistoryWrite traits in crate::db::history. Non-shard-aware header/range/check methods migrate as default impls onto the new HistoryRead. No more ShardedKey<_> in public return types. No more shard subkeys in method signatures. The splitter logic now lives in signet_storage_types::merge_and_split and is called only from the MDBX backend. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/README.md | 4 +- crates/hot/src/conformance/roundtrip.rs | 30 ++- crates/hot/src/db/history.rs | 104 +++++++- crates/hot/src/db/inconsistent.rs | 180 +------------ crates/hot/src/db/mod.rs | 4 +- crates/hot/src/db/read.rs | 326 +----------------------- crates/hot/src/lib.rs | 10 +- crates/hot/src/model/revm.rs | 6 +- crates/hot/src/model/traits.rs | 6 +- crates/hot/tests/load_genesis.rs | 2 +- crates/storage/README.md | 2 +- crates/storage/src/lib.rs | 2 +- crates/storage/src/unified.rs | 2 +- crates/storage/tests/unified.rs | 2 +- 14 files changed, 145 insertions(+), 535 deletions(-) diff --git a/crates/hot/README.md b/crates/hot/README.md index b936b2a..b05e47d 100644 --- a/crates/hot/README.md +++ b/crates/hot/README.md @@ -8,7 +8,7 @@ with opinionated serialization and predefined tables for blockchain state. ## Usage ```rust,ignore -use signet_hot::{HotKv, LegacyHistoryRead, HistoryWrite}; +use signet_hot::{HotKv, HistoryRead, HistoryWrite}; fn example(db: &D) -> Result<(), signet_hot::db::HotKvError> { // Read operations @@ -32,7 +32,7 @@ For a concrete implementation, see the `signet-hot-mdbx` crate. HotKv ← Transaction factory ├─ reader() → HotKvRead ← Read-only transactions │ └─ HotDbRead ← Typed accessors (blanket impl) - │ └─ LegacyHistoryRead ← History queries (blanket impl) + │ └─ HistoryRead ← History queries (blanket impl) └─ writer() → HotKvWrite ← Read-write transactions └─ UnsafeDbWrite ← Low-level writes (blanket impl) └─ HistoryWrite ← Safe chain operations (per-backend impl) diff --git a/crates/hot/src/conformance/roundtrip.rs b/crates/hot/src/conformance/roundtrip.rs index 96dca2d..2dcc161 100644 --- a/crates/hot/src/conformance/roundtrip.rs +++ b/crates/hot/src/conformance/roundtrip.rs @@ -1,7 +1,7 @@ //! Basic CRUD roundtrip tests for hot storage. use crate::{ - db::{HistoryWrite, HotDbRead, LegacyHistoryRead, LegacyUnsafeHistoryWrite, UnsafeDbWrite}, + db::{HistoryRead, HistoryWrite, HotDbRead, UnsafeDbWrite}, model::{HotKv, HotKvRead}, tables, }; @@ -155,47 +155,51 @@ pub fn test_bytecode_roundtrip(hot_kv: &T) { } } -/// Test account history via HotHistoryWrite/HotHistoryRead -pub fn test_account_history(hot_kv: &T) { +/// Test account history via HistoryWrite/HistoryRead +pub fn test_account_history(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr = address!("0x1111111111111111111111111111111111111111"); let touched_blocks = BlockNumberList::new([10, 20, 30]).unwrap(); - let latest_height = 100u64; // Write account history { let writer = hot_kv.writer().unwrap(); - writer.write_account_history(&addr, latest_height, &touched_blocks).unwrap(); + writer.append_account_history(&addr, &touched_blocks).unwrap(); writer.commit().unwrap(); } // Read account history { let reader = hot_kv.reader().unwrap(); - let read_history = reader.get_account_history(&addr, latest_height).unwrap(); + let read_history = reader.blocks_changed_account(&addr).unwrap(); assert!(read_history.is_some()); let history = read_history.unwrap(); assert_eq!(history.iter().collect::>(), vec![10, 20, 30]); } } -/// Test storage history via HotHistoryWrite/HotHistoryRead -pub fn test_storage_history(hot_kv: &T) { +/// Test storage history via HistoryWrite/HistoryRead +pub fn test_storage_history(hot_kv: &T) +where + T::RwTx: HistoryWrite, +{ let addr = address!("0x2222222222222222222222222222222222222222"); let slot = U256::from(42); let touched_blocks = BlockNumberList::new([5, 15, 25]).unwrap(); - let highest_block = 50u64; // Write storage history { let writer = hot_kv.writer().unwrap(); - writer.write_storage_history(&addr, slot, highest_block, &touched_blocks).unwrap(); + writer.append_storage_history(&addr, &slot, &touched_blocks).unwrap(); writer.commit().unwrap(); } // Read storage history { let reader = hot_kv.reader().unwrap(); - let read_history = reader.get_storage_history(&addr, slot, highest_block).unwrap(); + let read_history = reader.blocks_changed_storage(&addr, &slot).unwrap(); assert!(read_history.is_some()); let history = read_history.unwrap(); assert_eq!(history.iter().collect::>(), vec![5, 15, 25]); @@ -284,10 +288,10 @@ pub fn test_missing_reads(hot_kv: &T) { assert!(reader.header_by_hash(&missing_hash).unwrap().is_none()); // Missing account history - assert!(reader.get_account_history(&missing_addr, 1000).unwrap().is_none()); + assert!(reader.blocks_changed_account(&missing_addr).unwrap().is_none()); // Missing storage history - assert!(reader.get_storage_history(&missing_addr, missing_slot, 1000).unwrap().is_none()); + assert!(reader.blocks_changed_storage(&missing_addr, &missing_slot).unwrap().is_none()); // Missing account change assert!(reader.get_account_change(999999, &missing_addr).unwrap().is_none()); diff --git a/crates/hot/src/db/history.rs b/crates/hot/src/db/history.rs index d97ea3d..6a873ca 100644 --- a/crates/hot/src/db/history.rs +++ b/crates/hot/src/db/history.rs @@ -8,7 +8,7 @@ //! MemKv writes a single dup entry per addr). use crate::{ - db::{HistoryError, HotDbRead, LegacyHistoryRead, UnsafeDbWrite}, + db::{HistoryError, HotDbRead, UnsafeDbWrite}, model::HotKvRead, tables, }; @@ -181,6 +181,108 @@ pub trait HistoryRead: HotDbRead { Some(first) => self.get_storage_change(first, addr, slot), } } + + /// Get the last (highest) header in the database. + /// Returns None if the database is empty. + fn last_header(&self) -> Result, Self::Error> { + let mut cursor = self.traverse::()?; + Ok(cursor.last()?.map(|(_, header)| header)) + } + + /// Get the last (highest) block number in the database. + /// Returns None if the database is empty. + fn last_block_number(&self) -> Result, Self::Error> { + let mut cursor = self.traverse::()?; + Ok(cursor.last()?.map(|(number, _)| number)) + } + + /// Get the first (lowest) header in the database. + /// Returns None if the database is empty. + fn first_header(&self) -> Result, Self::Error> { + let mut cursor = self.traverse::()?; + Ok(cursor.first()?.map(|(_, header)| header)) + } + + /// Get the current chain tip (highest block number and hash). + /// Returns None if the database is empty. + fn get_chain_tip(&self) -> Result, Self::Error> { + let mut cursor = self.traverse::()?; + let Some((number, header)) = cursor.last()? else { + return Ok(None); + }; + let hash = header.hash(); + Ok(Some((number, hash))) + } + + /// Get the execution range (first and last block numbers with headers). + /// Returns None if the database is empty. + fn get_execution_range(&self) -> Result, Self::Error> { + let mut cursor = self.traverse::()?; + let Some((first, _)) = cursor.first()? else { + return Ok(None); + }; + let Some((last, _)) = cursor.last()? else { + return Ok(None); + }; + Ok(Some((first, last))) + } + + /// Check if a specific block number exists in history. + fn has_block(&self, number: u64) -> Result { + self.get_header(number).map(|opt| opt.is_some()) + } + + /// Get headers in a range (inclusive). + fn get_headers_range(&self, start: u64, end: u64) -> Result, Self::Error> { + self.traverse::()? + .iter_from(&start)? + .take_while(|r| r.as_ref().is_ok_and(|(num, _)| *num <= end)) + .map(|r| r.map(|(_, header)| header)) + .collect() + } + + /// Validate that `height` is within the stored block range. + /// + /// Returns `Ok(())` if `height` is `None` (current state) or within the + /// range of stored blocks. Returns an error if the database has no + /// blocks or if the height is out of range. + fn check_height(&self, height: Option) -> Result<(), HistoryError> { + let Some(height) = height else { return Ok(()) }; + let Some((first, last)) = self.get_execution_range().map_err(HistoryError::Db)? else { + return Err(HistoryError::NoBlocks); + }; + if height < first || height > last { + return Err(HistoryError::HeightOutOfRange { height, first, last }); + } + Ok(()) + } + + /// Get account state at a height, with range validation. + /// + /// Validates that `height` is within the stored block range before + /// delegating to [`Self::get_account_at_height`]. + fn get_account_at_height_checked( + &self, + addr: &Address, + height: Option, + ) -> Result, HistoryError> { + self.check_height(height)?; + self.get_account_at_height(addr, height).map_err(HistoryError::Db) + } + + /// Get storage slot value at a height, with range validation. + /// + /// Validates that `height` is within the stored block range before + /// delegating to [`Self::get_storage_at_height`]. + fn get_storage_at_height_checked( + &self, + addr: &Address, + slot: &U256, + height: Option, + ) -> Result, HistoryError> { + self.check_height(height)?; + self.get_storage_at_height(addr, slot, height).map_err(HistoryError::Db) + } } impl HistoryRead for T where T: HotKvRead {} diff --git a/crates/hot/src/db/inconsistent.rs b/crates/hot/src/db/inconsistent.rs index 47e8dfd..0aa5bff 100644 --- a/crates/hot/src/db/inconsistent.rs +++ b/crates/hot/src/db/inconsistent.rs @@ -1,13 +1,7 @@ -use crate::{ - db::{HistoryError, LegacyHistoryRead}, - model::HotKvWrite, - tables, -}; +use crate::{model::HotKvWrite, tables}; use ahash::AHashMap; -use alloy::primitives::{Address, B256, BlockNumber, U256}; -use itertools::Itertools; -use signet_storage_types::{Account, BlockNumberList, SealedHeader, ShardedKey}; -use std::ops::RangeInclusive; +use alloy::primitives::{Address, B256, U256}; +use signet_storage_types::{Account, SealedHeader}; use trevm::revm::bytecode::Bytecode; /// Bundle state initialization type. @@ -105,171 +99,3 @@ pub trait UnsafeDbWrite: HotKvWrite + super::sealed::Sealed { } impl UnsafeDbWrite for T where T: HotKvWrite {} - -/// Trait for history write operations. -/// -/// These tables maintain historical information about accounts and storage -/// changes, and their contents can be used to reconstruct past states or -/// roll back changes. -pub trait LegacyUnsafeHistoryWrite: UnsafeDbWrite + LegacyHistoryRead { - /// Maintain a list of block numbers where an account was touched. - /// - /// Accounts are keyed - fn write_account_history( - &self, - address: &Address, - latest_height: u64, - touched: &BlockNumberList, - ) -> Result<(), Self::Error> { - self.queue_put_dual::(address, &latest_height, touched) - } - - /// Write storage history, by highest block number and touched block - /// numbers. - fn write_storage_history( - &self, - address: &Address, - slot: U256, - highest_block_number: u64, - touched: &BlockNumberList, - ) -> Result<(), Self::Error> { - let sharded_key = ShardedKey::new(slot, highest_block_number); - self.queue_put_dual::(address, &sharded_key, touched) - } - - /// Append account history indices for multiple accounts. - fn append_account_history_index( - &self, - index_updates: impl IntoIterator)>, - ) -> Result<(), HistoryError> { - for (acct, indices) in index_updates { - let existing = self.last_account_history(acct)?; - append_to_sharded_history( - existing, - indices, - |key| self.queue_delete_dual::(&acct, &key), - |height, list| self.write_account_history(&acct, height, list), - )?; - } - Ok(()) - } - - /// Append storage history indices for multiple (address, slot) pairs. - fn append_storage_history_index( - &self, - index_updates: impl IntoIterator)>, - ) -> Result<(), HistoryError> { - for ((addr, slot), indices) in index_updates { - let existing = self.last_storage_history(&addr, &slot)?; - append_to_sharded_history( - existing, - indices, - |key| self.queue_delete_dual::(&addr, &key), - |height, list| self.write_storage_history(&addr, slot, height, list), - )?; - } - Ok(()) - } - - /// Update the history indices for accounts and storage in the given block - /// range. - fn update_history_indices_inconsistent( - &self, - range: RangeInclusive, - ) -> Result<(), HistoryError> { - // account history stage - // TODO: estimate capacity from block range size for better allocation - let account_indices: AHashMap> = self - .traverse_dual::()? - .iter_from(range.start(), &Address::ZERO)? - .process_results(|iter| { - iter.take_while(|(num, _, _)| range.contains(num)) - .map(|(num, addr, _)| (addr, num)) - .into_group_map_by(|(addr, _)| *addr) - .into_iter() - .map(|(addr, pairs)| (addr, pairs.into_iter().map(|(_, num)| num).collect())) - .collect() - })?; - self.append_account_history_index(account_indices)?; - - // storage history stage - // TODO: estimate capacity from block range size for better allocation - let storage_indices: AHashMap<(Address, U256), Vec> = self - .traverse_dual::()? - .iter_from(&(*range.start(), Address::ZERO), &U256::ZERO)? - .process_results(|iter| { - iter.take_while(|(num_addr, _, _)| range.contains(&num_addr.0)) - .map(|(num_addr, slot, _)| ((num_addr.1, slot), num_addr.0)) - .into_group_map_by(|(key, _)| *key) - .into_iter() - .map(|(key, pairs)| (key, pairs.into_iter().map(|(_, num)| num).collect())) - .collect() - })?; - self.append_storage_history_index(storage_indices)?; - - Ok(()) - } -} - -impl LegacyUnsafeHistoryWrite for T where T: UnsafeDbWrite + HotKvWrite {} - -/// Append indices to a sharded history entry, handling shard splitting. -/// -/// This helper handles the common pattern of: -/// 1. Appending new block numbers to an existing shard -/// 2. Deleting the old shard if it exists -/// 3. Splitting into multiple shards if the result exceeds the shard size -/// -/// # Arguments -/// - `existing`: The current last shard (key, list) if any -/// - `indices`: New block numbers to append -/// - `delete_old`: Called to delete the old shard key before writing new ones -/// - `write_shard`: Called for each resulting shard (highest_block, list) -fn append_to_sharded_history( - existing: Option<(K, BlockNumberList)>, - indices: impl IntoIterator, - mut delete_old: D, - mut write_shard: W, -) -> Result<(), HistoryError> -where - E: std::error::Error, - D: FnMut(K) -> Result<(), E>, - W: FnMut(u64, &BlockNumberList) -> Result<(), E>, -{ - let (old_key, last_shard) = - existing.map_or_else(|| (None, BlockNumberList::default()), |(k, list)| (Some(k), list)); - let mut last_shard = last_shard; - - last_shard.append(indices).map_err(HistoryError::IntList)?; - - // Delete the existing shard before writing new ones to avoid duplicates - if let Some(key) = old_key { - delete_old(key).map_err(HistoryError::Db)?; - } - - // Fast path: all indices fit in one shard - if last_shard.len() <= ShardedKey::SHARD_COUNT as u64 { - return write_shard(u64::MAX, &last_shard).map_err(HistoryError::Db); - } - - // Slow path: rechunk into multiple shards - // Reuse a single buffer to avoid allocating a new Vec per chunk - let mut chunk_buf = Vec::with_capacity(ShardedKey::SHARD_COUNT); - let mut iter = last_shard.iter().peekable(); - - while iter.peek().is_some() { - chunk_buf.clear(); - chunk_buf.extend(iter.by_ref().take(ShardedKey::SHARD_COUNT)); - - let highest = if iter.peek().is_some() { - *chunk_buf.last().expect("chunk_buf is non-empty") - } else { - // Insert last list with `u64::MAX` - u64::MAX - }; - - let shard = BlockNumberList::new_pre_sorted(chunk_buf.iter().copied()); - write_shard(highest, &shard).map_err(HistoryError::Db)?; - } - Ok(()) -} diff --git a/crates/hot/src/db/mod.rs b/crates/hot/src/db/mod.rs index 43993c1..c222018 100644 --- a/crates/hot/src/db/mod.rs +++ b/crates/hot/src/db/mod.rs @@ -7,10 +7,10 @@ pub mod history; pub use history::{HistoryRead, HistoryWrite}; mod inconsistent; -pub use inconsistent::{BundleInit, LegacyUnsafeHistoryWrite, UnsafeDbWrite}; +pub use inconsistent::{BundleInit, UnsafeDbWrite}; mod read; -pub use read::{HotDbRead, LegacyHistoryRead}; +pub use read::HotDbRead; pub(crate) mod sealed { use crate::model::HotKvRead; diff --git a/crates/hot/src/db/read.rs b/crates/hot/src/db/read.rs index 6fc8cd1..74062be 100644 --- a/crates/hot/src/db/read.rs +++ b/crates/hot/src/db/read.rs @@ -1,6 +1,6 @@ -use crate::{db::HistoryError, model::HotKvRead, tables}; +use crate::{model::HotKvRead, tables}; use alloy::primitives::{Address, B256, U256}; -use signet_storage_types::{Account, BlockNumberList, SealedHeader, ShardedKey}; +use signet_storage_types::{Account, SealedHeader}; use trevm::revm::bytecode::Bytecode; /// Trait for database read operations on standard hot tables. @@ -47,325 +47,3 @@ pub trait HotDbRead: HotKvRead + super::sealed::Sealed { } impl HotDbRead for T where T: HotKvRead {} - -/// Trait for history read operations. -/// -/// These tables maintain historical information about accounts and storage -/// changes, and their contents can be used to reconstruct past states or -/// roll back changes. -/// -/// This is a high-level trait that provides convenient methods for reading -/// common data types from predefined hot storage history tables. It builds -/// upon the lower-level [`HotDbRead`] trait, which provides raw key-value -/// access. -/// -/// Users should prefer this trait unless customizations are needed to the -/// table set. -pub trait LegacyHistoryRead: HotDbRead { - /// Get the list of block numbers where an account was touched. - /// Get the list of block numbers where an account was touched. - fn get_account_history( - &self, - address: &Address, - latest_height: u64, - ) -> Result, Self::Error> { - self.get_dual::(address, &latest_height) - } - - /// Get the last (highest) account history entry for an address. - fn last_account_history( - &self, - address: Address, - ) -> Result, Self::Error> { - let mut cursor = self.traverse_dual::()?; - - // Move the cursor to the last entry for the given address - let Some(res) = cursor.last_of_k1(&address)? else { - return Ok(None); - }; - - Ok(Some((res.1, res.2))) - } - - /// Get the account change (pre-state) for an account at a specific block. - /// - /// If the return value is `None`, the account was not changed in that - /// block. - fn get_account_change( - &self, - block_number: u64, - address: &Address, - ) -> Result, Self::Error> { - self.get_dual::(&block_number, address) - } - - /// Get the storage history for an account and storage slot. The returned - /// list will contain block numbers where the storage slot was changed. - fn get_storage_history( - &self, - address: &Address, - slot: U256, - highest_block_number: u64, - ) -> Result, Self::Error> { - let sharded_key = ShardedKey::new(slot, highest_block_number); - self.get_dual::(address, &sharded_key) - } - - /// Get the last (highest) storage history entry for an address and slot. - fn last_storage_history( - &self, - address: &Address, - slot: &U256, - ) -> Result, BlockNumberList)>, Self::Error> { - let mut cursor = self.traverse_dual::()?; - - // Seek to the highest possible key for this (address, slot) combination. - // ShardedKey encodes as slot || highest_block_number, so seeking to - // (address, ShardedKey::new(slot, u64::MAX)) positions us at or after - // the last shard for this slot. - let target = ShardedKey::new(*slot, u64::MAX); - let result = cursor.next_dual_above(address, &target)?; - - // Check if we found an exact match for this address and slot - if let Some((addr, sharded_key, list)) = result - && addr == *address - && sharded_key.key == *slot - { - return Ok(Some((sharded_key, list))); - } - - // The cursor is positioned at or after our target. Go backwards to find - // the last entry for this (address, slot). - let Some((addr, sharded_key, list)) = cursor.previous_k2()? else { - return Ok(None); - }; - - if addr == *address && sharded_key.key == *slot { - Ok(Some((sharded_key, list))) - } else { - Ok(None) - } - } - - /// Get the storage change (before state) for a specific storage slot at a - /// specific block. - /// - /// If the return value is `None`, the storage slot was not changed in that - /// block. If the return value is `Some(value)`, the value is the pre-state - /// of the storage slot before the change in that block. If the value is - /// `U256::ZERO`, that indicates that the storage slot was not set before - /// the change. - fn get_storage_change( - &self, - block_number: u64, - address: &Address, - slot: &U256, - ) -> Result, Self::Error> { - let block_number_address = (block_number, *address); - self.get_dual::(&block_number_address, slot) - } - - /// Get the last (highest) header in the database. - /// Returns None if the database is empty. - fn last_header(&self) -> Result, Self::Error> { - let mut cursor = self.traverse::()?; - Ok(cursor.last()?.map(|(_, header)| header)) - } - - /// Get the last (highest) block number in the database. - /// Returns None if the database is empty. - fn last_block_number(&self) -> Result, Self::Error> { - let mut cursor = self.traverse::()?; - Ok(cursor.last()?.map(|(number, _)| number)) - } - - /// Get the first (lowest) header in the database. - /// Returns None if the database is empty. - fn first_header(&self) -> Result, Self::Error> { - let mut cursor = self.traverse::()?; - Ok(cursor.first()?.map(|(_, header)| header)) - } - - /// Get the current chain tip (highest block number and hash). - /// Returns None if the database is empty. - fn get_chain_tip(&self) -> Result, Self::Error> { - let mut cursor = self.traverse::()?; - let Some((number, header)) = cursor.last()? else { - return Ok(None); - }; - let hash = header.hash(); - Ok(Some((number, hash))) - } - - /// Get the execution range (first and last block numbers with headers). - /// Returns None if the database is empty. - fn get_execution_range(&self) -> Result, Self::Error> { - let mut cursor = self.traverse::()?; - let Some((first, _)) = cursor.first()? else { - return Ok(None); - }; - let Some((last, _)) = cursor.last()? else { - return Ok(None); - }; - Ok(Some((first, last))) - } - - /// Get account state, optionally at a specific historical block height. - /// - /// When `height` is `Some`, reconstructs the account state as it was at - /// that block height by consulting history and change set tables. When - /// `None`, returns the current value from `PlainAccountState`. - /// - /// If no changes exist after the given height, the current value is - /// returned (the account has not been modified since that height). - /// - /// # Note - /// - /// This method does **not** validate `height` against the stored block - /// range. Heights past the chain tip silently return current state, and - /// heights before the first block return the pre-state of the earliest - /// change. Use [`Self::get_account_at_height_checked`] or - /// [`HotKv::revm_reader_at_height`] for validated access. - /// - /// [`HotKv::revm_reader_at_height`]: crate::model::HotKv::revm_reader_at_height - fn get_account_at_height( - &self, - address: &Address, - height: Option, - ) -> Result, Self::Error> { - let Some(height) = height else { - return self.get_account(address); - }; - - let mut cursor = self.traverse_dual::()?; - - // Seek to the first shard with key2 >= height + 1 - let result = cursor.next_dual_above(address, &(height + 1))?; - - // Verify address matches; seek could overshoot to the next address - let Some((_, _, list)) = result.filter(|(addr, _, _)| *addr == *address) else { - // No history after height — account unchanged, use current value - return self.get_account(address); - }; - - // rank(height) = count of values <= height; select(rank) = first value > height - let rank = list.rank(height); - let Some(first_change) = list.select(rank) else { - // Defensive: shard key2 > height and is in the list, so this - // should not happen. Fall back to current value. - return self.get_account(address); - }; - - self.get_account_change(first_change, address) - } - - /// Get storage slot value, optionally at a specific historical block - /// height. - /// - /// When `height` is `Some`, reconstructs the storage value as it was at - /// that block height by consulting history and change set tables. When - /// `None`, returns the current value from `PlainStorageState`. - /// - /// If no changes exist after the given height, the current value is - /// returned (the slot has not been modified since that height). - /// - /// # Note - /// - /// This method does **not** validate `height` against the stored block - /// range. Heights past the chain tip silently return current state, and - /// heights before the first block return the pre-state of the earliest - /// change. Use [`Self::get_storage_at_height_checked`] or - /// [`HotKv::revm_reader_at_height`] for validated access. - /// - /// [`HotKv::revm_reader_at_height`]: crate::model::HotKv::revm_reader_at_height - fn get_storage_at_height( - &self, - address: &Address, - slot: &U256, - height: Option, - ) -> Result, Self::Error> { - let Some(height) = height else { - return self.get_storage(address, slot); - }; - - let mut cursor = self.traverse_dual::()?; - - // Seek to first shard with (address, ShardedKey { slot, block >= height+1 }) - let target = ShardedKey::new(*slot, height + 1); - let result = cursor.next_dual_above(address, &target)?; - - // Verify address AND slot match; seek could land on a different slot - let Some((_, _, list)) = - result.filter(|(addr, sk, _)| *addr == *address && sk.key == *slot) - else { - // No history after height — slot unchanged, use current value - return self.get_storage(address, slot); - }; - - let rank = list.rank(height); - let Some(first_change) = list.select(rank) else { - return self.get_storage(address, slot); - }; - - self.get_storage_change(first_change, address, slot) - } - - /// Validate that `height` is within the stored block range. - /// - /// Returns `Ok(())` if `height` is `None` (current state) or within the - /// range of stored blocks. Returns an error if the database has no - /// blocks or if the height is out of range. - fn check_height(&self, height: Option) -> Result<(), HistoryError> { - let Some(height) = height else { return Ok(()) }; - let Some((first, last)) = self.get_execution_range().map_err(HistoryError::Db)? else { - return Err(HistoryError::NoBlocks); - }; - if height < first || height > last { - return Err(HistoryError::HeightOutOfRange { height, first, last }); - } - Ok(()) - } - - /// Get account state at a height, with range validation. - /// - /// Validates that `height` is within the stored block range before - /// delegating to [`Self::get_account_at_height`]. - fn get_account_at_height_checked( - &self, - address: &Address, - height: Option, - ) -> Result, HistoryError> { - self.check_height(height)?; - self.get_account_at_height(address, height).map_err(HistoryError::Db) - } - - /// Get storage slot value at a height, with range validation. - /// - /// Validates that `height` is within the stored block range before - /// delegating to [`Self::get_storage_at_height`]. - fn get_storage_at_height_checked( - &self, - address: &Address, - slot: &U256, - height: Option, - ) -> Result, HistoryError> { - self.check_height(height)?; - self.get_storage_at_height(address, slot, height).map_err(HistoryError::Db) - } - - /// Check if a specific block number exists in history. - fn has_block(&self, number: u64) -> Result { - self.get_header(number).map(|opt| opt.is_some()) - } - - /// Get headers in a range (inclusive). - fn get_headers_range(&self, start: u64, end: u64) -> Result, Self::Error> { - self.traverse::()? - .iter_from(&start)? - .take_while(|r| r.as_ref().is_ok_and(|(num, _)| *num <= end)) - .map(|r| r.map(|(_, header)| header)) - .collect() - } -} - -impl LegacyHistoryRead for T where T: HotDbRead {} diff --git a/crates/hot/src/lib.rs b/crates/hot/src/lib.rs index e4288e9..d3a1256 100644 --- a/crates/hot/src/lib.rs +++ b/crates/hot/src/lib.rs @@ -7,7 +7,7 @@ //! # Quick Start //! //! ```ignore -//! use signet_hot::{HotKv, LegacyHistoryRead, HistoryWrite}; +//! use signet_hot::{HotKv, HistoryRead, HistoryWrite}; //! //! fn example(db: &D) -> Result<(), signet_hot::db::HotKvError> { //! // Read operations @@ -31,7 +31,7 @@ //! HotKv ← Transaction factory //! ├─ reader() → HotKvRead ← Read-only transactions //! │ └─ HotDbRead ← Typed accessors (blanket impl) -//! │ └─ LegacyHistoryRead ← History queries (blanket impl) +//! │ └─ HistoryRead ← History queries (blanket impl) //! └─ writer() → HotKvWrite ← Read-write transactions //! └─ UnsafeDbWrite ← Low-level writes (blanket impl) //! └─ HistoryWrite ← Safe chain operations (per-backend impl) @@ -56,7 +56,7 @@ //! These traits provide methods for common operations such as getting, //! setting, and deleting key-value pairs in hot storage tables. The raw //! key-value operations use byte slices for maximum flexibility. The -//! [`LegacyHistoryRead`] and [`HistoryWrite`] traits provide higher-level +//! [`HistoryRead`] and [`HistoryWrite`] traits provide higher-level //! abstractions that work with the predefined tables and their associated key //! and value types. //! @@ -73,7 +73,7 @@ //! See the [`Table`] trait documentation for more information on defining and //! using tables. //! -//! [`LegacyHistoryRead`]: db::LegacyHistoryRead +//! [`HistoryRead`]: db::HistoryRead //! [`HistoryWrite`]: db::HistoryWrite //! [`HotKvRead`]: model::HotKvRead //! [`HotKvWrite`]: model::HotKvWrite @@ -109,7 +109,7 @@ pub mod connect; pub use connect::HotConnect; pub mod db; -pub use db::{HistoryError, HistoryWrite, LegacyHistoryRead}; +pub use db::{HistoryError, HistoryRead, HistoryWrite}; pub mod model; pub use model::HotKv; diff --git a/crates/hot/src/model/revm.rs b/crates/hot/src/model/revm.rs index 3977677..5b19d91 100644 --- a/crates/hot/src/model/revm.rs +++ b/crates/hot/src/model/revm.rs @@ -1,5 +1,5 @@ use crate::{ - db::LegacyHistoryRead, + db::HistoryRead, model::{HotKvError, HotKvRead, HotKvWrite}, tables::{self, Bytecodes, DualKey, PlainAccountState, SingleKey, Table}, }; @@ -39,10 +39,10 @@ impl RevmRead { /// current state, and heights before the first block return the /// pre-state of the earliest change. Use /// [`HotKv::revm_reader_at_height`] which validates, or call - /// [`LegacyHistoryRead::check_height`] manually. + /// [`HistoryRead::check_height`] manually. /// /// [`HotKv::revm_reader_at_height`]: crate::model::HotKv::revm_reader_at_height - /// [`LegacyHistoryRead::check_height`]: crate::db::LegacyHistoryRead::check_height + /// [`HistoryRead::check_height`]: crate::db::HistoryRead::check_height pub const fn at_height(reader: T, height: u64) -> Self { Self { reader, height: Some(height) } } diff --git a/crates/hot/src/model/traits.rs b/crates/hot/src/model/traits.rs index 8f09566..7d4ee1c 100644 --- a/crates/hot/src/model/traits.rs +++ b/crates/hot/src/model/traits.rs @@ -1,5 +1,5 @@ use crate::{ - db::LegacyHistoryRead, + db::HistoryRead, model::{ DualKeyTraverse, DualKeyTraverseMut, DualTableCursor, HotKvError, HotKvReadError, KvTraverse, KvTraverseMut, TableCursor, @@ -15,13 +15,13 @@ use std::borrow::Cow; /// This is the top-level trait for hot storage backends, providing /// transactional access through read-only and read-write transactions. /// -/// We recommend using [`LegacyHistoryRead`] and [`HistoryWrite`] for most use cases, +/// We recommend using [`HistoryRead`] and [`HistoryWrite`] for most use cases, /// as they provide higher-level abstractions over predefined tables. /// /// When implementing this trait, consult the [`model`] module documentation for /// details on the associated types and their requirements. /// -/// [`LegacyHistoryRead`]: crate::db::LegacyHistoryRead +/// [`HistoryRead`]: crate::db::HistoryRead /// [`HistoryWrite`]: crate::db::HistoryWrite /// [`model`]: crate::model #[auto_impl::auto_impl(&, Arc, Box)] diff --git a/crates/hot/tests/load_genesis.rs b/crates/hot/tests/load_genesis.rs index 95ef30b..4036f80 100644 --- a/crates/hot/tests/load_genesis.rs +++ b/crates/hot/tests/load_genesis.rs @@ -9,7 +9,7 @@ use alloy::{ }; use signet_hot::{ HotKv, - db::{HistoryRead, HistoryWrite, HotDbRead, LegacyHistoryRead, UnsafeDbWrite}, + db::{HistoryRead, HistoryWrite, HotDbRead, UnsafeDbWrite}, mem::MemKv, }; use signet_storage_types::EthereumHardfork; diff --git a/crates/storage/README.md b/crates/storage/README.md index 8911b66..b1ffaa2 100644 --- a/crates/storage/README.md +++ b/crates/storage/README.md @@ -70,5 +70,5 @@ Cold dispatch errors indicate either: Key types are re-exported for convenience: - `ExecutedBlock`, `ExecutedBlockBuilder` - Block data structures -- `HotKv`, `LegacyHistoryRead`, `HistoryWrite` - Hot storage traits +- `HotKv`, `HistoryRead`, `HistoryWrite` - Hot storage traits - `ColdStorageHandle`, `ColdStorageError` - Cold storage types diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index feac742..c389e22 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -87,7 +87,7 @@ pub use signet_cold::{ }; pub use signet_cold_mdbx::MdbxColdBackend; pub use signet_hot::{ - HistoryError, HistoryWrite, HotKv, LegacyHistoryRead, + HistoryError, HistoryRead, HistoryWrite, HotKv, model::{HotKvRead, RevmRead, RevmWrite}, }; pub use signet_hot_mdbx::{DatabaseArguments, DatabaseEnv}; diff --git a/crates/storage/src/unified.rs b/crates/storage/src/unified.rs index 5e9345e..5da8f52 100644 --- a/crates/storage/src/unified.rs +++ b/crates/storage/src/unified.rs @@ -8,7 +8,7 @@ use crate::StorageResult; use alloy::primitives::BlockNumber; use signet_cold::{BlockData, ColdReceipt, ColdStorage, ColdStorageBackend, ColdStorageError}; use signet_hot::{ - HistoryWrite, HotKv, LegacyHistoryRead, + HistoryRead, HistoryWrite, HotKv, model::{HotKvReadError, HotKvWrite, RevmRead}, }; use signet_storage_types::{ExecutedBlock, SealedHeader}; diff --git a/crates/storage/tests/unified.rs b/crates/storage/tests/unified.rs index 1611e93..ed238be 100644 --- a/crates/storage/tests/unified.rs +++ b/crates/storage/tests/unified.rs @@ -5,7 +5,7 @@ use alloy::{ primitives::{Address, B256, Signature, TxKind, U256}, }; use signet_cold::{ColdStorage, HeaderSpecifier, mem::MemColdBackend}; -use signet_hot::{HistoryWrite, HotKv, LegacyHistoryRead, mem::MemKv, model::HotKvWrite}; +use signet_hot::{HistoryRead, HistoryWrite, HotKv, mem::MemKv, model::HotKvWrite}; use signet_storage::UnifiedStorage; use signet_storage_types::{ ExecutedBlock, ExecutedBlockBuilder, Receipt, RecoveredTx, SealedHeader, TransactionSigned, From cdbe2bae3c488a22014876e9cf6a5474b3a91d6b Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 08:19:54 -0400 Subject: [PATCH 18/21] chore(storage-types): drop obsolete ShardedKey::SHARD_COUNT Count-based shard splitting was replaced by merge_and_split's size-based logic. The constant has no remaining users. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/types/src/sharded.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/types/src/sharded.rs b/crates/types/src/sharded.rs index bc5ffed..1805b5d 100644 --- a/crates/types/src/sharded.rs +++ b/crates/types/src/sharded.rs @@ -12,11 +12,6 @@ pub struct ShardedKey { pub highest_block_number: u64, } -impl ShardedKey<()> { - /// Number of indices in one shard. - pub const SHARD_COUNT: usize = 2000; -} - impl ShardedKey { /// Creates a new `ShardedKey`. pub const fn new(key: T, highest_block_number: u64) -> Self { From f46b0ea18faf450ac6da5859ea245a94f860e665 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 08:24:36 -0400 Subject: [PATCH 19/21] test(hot-mdbx): structural assertion that MDBX splits oversized history 200 sparse blocks (200 distinct roaring containers, written as two batches of 100 to satisfy the per-additions shard precondition) feed the MDBX HistoryWrite impl. Tests assert that at least 2 dup entries exist for the addr in AccountsHistory (and analogously for StorageHistory), and that the tail shard's subkey is u64::MAX. This is the only place ShardedKey<_> appears outside the abstraction crate by design. Also adds [[test]] required-features = ["test-utils"] to Cargo.toml so the integration test is skipped under --no-default-features. Co-Authored-By: Claude Sonnet 4.6 --- crates/hot-mdbx/Cargo.toml | 4 + crates/hot-mdbx/tests/history_sharding.rs | 126 ++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 crates/hot-mdbx/tests/history_sharding.rs diff --git a/crates/hot-mdbx/Cargo.toml b/crates/hot-mdbx/Cargo.toml index f5d1210..38778ff 100644 --- a/crates/hot-mdbx/Cargo.toml +++ b/crates/hot-mdbx/Cargo.toml @@ -34,6 +34,10 @@ serial_test = "3.3.1" signet-hot = { workspace = true, features = ["test-utils"] } tempfile.workspace = true +[[test]] +name = "history_sharding" +required-features = ["test-utils"] + [features] default = [] test-utils = ["signet-hot/test-utils", "dep:tempfile"] diff --git a/crates/hot-mdbx/tests/history_sharding.rs b/crates/hot-mdbx/tests/history_sharding.rs new file mode 100644 index 0000000..e7f8706 --- /dev/null +++ b/crates/hot-mdbx/tests/history_sharding.rs @@ -0,0 +1,126 @@ +//! MDBX-specific structural test: prove that signet-hot-mdbx splits the +//! AccountsHistory / StorageHistory tables into multiple shards when the +//! input exceeds MAX_SHARD_BYTES. + +use alloy::primitives::{Address, U256}; +use serial_test::serial; +use signet_hot::{ + HotKv, + db::{HistoryRead, HistoryWrite, UnsafeDbWrite}, + model::HotKvRead, + tables, +}; +use signet_hot_mdbx::test_utils::create_test_rw_db; +use signet_storage_types::{BlockNumberList, ShardedKey}; + +/// 200 sparse blocks: one per distinct 16-bit roaring container. +/// +/// Each block `i * 0x1_0000` falls in container `i`, so the roaring bitmap +/// cannot use run-length or bitmap compression across containers. The first +/// 100 blocks encode to ~1400 B (just under the 1500 B shard budget), and +/// the second 100 blocks encode to another ~1400 B. Appending all 200 in +/// two writes of 100 triggers the split: merged size ~2800 B > 1500 B. +/// +/// The two writes mirror the `merge_and_split_at_realistic_budget_respects_per_shard_size` +/// unit test in `signet-storage-types`. +fn sparse_blocks() -> Vec { + (0..200u64).map(|i| i * 0x1_0000).collect() +} + +#[test] +#[serial] +fn account_history_splits_on_oversized_input() { + let (_dir, db) = create_test_rw_db(); + let addr = Address::from_slice(&[0x1; 20]); + let blocks = sparse_blocks(); + // Split into two writes of 100 blocks each. Each batch individually fits + // within MAX_SHARD_BYTES (~1400 B < 1500 B), but merging both (~2800 B) + // forces the MDBX backend to create at least two dup entries. + let first_half = BlockNumberList::new(blocks[..100].iter().copied()).unwrap(); + let second_half = BlockNumberList::new(blocks[100..].iter().copied()).unwrap(); + + // Write in two transactions to match the incremental history-append pattern. + { + let writer = db.writer().unwrap(); + writer.append_account_history(&addr, &first_half).unwrap(); + writer.commit().unwrap(); + } + { + let writer = db.writer().unwrap(); + writer.append_account_history(&addr, &second_half).unwrap(); + writer.commit().unwrap(); + } + + let reader = db.reader().unwrap(); + + // Logical: round-trip succeeds and returns the same blocks. + let recovered = reader.blocks_changed_account(&addr).unwrap().unwrap(); + assert_eq!(recovered.iter().collect::>(), blocks); + + // Structural: count dup entries directly via the raw cursor. ≥2 expected. + let dup_count = + reader.traverse_dual::().unwrap().iter_k2(&addr).unwrap().count(); + assert!( + dup_count >= 2, + "expected MDBX to split oversized input into >=2 shards, got {dup_count}" + ); + + // Structural: the tail shard's subkey is u64::MAX. + let (_, tail_key, _) = reader + .traverse_dual::() + .unwrap() + .last_of_k1(&addr) + .unwrap() + .expect("tail shard must exist"); + assert_eq!(tail_key, u64::MAX, "tail shard must be at u64::MAX"); +} + +#[test] +#[serial] +fn storage_history_splits_on_oversized_input() { + let (_dir, db) = create_test_rw_db(); + let addr = Address::from_slice(&[0x2; 20]); + let slot = U256::from(0xCAFEu64); + let blocks = sparse_blocks(); + let first_half = BlockNumberList::new(blocks[..100].iter().copied()).unwrap(); + let second_half = BlockNumberList::new(blocks[100..].iter().copied()).unwrap(); + + // Write in two transactions. + { + let writer = db.writer().unwrap(); + writer.append_storage_history(&addr, &slot, &first_half).unwrap(); + writer.commit().unwrap(); + } + { + let writer = db.writer().unwrap(); + writer.append_storage_history(&addr, &slot, &second_half).unwrap(); + writer.commit().unwrap(); + } + + let reader = db.reader().unwrap(); + + // Logical: round-trip succeeds and returns the same blocks. + let recovered = reader.blocks_changed_storage(&addr, &slot).unwrap().unwrap(); + assert_eq!(recovered.iter().collect::>(), blocks); + + // Structural: count dup entries for this (addr, slot). + let count = reader + .traverse_dual::() + .unwrap() + .iter_k2(&addr) + .unwrap() + .filter(|r: &Result<(ShardedKey, _), _>| { + r.as_ref().is_ok_and(|(sk, _)| sk.key == slot) + }) + .count(); + assert!(count >= 2, "expected >=2 shards for (addr, slot), got {count}"); + + // Structural: the tail shard's subkey is ShardedKey(slot, u64::MAX). + let (_, tail_sk, _) = reader + .traverse_dual::() + .unwrap() + .last_of_k1(&addr) + .unwrap() + .expect("tail shard must exist"); + assert_eq!(tail_sk, ShardedKey::new(slot, u64::MAX)); +} From d6887c06dee4cac91b34aa312eaeeb9bd16f84ef Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 08:40:13 -0400 Subject: [PATCH 20/21] chore(hot): final cleanup from delamination review - Wire the six new history conformance entrypoints into conformance() so they run against both MemKv and MDBX. Closes coverage gap on multi-batch appends + standalone truncates. The two update_history_indices tests use non-overlapping block windows (1-5 vs 1001-1005) to avoid re-indexing shared change-set entries on the shared conformance store. - Add empirical-size comment to worst_case_dense_pack_fits_in_dupsort_budget explaining the 650 vs 750 adjustment. - Drop a "shard key" terminology leak from a comment in db/history.rs. - Update setup_history_kv docstring in revm.rs to describe logical state rather than shard layout. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot/src/conformance/history.rs | 57 +++++++++++++++------------ crates/hot/src/conformance/mod.rs | 6 +++ crates/hot/src/db/history.rs | 2 +- crates/hot/src/model/revm.rs | 4 +- crates/types/src/int_list.rs | 1 + 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/crates/hot/src/conformance/history.rs b/crates/hot/src/conformance/history.rs index 938cf70..646f146 100644 --- a/crates/hot/src/conformance/history.rs +++ b/crates/hot/src/conformance/history.rs @@ -117,27 +117,30 @@ where let slot1 = U256::from(1); let slot2 = U256::from(2); - // Phase 1: Write storage change sets for blocks 1-3 + // Phase 1: Write storage change sets for blocks 1001-1003 + // (distinct from blocks 1-5 used by test_update_history_indices_account; + // update_history_indices scans the shared change-set table, so each test + // that calls it must operate in a non-overlapping block window) { let writer = hot_kv.writer().unwrap(); - // Block 1: addr1.slot1 changed - writer.write_storage_prestate(1, addr1, &slot1, &U256::ZERO).unwrap(); + // Block 1001: addr1.slot1 changed + writer.write_storage_prestate(1001, addr1, &slot1, &U256::ZERO).unwrap(); - // Block 2: addr1.slot1 and addr1.slot2 changed - writer.write_storage_prestate(2, addr1, &slot1, &U256::from(100)).unwrap(); - writer.write_storage_prestate(2, addr1, &slot2, &U256::ZERO).unwrap(); + // Block 1002: addr1.slot1 and addr1.slot2 changed + writer.write_storage_prestate(1002, addr1, &slot1, &U256::from(100)).unwrap(); + writer.write_storage_prestate(1002, addr1, &slot2, &U256::ZERO).unwrap(); - // Block 3: addr1.slot2 changed - writer.write_storage_prestate(3, addr1, &slot2, &U256::from(200)).unwrap(); + // Block 1003: addr1.slot2 changed + writer.write_storage_prestate(1003, addr1, &slot2, &U256::from(200)).unwrap(); writer.commit().unwrap(); } - // Phase 2: Run update_history_indices for blocks 1-3 + // Phase 2: Run update_history_indices for blocks 1001-1003 { let writer = hot_kv.writer().unwrap(); - writer.update_history_indices(1..=3).unwrap(); + writer.update_history_indices(1001..=1003).unwrap(); writer.commit().unwrap(); } @@ -145,40 +148,40 @@ where { let reader = hot_kv.reader().unwrap(); - // addr1.slot1 should have history at blocks 1, 2 + // addr1.slot1 should have history at blocks 1001, 1002 let history1 = reader .blocks_changed_storage(&addr1, &slot1) .unwrap() .expect("addr1.slot1 should have history"); let blocks1: Vec = history1.iter().collect(); - assert_eq!(blocks1, vec![1, 2], "addr1.slot1 history mismatch"); + assert_eq!(blocks1, vec![1001, 1002], "addr1.slot1 history mismatch"); - // addr1.slot2 should have history at blocks 2, 3 + // addr1.slot2 should have history at blocks 1002, 1003 let history2 = reader .blocks_changed_storage(&addr1, &slot2) .unwrap() .expect("addr1.slot2 should have history"); let blocks2: Vec = history2.iter().collect(); - assert_eq!(blocks2, vec![2, 3], "addr1.slot2 history mismatch"); + assert_eq!(blocks2, vec![1002, 1003], "addr1.slot2 history mismatch"); } - // Phase 4: Write more change sets for blocks 4-5 + // Phase 4: Write more change sets for blocks 1004-1005 { let writer = hot_kv.writer().unwrap(); - // Block 4: addr1.slot1 changed - writer.write_storage_prestate(4, addr1, &slot1, &U256::from(300)).unwrap(); + // Block 1004: addr1.slot1 changed + writer.write_storage_prestate(1004, addr1, &slot1, &U256::from(300)).unwrap(); - // Block 5: addr1.slot1 changed again - writer.write_storage_prestate(5, addr1, &slot1, &U256::from(400)).unwrap(); + // Block 1005: addr1.slot1 changed again + writer.write_storage_prestate(1005, addr1, &slot1, &U256::from(400)).unwrap(); writer.commit().unwrap(); } - // Phase 5: Run update_history_indices for blocks 4-5 + // Phase 5: Run update_history_indices for blocks 1004-1005 { let writer = hot_kv.writer().unwrap(); - writer.update_history_indices(4..=5).unwrap(); + writer.update_history_indices(1004..=1005).unwrap(); writer.commit().unwrap(); } @@ -186,21 +189,25 @@ where { let reader = hot_kv.reader().unwrap(); - // addr1.slot1 should now have history at blocks 1, 2, 4, 5 + // addr1.slot1 should now have history at blocks 1001, 1002, 1004, 1005 let history1 = reader .blocks_changed_storage(&addr1, &slot1) .unwrap() .expect("addr1.slot1 should have history"); let blocks1: Vec = history1.iter().collect(); - assert_eq!(blocks1, vec![1, 2, 4, 5], "addr1.slot1 history mismatch after append"); + assert_eq!( + blocks1, + vec![1001, 1002, 1004, 1005], + "addr1.slot1 history mismatch after append" + ); - // addr1.slot2 should still have history at blocks 2, 3 (unchanged) + // addr1.slot2 should still have history at blocks 1002, 1003 (unchanged) let history2 = reader .blocks_changed_storage(&addr1, &slot2) .unwrap() .expect("addr1.slot2 should have history"); let blocks2: Vec = history2.iter().collect(); - assert_eq!(blocks2, vec![2, 3], "addr1.slot2 history should be unchanged"); + assert_eq!(blocks2, vec![1002, 1003], "addr1.slot2 history should be unchanged"); } } diff --git a/crates/hot/src/conformance/mod.rs b/crates/hot/src/conformance/mod.rs index 96e609d..779661c 100644 --- a/crates/hot/src/conformance/mod.rs +++ b/crates/hot/src/conformance/mod.rs @@ -40,4 +40,10 @@ where test_cursor_iter_from(hot_kv); test_cursor_dual_iter(hot_kv); test_cursor_dual_iter_from(hot_kv); + test_update_history_indices_account(hot_kv); + test_update_history_indices_storage(hot_kv); + test_history_append_removes_old_entries(hot_kv); + test_delete_dual_account_history(hot_kv); + test_delete_dual_storage_history(hot_kv); + test_delete_and_rewrite_dual(hot_kv); } diff --git a/crates/hot/src/db/history.rs b/crates/hot/src/db/history.rs index 6a873ca..77aa1bf 100644 --- a/crates/hot/src/db/history.rs +++ b/crates/hot/src/db/history.rs @@ -49,7 +49,7 @@ pub trait HistoryRead: HotDbRead { let Some(first) = iter.next().transpose()? else { return Ok(None); }; - // first is (u64, BlockNumberList) — the shard key and its list + // first is (k2, list) — k2 is the opaque dup subkey, internal to the backend let (_, mut merged) = first; for entry in iter { let (_, list) = entry?; diff --git a/crates/hot/src/model/revm.rs b/crates/hot/src/model/revm.rs index 5b19d91..29e449a 100644 --- a/crates/hot/src/model/revm.rs +++ b/crates/hot/src/model/revm.rs @@ -805,13 +805,13 @@ mod tests { /// - Block 5 changed account: pre-state was nonce=1, balance=100 /// - Block 10 changed account: pre-state was nonce=5, balance=500 /// - Current (PlainAccountState): nonce=10, balance=1000 - /// - History shard: (A, 10) → [5, 10] + /// - Account history: blocks [5, 10] for address A /// /// Storage slot 0x42 for address A: /// - Block 5 changed slot: pre-state was 0 /// - Block 10 changed slot: pre-state was 100 /// - Current (PlainStorageState): 200 - /// - History shard: (A, ShardedKey(0x42, 10)) → [5, 10] + /// - Storage history: blocks [5, 10] for (address A, slot 0x42) fn setup_history_kv() -> (MemKv, Address) { let mem_kv = MemKv::default(); let address = Address::from_slice(&[0x1; 20]); diff --git a/crates/types/src/int_list.rs b/crates/types/src/int_list.rs index 28eaa21..0b189c4 100644 --- a/crates/types/src/int_list.rs +++ b/crates/types/src/int_list.rs @@ -336,6 +336,7 @@ mod tests { /// hundreds of contiguous blocks, we stay comfortably under 1500 B. #[test] fn worst_case_dense_pack_fits_in_dupsort_budget() { + // 650 contiguous values empirically encode to ~1300 B; 750 was over budget. let list = IntegerList::new(0u64..650).unwrap(); let size = list.serialized_size(); assert!(size <= 1500, "dense pack of 650 blocks encoded as {size} B, expected <= 1500"); From 68116bc94762c35a66320f75666685d83aae74b0 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 22 May 2026 12:43:10 -0400 Subject: [PATCH 21/21] fix(hot): address PR review comments - merge_and_split moves from a free fn to IntegerList::merge_and_split, taking an iterator for additions (no materialization). Drops the .clone() at the MDBX call site. - HotKv::RwTx now bounds HistoryWrite, so UnifiedStorage and the conformance generics no longer need explicit `where H::RwTx: HistoryWrite` clauses. unified.rs reverts to a single impl block. - Module-level rustdoc in db/history.rs drops the historical "replaces" framing; trait-level rustdoc on HistoryRead drops the architecture paragraph (override prevention is a property of the blanket impl, not user-visible behavior). Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/hot-mdbx/src/tx.rs | 7 +- crates/hot/src/conformance/history.rs | 30 ++----- crates/hot/src/conformance/mod.rs | 7 +- crates/hot/src/conformance/range.rs | 15 +--- crates/hot/src/conformance/roundtrip.rs | 20 +---- crates/hot/src/conformance/unwind.rs | 5 +- crates/hot/src/db/history.rs | 22 ++--- crates/hot/src/model/traits.rs | 2 +- crates/storage/src/unified.rs | 37 ++++---- crates/types/src/int_list.rs | 112 ++++++++++-------------- crates/types/src/lib.rs | 2 +- 11 files changed, 90 insertions(+), 169 deletions(-) diff --git a/crates/hot-mdbx/src/tx.rs b/crates/hot-mdbx/src/tx.rs index c16d996..e076363 100644 --- a/crates/hot-mdbx/src/tx.rs +++ b/crates/hot-mdbx/src/tx.rs @@ -405,7 +405,6 @@ macro_rules! impl_history_write { new_blocks: &signet_storage_types::BlockNumberList, ) -> Result<(), signet_hot::db::HistoryError> { use signet_hot::{db::HistoryError, model::HotKvWrite, tables}; - use signet_storage_types::merge_and_split; let existing_tail = self .get_dual::(addr, &u64::MAX) @@ -418,7 +417,7 @@ macro_rules! impl_history_write { } let (first, second) = - merge_and_split(existing_tail, new_blocks.clone(), MAX_SHARD_BYTES); + existing_tail.merge_and_split(new_blocks.iter(), MAX_SHARD_BYTES); match second { None => self @@ -441,7 +440,7 @@ macro_rules! impl_history_write { new_blocks: &signet_storage_types::BlockNumberList, ) -> Result<(), signet_hot::db::HistoryError> { use signet_hot::{db::HistoryError, model::HotKvWrite, tables}; - use signet_storage_types::{ShardedKey, merge_and_split}; + use signet_storage_types::ShardedKey; let tail_key = ShardedKey::new(*slot, u64::MAX); let existing_tail = self @@ -455,7 +454,7 @@ macro_rules! impl_history_write { } let (first, second) = - merge_and_split(existing_tail, new_blocks.clone(), MAX_SHARD_BYTES); + existing_tail.merge_and_split(new_blocks.iter(), MAX_SHARD_BYTES); match second { None => self diff --git a/crates/hot/src/conformance/history.rs b/crates/hot/src/conformance/history.rs index 646f146..7a013f4 100644 --- a/crates/hot/src/conformance/history.rs +++ b/crates/hot/src/conformance/history.rs @@ -12,10 +12,7 @@ use signet_storage_types::{Account, BlockNumberList}; /// This test verifies that: /// 1. Account change sets are correctly indexed into account history /// 2. Appending to existing history works correctly -pub fn test_update_history_indices_account(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_update_history_indices_account(hot_kv: &T) { let addr1 = address!("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); let addr2 = address!("0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); @@ -109,10 +106,7 @@ where /// 1. Storage change sets are correctly indexed into storage history /// 2. Appending to existing history works correctly /// 3. Different slots for the same address are tracked separately -pub fn test_update_history_indices_storage(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_update_history_indices_storage(hot_kv: &T) { let addr1 = address!("0xcccccccccccccccccccccccccccccccccccccccc"); let slot1 = U256::from(1); let slot2 = U256::from(2); @@ -216,10 +210,7 @@ where /// This test verifies that after appending an initial list and then a new /// block via `update_history_indices`, `blocks_changed_account` returns the /// expected union of all blocks. -pub fn test_history_append_removes_old_entries(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_history_append_removes_old_entries(hot_kv: &T) { let addr = address!("0xdddddddddddddddddddddddddddddddddddddddd"); // Phase 1: Append account history for blocks 10, 20, 30 @@ -269,10 +260,7 @@ where /// 1. Appending two disjoint sets of blocks for the same address works /// 2. `truncate_account_history_above` removes blocks above the cutoff /// 3. Other addresses are not affected -pub fn test_delete_dual_account_history(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_delete_dual_account_history(hot_kv: &T) { let addr1 = address!("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"); let addr2 = address!("0xffffffffffffffffffffffffffffffffffffffff"); @@ -335,10 +323,7 @@ where /// 1. Appending storage history for two slots works correctly /// 2. `truncate_storage_history_above(addr, slot1, 0)` removes all blocks for slot1 /// 3. Other slots for the same address are not affected -pub fn test_delete_dual_storage_history(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_delete_dual_storage_history(hot_kv: &T) { let addr = address!("0x1111111111111111111111111111111111111111"); let slot1 = U256::from(100); let slot2 = U256::from(200); @@ -400,10 +385,7 @@ where /// /// This test verifies that after truncating all history for an address, we /// can append new blocks and read them back correctly. -pub fn test_delete_and_rewrite_dual(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_delete_and_rewrite_dual(hot_kv: &T) { let addr = address!("0x2222222222222222222222222222222222222222"); // Phase 1: Append initial history [1, 2, 3] diff --git a/crates/hot/src/conformance/mod.rs b/crates/hot/src/conformance/mod.rs index 779661c..fb0e9b3 100644 --- a/crates/hot/src/conformance/mod.rs +++ b/crates/hot/src/conformance/mod.rs @@ -12,17 +12,14 @@ pub use range::*; pub use roundtrip::*; pub use unwind::*; -use crate::{db::HistoryWrite, model::HotKv}; +use crate::model::HotKv; /// Run all conformance tests against a [`HotKv`] implementation. /// /// Tests share the provided store instance. Additional test functions /// (cursor, edge-case, history, range) are exported for use in isolation /// with a fresh store. -pub fn conformance(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn conformance(hot_kv: &T) { test_header_roundtrip(hot_kv); test_account_roundtrip(hot_kv); test_storage_roundtrip(hot_kv); diff --git a/crates/hot/src/conformance/range.rs b/crates/hot/src/conformance/range.rs index ccc07d1..4870e41 100644 --- a/crates/hot/src/conformance/range.rs +++ b/crates/hot/src/conformance/range.rs @@ -235,10 +235,7 @@ pub fn test_take_range(hot_kv: &T) { /// 1. All k2 entries for k1 values within the range are deleted /// 2. k1 values outside the range remain intact /// 3. Edge cases work correctly -pub fn test_clear_range_dual(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_clear_range_dual(hot_kv: &T) { let addr1 = address!("0x1000000000000000000000000000000000000001"); let addr2 = address!("0x2000000000000000000000000000000000000002"); let addr3 = address!("0x3000000000000000000000000000000000000003"); @@ -326,10 +323,7 @@ where /// Test take_range_dual on a dual-keyed table. /// /// Similar to clear_range_dual but also returns the removed (k1, k2) pairs. -pub fn test_take_range_dual(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_take_range_dual(hot_kv: &T) { let addr1 = address!("0xa000000000000000000000000000000000000001"); let addr2 = address!("0xb000000000000000000000000000000000000002"); let addr3 = address!("0xc000000000000000000000000000000000000003"); @@ -417,10 +411,7 @@ where /// 1. Multiple storage slots can be written for an address /// 2. `write_changed_storage` with `wipe_storage: true` clears all slots /// 3. After wipe, all slots return None -pub fn test_write_changed_storage_wipe(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_write_changed_storage_wipe(hot_kv: &T) { let addr = address!("0x1111111111111111111111111111111111111111"); // Setup: write multiple storage slots for an address diff --git a/crates/hot/src/conformance/roundtrip.rs b/crates/hot/src/conformance/roundtrip.rs index 2dcc161..ff7c286 100644 --- a/crates/hot/src/conformance/roundtrip.rs +++ b/crates/hot/src/conformance/roundtrip.rs @@ -156,10 +156,7 @@ pub fn test_bytecode_roundtrip(hot_kv: &T) { } /// Test account history via HistoryWrite/HistoryRead -pub fn test_account_history(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_account_history(hot_kv: &T) { let addr = address!("0x1111111111111111111111111111111111111111"); let touched_blocks = BlockNumberList::new([10, 20, 30]).unwrap(); @@ -181,10 +178,7 @@ where } /// Test storage history via HistoryWrite/HistoryRead -pub fn test_storage_history(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_storage_history(hot_kv: &T) { let addr = address!("0x2222222222222222222222222222222222222222"); let slot = U256::from(42); let touched_blocks = BlockNumberList::new([5, 15, 25]).unwrap(); @@ -207,10 +201,7 @@ where } /// Test account change sets via HotHistoryWrite/HotHistoryRead -pub fn test_account_changes(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_account_changes(hot_kv: &T) { let addr = address!("0x3333333333333333333333333333333333333333"); let pre_state = Account { nonce: 10, balance: U256::from(5000), bytecode_hash: None }; let block_number = 100u64; @@ -236,10 +227,7 @@ where } /// Test storage change sets via HotHistoryWrite/HotHistoryRead -pub fn test_storage_changes(hot_kv: &T) -where - T::RwTx: HistoryWrite, -{ +pub fn test_storage_changes(hot_kv: &T) { let addr = address!("0x4444444444444444444444444444444444444444"); let slot = U256::from(153); let pre_value = U256::from(12345); diff --git a/crates/hot/src/conformance/unwind.rs b/crates/hot/src/conformance/unwind.rs index c91190c..3471c6d 100644 --- a/crates/hot/src/conformance/unwind.rs +++ b/crates/hot/src/conformance/unwind.rs @@ -228,10 +228,7 @@ pub fn make_account_info(nonce: u64, balance: U256, code_hash: Option) -> /// - Headers and header number mappings /// - Account and storage change sets /// - Account and storage history indices -pub fn test_unwind_conformance(store_a: &Kv, store_b: &Kv) -where - Kv::RwTx: HistoryWrite, -{ +pub fn test_unwind_conformance(store_a: &Kv, store_b: &Kv) { // Test addresses let addr1 = address!("0x1111111111111111111111111111111111111111"); let addr2 = address!("0x2222222222222222222222222222222222222222"); diff --git a/crates/hot/src/db/history.rs b/crates/hot/src/db/history.rs index 77aa1bf..34caf92 100644 --- a/crates/hot/src/db/history.rs +++ b/crates/hot/src/db/history.rs @@ -1,11 +1,10 @@ //! Logical history reads and writes. //! -//! These traits replace the shard-leaking surface in `db::read` and -//! `db::inconsistent`. [`HistoryRead`] is blanket-impled on [`HotKvRead`] and -//! cannot be overridden — the KV-table layout is mandated by the abstraction. -//! [`HistoryWrite`] is required per-backend; each backend chooses its -//! splitting policy (MDBX uses [`signet_storage_types::merge_and_split`]; -//! MemKv writes a single dup entry per addr). +//! [`HistoryRead`] is blanket-impled on [`HotKvRead`] and describes +//! logical history queries. [`HistoryWrite`] is required per-backend; each +//! backend chooses its splitting policy (MDBX uses +//! [`signet_storage_types::IntegerList::merge_and_split`] — see below; MemKv +//! writes a single dup entry per addr). use crate::{ db::{HistoryError, HotDbRead, UnsafeDbWrite}, @@ -33,11 +32,6 @@ use trevm::revm::{ const ADDRESS_MAX: Address = address!("0xffffffffffffffffffffffffffffffffffffffff"); /// Logical reads against history + changeset tables. -/// -/// Default-impl-only. Backends cannot override — the blanket impl below -/// occupies the slot, and orphan rules prevent downstream impls. This is -/// structural enforcement of "the KV-table access pattern is mandated by -/// the abstraction". pub trait HistoryRead: HotDbRead { /// All block numbers where `addr` was touched. `None` if no history. fn blocks_changed_account( @@ -287,11 +281,7 @@ pub trait HistoryRead: HotDbRead { impl HistoryRead for T where T: HotKvRead {} -/// Logical writes against history + changeset tables. Required per backend. -/// -/// Backends that implement this trait choose their own shard-splitting policy. -/// The default `update_history_indices` bulk operation is expressed in terms of -/// the four required primitives and works for any backend. +/// Logical writes against history + changeset tables. pub trait HistoryWrite: UnsafeDbWrite + HistoryRead { /// Merge `new_blocks` into `addr`'s account history. /// diff --git a/crates/hot/src/model/traits.rs b/crates/hot/src/model/traits.rs index 7d4ee1c..6c2f7d4 100644 --- a/crates/hot/src/model/traits.rs +++ b/crates/hot/src/model/traits.rs @@ -30,7 +30,7 @@ pub trait HotKv { type RoTx: HotKvRead; /// The read-write transaction type. - type RwTx: HotKvWrite; + type RwTx: HotKvWrite + crate::db::HistoryWrite; /// Create a read-only transaction. fn reader(&self) -> Result; diff --git a/crates/storage/src/unified.rs b/crates/storage/src/unified.rs index 5da8f52..b799658 100644 --- a/crates/storage/src/unified.rs +++ b/crates/storage/src/unified.rs @@ -176,28 +176,7 @@ impl UnifiedStorage { let cold_data: Vec<_> = blocks.into_iter().map(BlockData::from).collect(); self.cold.append_blocks(cold_data).await } -} - -impl UnifiedStorage { - /// Spawn a unified storage with a type-erased cold backend. - /// - /// Erases the concrete cold backend behind - /// [`signet_cold::ErasedBackend`], so callers can hold a - /// `UnifiedStorage` without propagating a backend generic. - pub fn spawn_erased( - hot: H, - cold_backend: B, - cancel_token: CancellationToken, - ) -> Self { - let cold = ColdStorage::new_erased(cold_backend, cancel_token); - Self::new(hot, cold) - } -} -impl UnifiedStorage -where - H::RwTx: HistoryWrite, -{ /// Append executed blocks to both hot and cold storage. /// /// This method: @@ -338,6 +317,22 @@ where } } +impl UnifiedStorage { + /// Spawn a unified storage with a type-erased cold backend. + /// + /// Erases the concrete cold backend behind + /// [`signet_cold::ErasedBackend`], so callers can hold a + /// `UnifiedStorage` without propagating a backend generic. + pub fn spawn_erased( + hot: H, + cold_backend: B, + cancel_token: CancellationToken, + ) -> Self { + let cold = ColdStorage::new_erased(cold_backend, cancel_token); + Self::new(hot, cold) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/types/src/int_list.rs b/crates/types/src/int_list.rs index 0b189c4..329317b 100644 --- a/crates/types/src/int_list.rs +++ b/crates/types/src/int_list.rs @@ -172,66 +172,51 @@ impl IntegerList { pub fn serialize_into(&self, writer: W) -> std::io::Result<()> { self.0.serialize_into(writer) } -} -/// Append `additions` to `existing`, splitting off a tail shard iff the -/// merged list exceeds `max_bytes` after roaring encoding. -/// -/// Returns `(first, second)`: -/// - `first`: always returned; contains the lower block numbers. The caller -/// writes this with subkey = `first.max()` if `second.is_some()`, otherwise -/// `u64::MAX`. -/// - `second`: `Some(tail)` iff a split occurred. Caller writes with subkey -/// `u64::MAX`. -/// -/// Preconditions (caller's responsibility): -/// - Every block in `additions` is strictly greater than every block in -/// `existing`. The function does not sort or deduplicate. -/// - `existing.serialized_size() <= max_bytes`. -/// -/// Postconditions: -/// - `first.serialized_size() <= max_bytes`. -/// - If `second.is_some()`, `second.serialized_size() <= max_bytes` and -/// `second.min() > first.max()`. -/// - `first ∪ second == existing_before ∪ additions`. -/// -/// Allocation: zero on the no-split fast path beyond what -/// `IntegerList::push` already does for roaring container growth. One -/// `IntegerList` allocation when a split occurs. -pub fn merge_and_split( - mut existing: IntegerList, - additions: IntegerList, - max_bytes: usize, -) -> (IntegerList, Option) { - debug_assert!( - additions.serialized_size() <= max_bytes, - "additions exceed a single shard's budget", - ); - let mut tail: Option = None; - - for block in additions.iter() { - if let Some(t) = tail.as_mut() { - t.push(block).expect("strictly increasing"); - continue; - } - existing.push(block).expect("strictly increasing"); - if existing.serialized_size() > max_bytes { - // Overflow: pop the just-pushed block out of `existing` and - // start the tail shard with it as the seed. - let popped = existing.pop_max().expect("just pushed"); - debug_assert_eq!(popped, block); - let mut t = IntegerList::empty(); - t.push(block).expect("first push always succeeds"); - tail = Some(t); + /// Append `additions` to `self`, splitting off a tail iff the merged + /// list exceeds `max_bytes` after roaring encoding. + /// + /// Returns `(first, second)`: + /// - `first`: the lower-block-number portion (always returned). + /// - `second`: `Some(tail)` iff a split occurred; contains the higher + /// block numbers. The caller is responsible for choosing subkeys + /// for the two shards (e.g., `first.max()` and `u64::MAX`). + /// + /// Every block yielded by `additions` must be strictly greater than + /// every block already in `self`. If not, the underlying + /// [`IntegerList::push`] will panic. + /// + /// Allocation: zero on the no-split fast path beyond roaring container + /// growth. One `IntegerList` allocation when a split occurs. + pub fn merge_and_split( + mut self, + additions: impl IntoIterator, + max_bytes: usize, + ) -> (Self, Option) { + let mut tail: Option = None; + + for block in additions { + if let Some(t) = tail.as_mut() { + t.push(block).expect("strictly increasing"); + continue; + } + self.push(block).expect("strictly increasing"); + if self.serialized_size() > max_bytes { + let popped = self.pop_max().expect("just pushed"); + debug_assert_eq!(popped, block); + let mut t = Self::empty(); + t.push(block).expect("first push always succeeds"); + tail = Some(t); + } } - } - (existing, tail) + (self, tail) + } } #[cfg(test)] mod tests { - use super::{IntegerList, merge_and_split}; + use super::IntegerList; #[test] fn pop_max_returns_and_removes_largest() { @@ -259,9 +244,8 @@ mod tests { #[test] fn merge_and_split_no_split_when_under_budget() { let existing = IntegerList::new([1u64, 2, 3]).unwrap(); - let additions = IntegerList::new([4u64, 5]).unwrap(); // 1500 B is generous for 5 dense values - let (first, second) = merge_and_split(existing, additions, 1500); + let (first, second) = existing.merge_and_split([4u64, 5], 1500); assert_eq!(first.iter().collect::>(), vec![1, 2, 3, 4, 5]); assert!(second.is_none()); } @@ -269,8 +253,7 @@ mod tests { #[test] fn merge_and_split_no_additions_returns_existing() { let existing = IntegerList::new([10u64, 20]).unwrap(); - let additions = IntegerList::empty(); - let (first, second) = merge_and_split(existing, additions, 1500); + let (first, second) = existing.merge_and_split(std::iter::empty(), 1500); assert_eq!(first.iter().collect::>(), vec![10, 20]); assert!(second.is_none()); } @@ -282,7 +265,6 @@ mod tests { // around 14 B, so we need a tiny budget to force a split. Set budget // small enough that ~50 entries push us over. let existing = IntegerList::new(0u64..50).unwrap(); - let additions = IntegerList::new(50u64..100).unwrap(); // Compute the budget so existing alone fits but existing + additions // doesn't. @@ -291,7 +273,7 @@ mod tests { assert!(combined > existing_size, "test setup broken: combined didn't grow"); let budget = existing_size + (combined - existing_size) / 2; - let (first, second) = merge_and_split(existing, additions, budget); + let (first, second) = existing.merge_and_split(50u64..100, budget); let second = second.expect("split should have occurred"); // first ∪ second == 0..100, with second's min > first's max. @@ -315,15 +297,15 @@ mod tests { let combined_blocks: Vec = (0..70u64).map(|i| i * 0x1_0000).collect(); let existing = IntegerList::new(existing_blocks).unwrap(); - let additions = IntegerList::new(addition_blocks).unwrap(); + let additions_size = + IntegerList::new(addition_blocks.iter().copied()).unwrap().serialized_size(); let combined_size = IntegerList::new(combined_blocks).unwrap().serialized_size(); - let additions_size = additions.serialized_size(); let existing_size = existing.serialized_size(); // Budget that fits both existing and additions alone, but not combined. let budget = existing_size.max(additions_size) + 16; assert!(combined_size > budget, "test setup broken: combined fits in budget"); - let (first, second) = merge_and_split(existing, additions, budget); + let (first, second) = existing.merge_and_split(addition_blocks, budget); let second = second.expect("split should have occurred"); let first_max = first.max().unwrap(); @@ -363,12 +345,12 @@ mod tests { let blocks: Vec = (0..200u64).map(|i| i * 0x1_0000).collect(); let half = blocks.len() / 2; let existing = IntegerList::new(blocks[..half].iter().copied()).unwrap(); - let additions = IntegerList::new(blocks[half..].iter().copied()).unwrap(); + let additions: Vec = blocks[half..].to_vec(); assert!(existing.serialized_size() <= 1500); - assert!(additions.serialized_size() <= 1500); + assert!(IntegerList::new(additions.iter().copied()).unwrap().serialized_size() <= 1500); - let (first, second) = merge_and_split(existing, additions, 1500); + let (first, second) = existing.merge_and_split(additions, 1500); assert!(first.serialized_size() <= 1500); if let Some(second) = &second { assert!(second.serialized_size() <= 1500); diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 0f6a128..4924da9 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -23,7 +23,7 @@ pub use events::{DbSignetEvent, DbZenithHeader}; mod indexed_receipt; pub use indexed_receipt::IndexedReceipt; mod int_list; -pub use int_list::{BlockNumberList, IntegerList, IntegerListError, merge_and_split}; +pub use int_list::{BlockNumberList, IntegerList, IntegerListError}; mod sharded; pub use sharded::ShardedKey; pub use signet_evm::{Account, EthereumHardfork, genesis_header};