From 6acb2436698a5dab925ac785c49697edc32f722d Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sat, 27 Jun 2026 15:37:04 +0530 Subject: [PATCH 01/20] feat: init database crate --- lean_client/Cargo.toml | 1 + lean_client/database/Cargo.toml | 5 +++++ lean_client/database/src/lib.rs | 14 ++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 lean_client/database/Cargo.toml create mode 100644 lean_client/database/src/lib.rs diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 5a394eac..7d7c660d 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -2,6 +2,7 @@ members = [ "chain", "containers", + "database", "env-config", "fork_choice", "http_api", diff --git a/lean_client/database/Cargo.toml b/lean_client/database/Cargo.toml new file mode 100644 index 00000000..64395ced --- /dev/null +++ b/lean_client/database/Cargo.toml @@ -0,0 +1,5 @@ +[package] +name = "database" +edition = { workspace = true } + +[dependencies] \ No newline at end of file diff --git a/lean_client/database/src/lib.rs b/lean_client/database/src/lib.rs new file mode 100644 index 00000000..b93cf3ff --- /dev/null +++ b/lean_client/database/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} From 19c6afd4d79086c6f484b57feabb1c78d0c8716a Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sun, 28 Jun 2026 01:56:59 +0530 Subject: [PATCH 02/20] chore: pull database as a grandine dep and create storage crate --- lean_client/Cargo.lock | 156 +++++++++++++++++-- lean_client/Cargo.toml | 7 +- lean_client/database/Cargo.toml | 5 - lean_client/storage/Cargo.toml | 9 ++ lean_client/{database => storage}/src/lib.rs | 0 5 files changed, 155 insertions(+), 22 deletions(-) delete mode 100644 lean_client/database/Cargo.toml create mode 100644 lean_client/storage/Cargo.toml rename lean_client/{database => storage}/src/lib.rs (100%) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 8f507b99..82f91643 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -68,7 +68,7 @@ dependencies = [ "rand 0.9.4", "rapidhash", "ruint", - "rustc-hash", + "rustc-hash 2.1.2", "serde", "sha3 0.10.8", ] @@ -627,6 +627,24 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.10.5", + "proc-macro2 1.0.106", + "quote 1.0.45", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.117", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -798,16 +816,33 @@ dependencies = [ "serde", ] +[[package]] +name = "bytesize" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d" +dependencies = [ + "serde_core", +] + [[package]] name = "cc" -version = "1.2.60" +version = "1.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" dependencies = [ - "find-msvc-tools", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -882,6 +917,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.6.0" @@ -1297,6 +1343,27 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "database" +version = "0.0.0" +source = "git+https://github.com/grandinetech/grandine?rev=64afdee3c6be79fceffb66933dcb69a943f3f1ae#64afdee3c6be79fceffb66933dcb69a943f3f1ae" +dependencies = [ + "anyhow", + "bytesize", + "fs-err", + "futures", + "im", + "itertools 0.14.0", + "logging", + "reth-libmdbx", + "snap", + "std_ext", + "tap", + "thiserror 2.0.18", + "tracing", + "unwrap_none", +] + [[package]] name = "dedicated_executor" version = "0.1.0" @@ -1832,12 +1899,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - [[package]] name = "fixed-hash" version = "0.8.0" @@ -1899,6 +1960,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "funty" version = "2.0.0" @@ -2486,7 +2557,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2949,6 +3020,7 @@ dependencies = [ "parking_lot", "reqwest", "ssz", + "storage", "tikv-jemallocator", "tokio", "tracing", @@ -3077,6 +3149,16 @@ version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -4650,9 +4732,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -4670,7 +4752,7 @@ dependencies = [ "lru-slab", "rand 0.9.4", "ring", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "rustls-pki-types", "slab", @@ -4689,7 +4771,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.52.0", ] @@ -5016,6 +5098,31 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" +[[package]] +name = "reth-libmdbx" +version = "1.3.12" +source = "git+https://github.com/paradigmxyz/reth.git?rev=6f8e7258f4733279080e4bd8345ce50538a40d6e#6f8e7258f4733279080e4bd8345ce50538a40d6e" +dependencies = [ + "bitflags", + "byteorder", + "derive_more", + "indexmap 2.14.0", + "parking_lot", + "reth-mdbx-sys", + "smallvec", + "thiserror 2.0.18", + "tracing", +] + +[[package]] +name = "reth-mdbx-sys" +version = "1.3.12" +source = "git+https://github.com/paradigmxyz/reth.git?rev=6f8e7258f4733279080e4bd8345ce50538a40d6e#6f8e7258f4733279080e4bd8345ce50538a40d6e" +dependencies = [ + "bindgen", + "cc", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -5131,6 +5238,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -5722,6 +5835,13 @@ dependencies = [ "triomphe", ] +[[package]] +name = "storage" +version = "0.0.0" +dependencies = [ + "database", +] + [[package]] name = "strength_reduce" version = "0.2.4" @@ -6441,6 +6561,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unwrap_none" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "461d0c5956fcc728ecc03a3a961e4adc9a7975d86f6f8371389a289517c02ca9" + [[package]] name = "url" version = "2.5.8" diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 7d7c660d..4bd5bb8f 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -2,13 +2,13 @@ members = [ "chain", "containers", - "database", "env-config", "fork_choice", "http_api", "metrics", "networking", - "spec_test_fixtures", + "spec_test_fixtures", + "storage", "validator", "xmss", ] @@ -230,6 +230,7 @@ http_api = { path = "./http_api" } metrics = { path = "./metrics" } networking = { path = "./networking" } spec_test_fixtures = { path = "./spec_test_fixtures" } +storage = { path = "./storage" } validator = { path = "./validator" } xmss = { path = "./xmss" } @@ -239,6 +240,7 @@ axum = "0.8.8" bitvec = "1.0.1" bls = { git = "https://github.com/grandinetech/grandine", package = "bls", features = ["blst"], rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } clap = { version = "4", features = ["derive"] } +database = { git = "https://github.com/grandinetech/grandine", package = "database", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } derive_more = "2.1.1" discv5 = "0.10.2" enr = { version = "0.13", features = ["k256"] } @@ -331,6 +333,7 @@ networking = { workspace = true } num_cpus = { workspace = true } parking_lot = { workspace = true } ssz = { workspace = true } +storage = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/lean_client/database/Cargo.toml b/lean_client/database/Cargo.toml deleted file mode 100644 index 64395ced..00000000 --- a/lean_client/database/Cargo.toml +++ /dev/null @@ -1,5 +0,0 @@ -[package] -name = "database" -edition = { workspace = true } - -[dependencies] \ No newline at end of file diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml new file mode 100644 index 00000000..7b623e2f --- /dev/null +++ b/lean_client/storage/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "storage" +edition = { workspace = true } + +[dependencies] +database = { workspace = true } + +[lints] +workspace = true diff --git a/lean_client/database/src/lib.rs b/lean_client/storage/src/lib.rs similarity index 100% rename from lean_client/database/src/lib.rs rename to lean_client/storage/src/lib.rs From 57d0438bca945c0e1702f8fe9a1ac38e3d0b7a90 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sun, 28 Jun 2026 04:42:50 +0530 Subject: [PATCH 03/20] feat: add leanspec consts for db --- lean_client/storage/src/lib.rs | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index b93cf3ff..2dd60d57 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,14 +1,24 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +use database::{Database, DatabaseMode, RestartMessage}; + +const BLOCKS_TABLE_NAME: &str = "blocks"; + +const BLOCKS_CREATE_INDEX: &str = "slots_to_block_roots_index"; + +const STATES_TABLE_NAME: &str = "states"; + +const STATES_CREATE_INDEX: &str = "slots_to_state_roots_index"; + +const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; + +const CHECKPOINTS_KEY_JUSTIFIED: &str = "justified"; + +const CHECKPOINTS_KEY_FINALIZED: &str = "finalized"; + +const CHECKPOINTS_KEY_HEAD: &str = "head"; + +const CHECKPOINTS_KEY_GENESIS_TIME: &str = "genesis_time"; + +const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; + +const STATE_ROOT_INDEX_TABLE_NAME: &str = "state_root_index"; + From e0233e05e60fb25bac70a2125dd16d84b5c90419 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sun, 28 Jun 2026 16:16:22 +0530 Subject: [PATCH 04/20] feat: add databases for each KV pair defined in spec --- lean_client/storage/Cargo.toml | 2 + lean_client/storage/src/lib.rs | 102 ++++++++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml index 7b623e2f..2632a54a 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/storage/Cargo.toml @@ -4,6 +4,8 @@ edition = { workspace = true } [dependencies] database = { workspace = true } +bytesize = { workspace = true } +anyhow = { workspace = true } [lints] workspace = true diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 2dd60d57..e08f96b1 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,24 +1,96 @@ -use database::{Database, DatabaseMode, RestartMessage}; +use std::path::Path; -const BLOCKS_TABLE_NAME: &str = "blocks"; +use anyhow::Result; +use bytesize::ByteSize; +use database::{Database, DatabaseMode}; -const BLOCKS_CREATE_INDEX: &str = "slots_to_block_roots_index"; +pub struct Storage { + pub blocks: Blocks, + pub states: States, + pub checkpoints: Checkpoints, + pub slot_index: SlotIndex, + pub state_root_index: StateRootIndex, +} -const STATES_TABLE_NAME: &str = "states"; +impl Storage { + pub fn new() -> Result { + Ok(Self { + blocks: Blocks::new()?, + states: States::new()?, + checkpoints: Checkpoints::new()?, + slot_index: SlotIndex::new()?, + state_root_index: StateRootIndex::new()?, + }) + } +} -const STATES_CREATE_INDEX: &str = "slots_to_state_roots_index"; +pub struct Blocks(Database); +pub struct States(Database); +pub struct Checkpoints(Database); +pub struct SlotIndex(Database); +pub struct StateRootIndex(Database); -const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; +impl Blocks { + pub fn new() -> Result { + let db = Database::persistent( + "blocks", + Path::new("./database/blocks"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} -const CHECKPOINTS_KEY_JUSTIFIED: &str = "justified"; +impl States { + pub fn new() -> Result { + let db = Database::persistent( + "states", + Path::new("./database/states"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} -const CHECKPOINTS_KEY_FINALIZED: &str = "finalized"; +impl Checkpoints { + pub fn new() -> Result { + let db = Database::persistent( + "checkpoints", + Path::new("./database/checkpoints"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} -const CHECKPOINTS_KEY_HEAD: &str = "head"; - -const CHECKPOINTS_KEY_GENESIS_TIME: &str = "genesis_time"; - -const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; - -const STATE_ROOT_INDEX_TABLE_NAME: &str = "state_root_index"; +impl SlotIndex { + pub fn new() -> Result { + let db = Database::persistent( + "slot_index", + Path::new("./database/slot_index"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} +impl StateRootIndex { + pub fn new() -> Result { + let db = Database::persistent( + "state_root_index", + Path::new("./database/state_root_index"), + ByteSize::gib(2), + DatabaseMode::ReadWrite, + None, + )?; + Ok(Self(db)) + } +} From 4376cdd657165a0bea0e541ffe592e9dbb18ad5b Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Tue, 30 Jun 2026 04:00:48 +0530 Subject: [PATCH 05/20] feat: add Storage impl with builderplate functions from spec --- lean_client/Cargo.lock | 4 ++ lean_client/Cargo.toml | 1 + lean_client/storage/Cargo.toml | 2 + lean_client/storage/src/lib.rs | 113 ++++++++++++++++++++++++++++----- 4 files changed, 105 insertions(+), 15 deletions(-) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 82f91643..7c16bf00 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -5839,7 +5839,11 @@ dependencies = [ name = "storage" version = "0.0.0" dependencies = [ + "anyhow", + "bytesize", + "containers", "database", + "ssz", ] [[package]] diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 4bd5bb8f..f154849b 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -239,6 +239,7 @@ async-trait = "0.1" axum = "0.8.8" bitvec = "1.0.1" bls = { git = "https://github.com/grandinetech/grandine", package = "bls", features = ["blst"], rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } +bytesize = { version = '2', features = ['serde'] } clap = { version = "4", features = ["derive"] } database = { git = "https://github.com/grandinetech/grandine", package = "database", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } derive_more = "2.1.1" diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml index 2632a54a..337bde2d 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/storage/Cargo.toml @@ -6,6 +6,8 @@ edition = { workspace = true } database = { workspace = true } bytesize = { workspace = true } anyhow = { workspace = true } +containers = { workspace = true } +ssz = { workspace = true } [lints] workspace = true diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index e08f96b1..0de5526b 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -4,12 +4,15 @@ use anyhow::Result; use bytesize::ByteSize; use database::{Database, DatabaseMode}; +use containers::{Block, State, Checkpoint, Slot}; +use ssz::H256; + pub struct Storage { - pub blocks: Blocks, - pub states: States, - pub checkpoints: Checkpoints, - pub slot_index: SlotIndex, - pub state_root_index: StateRootIndex, + blocks: Blocks, + states: States, + checkpoints: Checkpoints, + slot_index: SlotIndex, + state_root_index: StateRootIndex, } impl Storage { @@ -22,16 +25,96 @@ impl Storage { state_root_index: StateRootIndex::new()?, }) } + + pub fn get_block(self, root: H256) -> Result> { + let blocks_db = self.blocks; + + Ok(None) + } + + pub fn put_block(self, block: Block, root: H256) { + let blocks_db = self.blocks; + } + + pub fn get_state(self, root: H256) -> Result> { + let states_db = self.states; + + Ok(None) + } + + pub fn put_state(self, state: State, root: H256) { + let states_db = self.states; + } + + pub fn get_justified_checkpoint(self) -> Result> { + let checkpoints_db = self.checkpoints; + + Ok(None) + } + + pub fn put_justified_checkpoint(self, checkpoint: Checkpoint) { + let checkpoints_db = self.checkpoints; + } + + pub fn get_finalized_checkpoint(self) -> Result> { + let checkpoints_db = self.checkpoints; + + Ok(None) + } + + pub fn put_finalized_checkpoint(self, checkpoint: Checkpoint) { + let checkpoints_db = self.checkpoints; + } + + pub fn get_head_root(self) -> Result> { + let checkpoints_db = self.checkpoints; + + Ok(None) + } + + pub fn put_head_root(self, root: H256) { + let checkpoints_db = self.checkpoints; + } + + pub fn get_block_root_by_slot(self, slot: Slot) -> Result> { + let slot_index_db = self.slot_index; + + Ok(None) + } + + pub fn put_block_root_by_slot(self, slot: Slot, root: H256) { + let slot_index_db = self.slot_index; + } + + pub fn get_block_root_by_state_root(self, state_root: H256) -> Result> { + let state_root_index_db = self.state_root_index; + + Ok(None) + } + + pub fn put_block_root_by_state_root(self, state_root: H256, block_root: H256) { + let state_root_index_db = self.state_root_index; + } + + pub fn get_genesis_time(self) -> Result> { + let checkpoints_db = self.checkpoints; + + Ok(None) + } + + pub fn put_genesis_time(self, genesis_time: u64) { + let checkpoints_db = self.checkpoints; + } } -pub struct Blocks(Database); -pub struct States(Database); -pub struct Checkpoints(Database); -pub struct SlotIndex(Database); -pub struct StateRootIndex(Database); +struct Blocks(Database); +struct States(Database); +struct Checkpoints(Database); +struct SlotIndex(Database); +struct StateRootIndex(Database); impl Blocks { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "blocks", Path::new("./database/blocks"), @@ -44,7 +127,7 @@ impl Blocks { } impl States { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "states", Path::new("./database/states"), @@ -57,7 +140,7 @@ impl States { } impl Checkpoints { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "checkpoints", Path::new("./database/checkpoints"), @@ -70,7 +153,7 @@ impl Checkpoints { } impl SlotIndex { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "slot_index", Path::new("./database/slot_index"), @@ -83,7 +166,7 @@ impl SlotIndex { } impl StateRootIndex { - pub fn new() -> Result { + fn new() -> Result { let db = Database::persistent( "state_root_index", Path::new("./database/state_root_index"), From 2a25e9ea9ce94f9750bb46a926a90fe885c7e832 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Wed, 1 Jul 2026 03:55:24 +0530 Subject: [PATCH 06/20] feat: storage spec API functions --- lean_client/storage/src/lib.rs | 123 +++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 50 deletions(-) diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 0de5526b..e9518d0e 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -4,8 +4,8 @@ use anyhow::Result; use bytesize::ByteSize; use database::{Database, DatabaseMode}; -use containers::{Block, State, Checkpoint, Slot}; -use ssz::H256; +use containers::{Block, Checkpoint, Slot, State}; +use ssz::{H256, SszReadDefault as _, SszWrite as _}; pub struct Storage { blocks: Blocks, @@ -26,84 +26,107 @@ impl Storage { }) } - pub fn get_block(self, root: H256) -> Result> { - let blocks_db = self.blocks; - - Ok(None) + pub fn get_block(&self, root: H256) -> Result> { + match self.blocks.0.get(root)? { + Some(block_bytes) => Ok(Some(Block::from_ssz_default(&block_bytes)?)), + None => Ok(None), + } } - pub fn put_block(self, block: Block, root: H256) { - let blocks_db = self.blocks; + pub fn put_block(&self, block: Block, root: H256) -> Result<()> { + let block_bytes = block.to_ssz()?; + self.blocks.0.put(root, block_bytes)?; + Ok(()) } - pub fn get_state(self, root: H256) -> Result> { - let states_db = self.states; - - Ok(None) + pub fn get_state(&self, root: H256) -> Result> { + match self.states.0.get(root)? { + Some(state_bytes) => Ok(Some(State::from_ssz_default(&state_bytes)?)), + None => Ok(None), + } } - pub fn put_state(self, state: State, root: H256) { - let states_db = self.states; + pub fn put_state(&self, state: State, root: H256) -> Result<()> { + let state_bytes = state.to_ssz()?; + self.states.0.put(root, state_bytes)?; + Ok(()) } - pub fn get_justified_checkpoint(self) -> Result> { - let checkpoints_db = self.checkpoints; - - Ok(None) + pub fn get_justified_checkpoint(&self) -> Result> { + match self.checkpoints.0.get("justified")? { + Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), + None => Ok(None), + } } - pub fn put_justified_checkpoint(self, checkpoint: Checkpoint) { - let checkpoints_db = self.checkpoints; + pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { + let checkpoint_bytes = checkpoint.to_ssz()?; + self.checkpoints.0.put("justified", checkpoint_bytes)?; + Ok(()) } - pub fn get_finalized_checkpoint(self) -> Result> { - let checkpoints_db = self.checkpoints; - - Ok(None) + pub fn get_finalized_checkpoint(&self) -> Result> { + match self.checkpoints.0.get("finalized")? { + Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), + None => Ok(None), + } } - pub fn put_finalized_checkpoint(self, checkpoint: Checkpoint) { - let checkpoints_db = self.checkpoints; + pub fn put_finalized_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { + let checkpoint_bytes = checkpoint.to_ssz()?; + self.checkpoints.0.put("finalized", checkpoint_bytes)?; + Ok(()) } - pub fn get_head_root(self) -> Result> { - let checkpoints_db = self.checkpoints; - - Ok(None) + pub fn get_head_root(&self) -> Result> { + match self.checkpoints.0.get("head")? { + Some(head_root_bytes) => Ok(Some(H256::from_slice(&head_root_bytes))), + None => Ok(None), + } } - pub fn put_head_root(self, root: H256) { - let checkpoints_db = self.checkpoints; + pub fn put_head_root(&self, root: H256) -> Result<()> { + self.checkpoints.0.put("head", root)?; + Ok(()) } - pub fn get_block_root_by_slot(self, slot: Slot) -> Result> { - let slot_index_db = self.slot_index; - - Ok(None) + pub fn get_block_root_by_slot(&self, slot: Slot) -> Result> { + let slot_bytes = slot.0.to_be_bytes(); + match self.slot_index.0.get(slot_bytes)? { + Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), + None => Ok(None), + } } - pub fn put_block_root_by_slot(self, slot: Slot, root: H256) { - let slot_index_db = self.slot_index; + pub fn put_block_root_by_slot(&self, slot: Slot, root: H256) -> Result<()> { + let slot_bytes = slot.0.to_be_bytes(); + self.slot_index.0.put(slot_bytes, root)?; + Ok(()) } - pub fn get_block_root_by_state_root(self, state_root: H256) -> Result> { - let state_root_index_db = self.state_root_index; - - Ok(None) + pub fn get_block_root_by_state_root(&self, state_root: H256) -> Result> { + match self.state_root_index.0.get(state_root)? { + Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), + None => Ok(None), + } } - pub fn put_block_root_by_state_root(self, state_root: H256, block_root: H256) { - let state_root_index_db = self.state_root_index; + pub fn put_block_root_by_state_root(&self, state_root: H256, block_root: H256) -> Result<()> { + self.state_root_index.0.put(state_root, block_root)?; + Ok(()) } - pub fn get_genesis_time(self) -> Result> { - let checkpoints_db = self.checkpoints; - - Ok(None) + pub fn get_genesis_time(&self) -> Result> { + match self.checkpoints.0.get("genesis_time")? { + Some(genesis_time_bytes) => Ok(Some(u64::from_ssz_default(&genesis_time_bytes)?)), + None => Ok(None), + } } - pub fn put_genesis_time(self, genesis_time: u64) { - let checkpoints_db = self.checkpoints; + pub fn put_genesis_time(&self, genesis_time: u64) -> Result<()> { + let genesis_time_bytes = genesis_time.to_ssz()?; + self.checkpoints.0.put("genesis_time", genesis_time_bytes)?; + Ok(()) } } From efe3e0a5ab78b67ac63e8ac0e25c23e163932664 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Thu, 2 Jul 2026 23:00:24 +0530 Subject: [PATCH 07/20] feat: implement libmdbx with custom compression per database instance --- lean_client/Cargo.lock | 126 ++++-- lean_client/Cargo.toml | 9 + lean_client/storage/Cargo.toml | 17 +- lean_client/storage/src/database.rs | 631 ++++++++++++++++++++++++++++ lean_client/storage/src/lib.rs | 9 +- 5 files changed, 760 insertions(+), 32 deletions(-) create mode 100644 lean_client/storage/src/database.rs diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 7c16bf00..dd763c61 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -636,7 +636,7 @@ dependencies = [ "bitflags", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2 1.0.106", "quote 1.0.45", "regex", @@ -831,6 +831,8 @@ version = "1.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -1340,28 +1342,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", -] - -[[package]] -name = "database" -version = "0.0.0" -source = "git+https://github.com/grandinetech/grandine?rev=64afdee3c6be79fceffb66933dcb69a943f3f1ae#64afdee3c6be79fceffb66933dcb69a943f3f1ae" -dependencies = [ - "anyhow", - "bytesize", - "fs-err", - "futures", - "im", - "itertools 0.14.0", - "logging", - "reth-libmdbx", - "snap", - "std_ext", - "tap", - "thiserror 2.0.18", - "tracing", - "unwrap_none", + "syn 2.0.117", ] [[package]] @@ -1967,7 +1948,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" dependencies = [ "autocfg", - "tokio", ] [[package]] @@ -2557,7 +2537,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -2936,6 +2916,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.95" @@ -4423,6 +4413,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "polling" version = "3.11.0" @@ -4734,7 +4730,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -4771,7 +4767,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.52.0", ] @@ -5842,8 +5838,19 @@ dependencies = [ "anyhow", "bytesize", "containers", - "database", + "fs-err", + "futures", + "im", + "itertools 0.14.0", + "lz4_flex", + "reth-libmdbx", "ssz", + "tap", + "tempfile", + "test-case", + "thiserror 2.0.18", + "unwrap_none", + "zstd", ] [[package]] @@ -6020,12 +6027,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", ] +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2 1.0.106", + "quote 1.0.45", + "syn 2.0.117", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2 1.0.106", + "quote 1.0.45", + "syn 2.0.117", + "test-case-core", +] + [[package]] name = "test-generator" version = "0.3.1" @@ -7429,3 +7469,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index f154849b..7746917a 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -247,6 +247,7 @@ discv5 = "0.10.2" enr = { version = "0.13", features = ["k256"] } eth_ssz = { package = "ethereum_ssz", version = "0.10.0" } ethereum-types = "0.14" +fs-err = "3" futures = "0.3" features = { git = "https://github.com/grandinetech/grandine", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } git-version = "0.3" @@ -254,11 +255,13 @@ hex = "0.4.3" indexmap = "2" http-body-util = "0.1" http_api_utils = { git = "https://github.com/grandinetech/grandine", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } +im = "15" k256 = "0.13" rec_aggregation = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } backend = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } leansig = { git = "https://github.com/leanEthereum/leanSig", branch = "devnet4" } leansig_wrapper = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } +libmdbx = { git = "https://github.com/paradigmxyz/reth.git", package = "reth-libmdbx", rev = "6f8e7258f4733279080e4bd8345ce50538a40d6e" } libp2p = { git = "https://github.com/libp2p/rust-libp2p.git", rev = "91e8931e275bcd1c72791d18b09fea8b77209baf", default-features = false, features = [ 'dns', 'gossipsub', @@ -272,6 +275,7 @@ libp2p = { git = "https://github.com/libp2p/rust-libp2p.git", rev = "91e8931e275 'yamux' ] } libp2p-identity = { version = "0.2.13", features = ["secp256k1"] } +lz4_flex = "0.13" dedicated_executor = { path = "dedicated_executor" } num-bigint = "0.4" num-traits = "0.2" @@ -295,6 +299,9 @@ snap = "1.1" ssz = { git = "https://github.com/grandinetech/grandine", package = "ssz", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } ssz-types = "0.3" itertools = "0.14" +tap = "1" +tempfile = "3" +test-case = "3" test-generator = "0.3.1" thiserror = "2" tikv-jemallocator = { version = "0.6", features = ["stats", "unprefixed_malloc_on_supported_platforms"] } @@ -307,8 +314,10 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } tree-hash = "0.4.0" try_from_iterator = { git = "https://github.com/grandinetech/grandine", package = "try_from_iterator", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } typenum = "1.19" +unwrap_none = "0.1" yamux = "0.12" zeroize = "1.8" +zstd = "0.13.3" [package] name = "lean_client" diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml index 337bde2d..ba850c47 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/storage/Cargo.toml @@ -3,11 +3,24 @@ name = "storage" edition = { workspace = true } [dependencies] -database = { workspace = true } -bytesize = { workspace = true } anyhow = { workspace = true } +bytesize = { workspace = true } containers = { workspace = true } +fs-err = { workspace = true } +futures = { workspace = true } +im = { workspace = true } +itertools = { workspace = true } +libmdbx = { workspace = true } +lz4_flex = { workspace = true } ssz = { workspace = true } +tap = { workspace = true } +thiserror = { workspace = true } +unwrap_none = { workspace = true } +zstd = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +test-case = { workspace = true } [lints] workspace = true diff --git a/lean_client/storage/src/database.rs b/lean_client/storage/src/database.rs new file mode 100644 index 00000000..39b916d7 --- /dev/null +++ b/lean_client/storage/src/database.rs @@ -0,0 +1,631 @@ +use core::ops::{Range, RangeFrom, RangeToInclusive}; +use std::path::Path; +use std::{ + borrow::Cow, + sync::{Arc, Mutex}, +}; + +use anyhow::Result; +use bytesize::ByteSize; +use futures::channel::mpsc::UnboundedSender; +use im::OrdMap; +use itertools::Either; +use libmdbx::{DatabaseFlags, Environment, Geometry, ObjectLength, Stat, WriteFlags}; +use tap::Pipe as _; +use thiserror::Error; +use unwrap_none::UnwrapNone as _; + +const GROWTH_STEP: ByteSize = ByteSize::mib(256); + +const MAX_NAMED_DATABASES: usize = 10; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + None, + #[default] + Lz4, + Zstd, +} + +impl Compression { + fn compress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), + Self::Zstd => Ok(zstd::encode_all(data, 3)?), + } + } + + fn decompress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), + Self::Zstd => Ok(zstd::decode_all(data)?), + } + } + + fn decompress_pair(self, (key, compressed_value): (K, Cow<[u8]>)) -> Result<(K, Vec)> { + let value = self.decompress(&compressed_value)?; + Ok((key, value)) + } +} + +pub trait PrefixableKey { + const PREFIX: &'static str; + + #[must_use] + fn has_prefix(bytes: &[u8]) -> bool { + bytes.starts_with(Self::PREFIX.as_bytes()) + } +} + +#[derive(Debug)] +pub enum RestartMessage { + StorageMapFull(libmdbx::Error), +} + +impl RestartMessage { + pub fn send(self, tx: &UnboundedSender) { + // Nothing to do if the send fails: the receiver has been dropped. + drop(tx.unbounded_send(self)); + } +} + +#[derive(Clone, Copy)] +pub enum DatabaseMode { + ReadOnly, + ReadWrite, +} + +impl DatabaseMode { + #[must_use] + pub const fn is_read_only(self) -> bool { + matches!(self, Self::ReadOnly) + } + + #[must_use] + pub const fn mode_permissions(self) -> u16 { + match self { + // + // The UNIX permissions to set on created files. Zero value means to open existing, but do not create. + Self::ReadOnly => 0, + Self::ReadWrite => 0o600, + } + } + + #[must_use] + #[cfg(target_os = "linux")] + pub fn permissions(self) -> u32 { + self.mode_permissions().into() + } + + #[must_use] + #[cfg(not(target_os = "linux"))] + pub const fn permissions(self) -> u16 { + self.mode_permissions() + } +} + +pub struct Database { + kind: DatabaseKind, + compression: Compression, +} + +impl Database { + pub fn persistent( + name: &str, + directory: impl AsRef, + compression: Compression, + max_size: ByteSize, + mode: DatabaseMode, + restart_tx: Option>, + ) -> Result { + // If a database with the legacy name exists, keep using it. + // Otherwise, create a new database with the specified name. + // This check will not force existing users to resync. + let legacy_name = directory.as_ref().to_str().ok_or(Error)?; + + if !mode.is_read_only() { + fs_err::create_dir_all(&directory)?; + } + + // TODO(Grandine Team): The call to `set_max_dbs` and `MAX_NAMED_DATABASES` should be + // unnecessary if the default database is used. + let environment = Environment::builder() + .set_max_dbs(MAX_NAMED_DATABASES) + .set_geometry(Geometry { + size: Some(..usize::try_from(max_size.as_u64())?), + growth_step: Some(isize::try_from(GROWTH_STEP.as_u64())?), + shrink_threshold: None, + page_size: None, + }) + .open_with_permissions(directory.as_ref(), mode.permissions())?; + + let transaction = environment.begin_rw_txn()?; + let existing_db = transaction.open_db(Some(legacy_name)); + + let database_name = if existing_db.is_err() { + if !mode.is_read_only() { + transaction.create_db(Some(name), DatabaseFlags::default())?; + } + + name + } else { + legacy_name + } + .to_owned(); + + transaction.commit()?; + + Ok(Self { + kind: DatabaseKind::Persistent { + database_name, + environment, + restart_tx, + }, + compression, + }) + } + + #[must_use] + pub fn in_memory() -> Self { + Self { + kind: DatabaseKind::InMemory { + map: Mutex::default(), + }, + compression: Compression::Zstd, + } + } + + pub fn delete(&self, key: impl AsRef<[u8]>) -> Result<()> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_rw_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + if cursor.set::<()>(key.as_ref())?.is_some() { + cursor.del(WriteFlags::default())?; + transaction.commit()?; + } + } + DatabaseKind::InMemory { map } => { + map.lock() + .expect("in-memory database mutex is poisoned") + .remove(key.as_ref()); + } + } + + Ok(()) + } + + pub fn delete_range(&self, range: Range>) -> Result<()> { + let start = range.start.as_ref(); + let end = range.end.as_ref(); + + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_rw_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + let Some((mut key, ())) = cursor.set_range::, _>(start)? else { + return Ok(()); + }; + + while *key < *end { + cursor.del(WriteFlags::default())?; + match cursor.next::, _>()? { + Some((new_key, ())) => key = new_key, + None => break, + } + } + + transaction.commit()?; + } + DatabaseKind::InMemory { map } => { + // Update the map atomically for consistency with `Database::put_batch`. + // This should only make a difference if the method panics between mutations. + // The mutex will be left poisoned either way. + let mut map = map.lock().expect("in-memory database mutex is poisoned"); + let mut new_map = map.clone(); + + let end_pair = map.get_key_value(end); + let (below, _) = new_map.split(start); + let (_, above) = new_map.split(end); + + new_map = below.union(above); + + if let Some((key, value)) = end_pair { + new_map + .insert(key.clone(), value.clone()) + .expect_none("end_pair should have been discarded by OrdMap::split"); + } + + *map = new_map; + } + } + + Ok(()) + } + + pub fn contains_key(&self, key: impl AsRef<[u8]>) -> Result { + let contains_key = match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + transaction + .get::<()>(database.dbi(), key.as_ref())? + .is_some() + } + DatabaseKind::InMemory { map } => map + .lock() + .expect("in-memory database mutex is poisoned") + .contains_key(key.as_ref()), + }; + + Ok(contains_key) + } + + pub fn get(&self, key: impl AsRef<[u8]>) -> Result>> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + transaction + .get::>(database.dbi(), key.as_ref())? + .map(|compressed| self.compression.decompress(&compressed)) + } + DatabaseKind::InMemory { map } => map + .lock() + .expect("in-memory database mutex is poisoned") + .get(key.as_ref()) + .map(|compressed| self.compression.decompress(compressed)), + } + .transpose() + } + + pub fn db_stats(&self) -> Result> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + Some(transaction.db_stat(&database)?) + } + DatabaseKind::InMemory { map: _ } => None, + } + .pipe(Ok) + } + + pub fn iterate_all_keys_with_lengths( + &self, + ) -> Result, usize)>>> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + core::iter::from_fn(move || cursor.next().transpose()) + .map(|result| { + let (key, ObjectLength(length)) = result?; + Ok((key, length)) + }) + .pipe(Either::Left) + } + DatabaseKind::InMemory { map } => { + let map = map.lock().expect("in-memory database mutex is poisoned"); + + let it = map + .clone() + .into_iter() + .map(|(key, value)| Ok((Cow::Owned(key.to_vec()), value.len()))); + + it.pipe(Either::Right) + } + } + .pipe(Ok) + } + + #[expect(clippy::type_complexity)] + pub fn iterator_ascending( + &self, + range: RangeFrom>, + ) -> Result, Vec)>>> { + let start = range.start.as_ref(); + + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + cursor + .set_range(start) + .transpose() + .into_iter() + .chain(core::iter::from_fn(move || cursor.next().transpose())) + .map(|result| self.compression.decompress_pair(result?)) + .pipe(Either::Left) + } + DatabaseKind::InMemory { map } => { + let map = map.lock().expect("in-memory database mutex is poisoned"); + let start_pair = map.get_key_value(start); + let (_, mut above) = map.split(start); + + if let Some((key, value)) = start_pair { + above + .insert(key.clone(), value.clone()) + .expect_none("start_pair should have been discarded by OrdMap::split"); + } + + let it = above.into_iter().map(|(key, value)| { + Ok(( + Cow::Owned(key.to_vec()), + self.compression.decompress(value.as_ref())?, + )) + }); + + it.pipe(Either::Right) + } + } + .pipe(Ok) + } + + #[expect(clippy::type_complexity)] + pub fn iterator_descending( + &self, + range: RangeToInclusive>, + ) -> Result, Vec)>>> { + let end = range.end.as_ref(); + + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + cursor + .set_key(end) + .transpose() + .into_iter() + .chain(core::iter::from_fn(move || cursor.prev().transpose())) + .map(|result| self.compression.decompress_pair(result?)) + .pipe(Either::Left) + } + DatabaseKind::InMemory { map } => { + let map = map.lock().expect("in-memory database mutex is poisoned"); + let end_pair = map.get_key_value(end); + let (mut below, _) = map.split(end); + + if let Some((key, value)) = end_pair { + below + .insert(key.clone(), value.clone()) + .expect_none("end_pair should have been discarded by OrdMap::split"); + } + + let it = below.into_iter().rev().map(|(key, value)| { + Ok(( + Cow::Owned(key.to_vec()), + self.compression.decompress(value.as_ref())?, + )) + }); + + it.pipe(Either::Right) + } + } + .pipe(Ok) + } + + pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> { + self.put_batch(core::iter::once((key, value))) + } + + pub fn put_batch( + &self, + pairs: impl IntoIterator, impl AsRef<[u8]>)>, + ) -> Result<()> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx, + } => { + let transaction = environment.begin_rw_txn()?; + let database = transaction.open_db(Some(database_name))?; + + for (key, value) in pairs { + let key = key.as_ref(); + let compressed = self.compression.compress(value.as_ref())?; + transaction + .put(database.dbi(), key, compressed, WriteFlags::default()) + .map_err(|error| handle_write_error(error, restart_tx.as_ref()))?; + } + + transaction + .commit() + .map_err(|error| handle_write_error(error, restart_tx.as_ref()))?; + } + DatabaseKind::InMemory { map } => { + let mut map = map.lock().expect("in-memory database mutex is poisoned"); + let mut new_map = map.clone(); + + for (key, value) in pairs { + let key = key.as_ref().into(); + let compressed = self.compression.compress(value.as_ref())?.into(); + new_map.insert(key, compressed); + } + + *map = new_map; + } + } + + Ok(()) + } + + /// Returns the first key-value pair whose key is less than or equal to `key`. + /// + /// Behaves like [`im::OrdMap::get_prev`]. + /// + /// [`im::OrdMap::get_prev`]: https://docs.rs/im/15.1.0/im/ordmap/struct.OrdMap.html#method.get_prev + pub fn prev(&self, key: impl AsRef<[u8]>) -> Result, Vec)>> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + cursor + .set_key(key.as_ref()) + .transpose() + .or_else(|| cursor.prev().transpose()) + .transpose()? + .map(|pair| self.compression.decompress_pair(pair)) + } + DatabaseKind::InMemory { map } => map + .lock() + .expect("in-memory database mutex is poisoned") + .get_prev(key.as_ref()) + .map(|(key, value)| Ok((key.to_vec(), self.compression.decompress(value)?))), + } + .transpose() + } + + /// Returns the first key-value pair whose key is greater than or equal to `key`. + /// + /// Behaves like [`im::OrdMap::get_next`]. + /// + /// [`im::OrdMap::get_next`]: https://docs.rs/im/15.1.0/im/ordmap/struct.OrdMap.html#method.get_next + pub fn next(&self, key: impl AsRef<[u8]>) -> Result, Vec)>> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_ro_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + cursor + .set_range(key.as_ref())? + .map(|pair| self.compression.decompress_pair(pair)) + } + DatabaseKind::InMemory { map } => map + .lock() + .expect("in-memory database mutex is poisoned") + .get_next(key.as_ref()) + .map(|(key, value)| Ok((key.to_vec(), self.compression.decompress(value)?))), + } + .transpose() + } + + const fn kind(&self) -> &DatabaseKind { + &self.kind + } +} + +impl From for Database { + fn from(map: InMemoryMap) -> Self { + Self { + kind: DatabaseKind::InMemory { + map: Mutex::new(map), + }, + compression: Compression::Zstd, + } + } +} + +enum DatabaseKind { + Persistent { + // TODO(Grandine Team): It should be possible to remove `database_name` by using the default + // database (`None`), but that would probably force users to resync. + database_name: String, + environment: Environment, + restart_tx: Option>, + }, + InMemory { + // Various methods of `OrdMap` and `Database` clone the elements of this map, + // so they should be cheaply cloneable. This disqualifies `Vec` and `Box<[u8]>`. + // + // Various methods of `Database` return keys in the form of `Vec` or `Cow<[u8]>`. + // Converting between them and `Arc<[u8]>` is costly due to the reference count before data. + // Returning `Arc<[u8]>` from the methods would require a conversion in the persistent case + // because `libmdbx` cannot decode directly into `std::sync::Arc` or `triomphe::Arc`. + // + // `Bytes` can be cheaply converted to and from `Vec` if its capacity equals its length, + // but `Database` cannot benefit from that with its current API. + // Returning a `Vec` or `Cow` requires copying due to shared ownership. + // Writing requires copying due to the signature of `Database::put`. + // + // Some versions of `libmdbx` (including the one from `reth-libmdbx`) can decode into + // `lifetimed_bytes::Bytes`, which functions like `Cow<[u8]>`, but with the internal + // representation of `Bytes`. `lifetimed_bytes::Bytes` is necessarily distinct from + // `bytes::Bytes`, which makes it harder to use. + map: Mutex, + }, +} + +#[derive(Debug, Error)] +#[error("database directory path should be a valid Unicode string")] +struct Error; + +pub type InMemoryMap = OrdMap, Arc<[u8]>>; + +fn handle_write_error( + error: libmdbx::Error, + restart_tx: Option<&UnboundedSender>, +) -> libmdbx::Error { + if error == libmdbx::Error::MapFull { + if let Some(restart_tx) = restart_tx { + RestartMessage::StorageMapFull(error).send(restart_tx); + } + } + + error +} diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index e9518d0e..1f9495fa 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,8 +1,10 @@ +mod database; + use std::path::Path; +use crate::database::{Compression, Database, DatabaseMode}; use anyhow::Result; use bytesize::ByteSize; -use database::{Database, DatabaseMode}; use containers::{Block, Checkpoint, Slot, State}; use ssz::{H256, SszReadDefault as _, SszWrite as _}; @@ -141,6 +143,7 @@ impl Blocks { let db = Database::persistent( "blocks", Path::new("./database/blocks"), + Compression::Lz4, ByteSize::gib(2), DatabaseMode::ReadWrite, None, @@ -154,6 +157,7 @@ impl States { let db = Database::persistent( "states", Path::new("./database/states"), + Compression::Zstd, ByteSize::gib(2), DatabaseMode::ReadWrite, None, @@ -167,6 +171,7 @@ impl Checkpoints { let db = Database::persistent( "checkpoints", Path::new("./database/checkpoints"), + Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, None, @@ -180,6 +185,7 @@ impl SlotIndex { let db = Database::persistent( "slot_index", Path::new("./database/slot_index"), + Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, None, @@ -193,6 +199,7 @@ impl StateRootIndex { let db = Database::persistent( "state_root_index", Path::new("./database/state_root_index"), + Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, None, From 2da086b6672f72091a7620ee8ae6338feaecc36b Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 3 Jul 2026 00:29:53 +0530 Subject: [PATCH 08/20] chore: tests (all passing) --- lean_client/storage/src/database.rs | 366 +++++++++++++++++++++++++++ lean_client/storage/src/lib.rs | 371 ++++++++++++++++++++++++++-- 2 files changed, 721 insertions(+), 16 deletions(-) diff --git a/lean_client/storage/src/database.rs b/lean_client/storage/src/database.rs index 39b916d7..b8671dd6 100644 --- a/lean_client/storage/src/database.rs +++ b/lean_client/storage/src/database.rs @@ -204,6 +204,38 @@ impl Database { Ok(()) } + pub fn delete_batch(&self, keys: impl IntoIterator>) -> Result<()> { + match self.kind() { + DatabaseKind::Persistent { + database_name, + environment, + restart_tx: _, + } => { + let transaction = environment.begin_rw_txn()?; + let database = transaction.open_db(Some(database_name))?; + + let mut cursor = transaction.cursor(&database)?; + + for key in keys { + if cursor.set::<()>(key.as_ref())?.is_some() { + cursor.del(WriteFlags::default())?; + } + } + + transaction.commit()?; + } + DatabaseKind::InMemory { map } => { + let mut map = map.lock().expect("in-memory database mutex is poisoned"); + + for key in keys { + map.remove(key.as_ref()); + } + } + } + + Ok(()) + } + pub fn delete_range(&self, range: Range>) -> Result<()> { let start = range.start.as_ref(); let end = range.end.as_ref(); @@ -629,3 +661,337 @@ fn handle_write_error( error } + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + use test_case::test_case; + + use super::*; + + type Constructor = fn() -> Result; + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_delete(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.delete("C")?; + database.delete("D")?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("A", "1"), ("B", "2"), ("E", "5")], + )?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_delete_batch(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.delete_batch(["A", "C", "D"])?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("B", "2"), ("E", "5")], + )?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_delete_range_inclusive_exclusive(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.delete_range("B".."C")?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("A", "1"), ("C", "3"), ("E", "5")], + )?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_delete_range_between(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.delete_range("D".."F")?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("A", "1"), ("B", "2"), ("C", "3")], + )?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_contains_key(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert!(database.contains_key("A")?); + assert!(database.contains_key("B")?); + assert!(database.contains_key("C")?); + assert!(!database.contains_key("D")?); + assert!(database.contains_key("E")?); + assert!(!database.contains_key("F")?); + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_iterator_ascending(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert_pairs_eq( + database.iterator_ascending("0"..)?, + [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")], + )?; + + assert_pairs_eq( + database.iterator_ascending("A"..)?, + [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")], + )?; + + assert_pairs_eq( + database.iterator_ascending("B"..)?, + [("B", "2"), ("C", "3"), ("E", "5")], + )?; + + assert_pairs_eq( + database.iterator_ascending("C"..)?, + [("C", "3"), ("E", "5")], + )?; + + assert_pairs_eq(database.iterator_ascending("D"..)?, [("E", "5")])?; + assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; + assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_iterator_descending(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert_pairs_eq( + database.iterator_descending(..="F")?, + [("E", "5"), ("C", "3"), ("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq( + database.iterator_descending(..="E")?, + [("E", "5"), ("C", "3"), ("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq( + database.iterator_descending(..="D")?, + [("C", "3"), ("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq( + database.iterator_descending(..="C")?, + [("C", "3"), ("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq( + database.iterator_descending(..="B")?, + [("B", "2"), ("A", "1")], + )?; + + assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; + assert_pairs_eq(database.iterator_descending(..="0")?, [])?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_all_keys_iterator_with_lengths(constructor: Constructor) -> Result<()> { + let database = constructor()?; + let values = database + .iterate_all_keys_with_lengths()? + .map(|result| { + let (key, length) = result?; + let key_string = core::str::from_utf8(key.as_ref())?; + Ok((key_string.to_owned(), length)) + }) + .collect::>>()?; + + let compressed_len = Compression::Zstd.compress(b"A")?.len(); + + let expected = [ + ("A".to_owned(), compressed_len), + ("B".to_owned(), compressed_len), + ("C".to_owned(), compressed_len), + ("E".to_owned(), compressed_len), + ]; + + assert_eq!(values, expected); + + Ok(()) + } + + // This covers a bug we introduced and fixed while implementing in-memory mode. + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_iterators_do_not_modify_the_database(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; + assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; + + assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; + assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; + + assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; + assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; + + assert_pairs_eq(database.iterator_descending(..="0")?, [])?; + assert_pairs_eq(database.iterator_descending(..="0")?, [])?; + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_multiple_of_the_same_key(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + database.put_batch(vec![("A", "1"), ("A", "2"), ("A", "3")])?; + + assert_eq!(database.get("A")?, Some(to_bytes("3"))); + + Ok(()) + } + + // ```text + // 0 A B C D E F + // │ │ ├─┘ ├─┘ + // A B C E + // ``` + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_prev(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert!("0" < "A"); + + assert_eq!(database.prev("0")?, None); + assert_eq!(database.prev("A")?, Some(to_bytes_pair(("A", "1")))); + assert_eq!(database.prev("B")?, Some(to_bytes_pair(("B", "2")))); + assert_eq!(database.prev("C")?, Some(to_bytes_pair(("C", "3")))); + assert_eq!(database.prev("D")?, Some(to_bytes_pair(("C", "3")))); + assert_eq!(database.prev("E")?, Some(to_bytes_pair(("E", "5")))); + assert_eq!(database.prev("F")?, Some(to_bytes_pair(("E", "5")))); + + Ok(()) + } + + // ```text + // 0 A B C D E F + // └─┤ │ │ └─┤ + // A B C E + // ``` + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_next(constructor: Constructor) -> Result<()> { + let database = constructor()?; + + assert!("0" < "A"); + + assert_eq!(database.next("0")?, Some(to_bytes_pair(("A", "1")))); + assert_eq!(database.next("A")?, Some(to_bytes_pair(("A", "1")))); + assert_eq!(database.next("B")?, Some(to_bytes_pair(("B", "2")))); + assert_eq!(database.next("C")?, Some(to_bytes_pair(("C", "3")))); + assert_eq!(database.next("D")?, Some(to_bytes_pair(("E", "5")))); + assert_eq!(database.next("E")?, Some(to_bytes_pair(("E", "5")))); + assert_eq!(database.next("F")?, None); + + Ok(()) + } + + #[test_case(build_persistent_database)] + #[test_case(build_in_memory_database)] + fn test_isolation(constructor: Constructor) -> Result<()> { + let database = constructor()?; + let iterator = database.iterator_ascending("A"..)?; + + database.delete_range("A".."F")?; + + assert_pairs_eq(iterator, [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")])?; + + Ok(()) + } + + fn build_persistent_database() -> Result { + let database = Database::persistent( + "test_db", + TempDir::new()?, + Compression::Zstd, + ByteSize::mib(1), + DatabaseMode::ReadWrite, + None, + )?; + + populate_database(&database)?; + Ok(database) + } + + fn build_in_memory_database() -> Result { + let database = Database::in_memory(); + populate_database(&database)?; + Ok(database) + } + + fn populate_database(database: &Database) -> Result<()> { + // This indirectly tests `Database::put` and `Database::put_batch`. + database.put_batch(vec![("A", "1"), ("B", "2"), ("C", "3")])?; + database.put("E", "5")?; + Ok(()) + } + + fn assert_pairs_eq<'strings>( + actual_pairs: impl IntoIterator, impl AsRef<[u8]>)>>, + expected_pairs: impl IntoIterator, + ) -> Result<()> { + let actual_pairs = to_string_pairs(actual_pairs)?; + let expected_pairs = to_string_pairs(expected_pairs.into_iter().map(Ok))?; + + assert_eq!(actual_pairs, expected_pairs); + + Ok(()) + } + + fn to_string_pairs( + pairs: impl IntoIterator, impl AsRef<[u8]>)>>, + ) -> Result> { + pairs + .into_iter() + .map(|result| { + let (key, value) = result?; + let key_string = core::str::from_utf8(key.as_ref())?; + let value_string = core::str::from_utf8(value.as_ref())?; + Ok((key_string.to_owned(), value_string.to_owned())) + }) + .collect() + } + + fn to_bytes_pair((key, value): (&str, &str)) -> (Vec, Vec) { + (to_bytes(key), to_bytes(value)) + } + + fn to_bytes(string: &str) -> Vec { + string.as_bytes().to_vec() + } +} diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 1f9495fa..5333a297 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,5 +1,9 @@ mod database; +// Dev-dependency reserved for parametrized tests we haven't written yet. +#[cfg(test)] +use ::test_case as _; + use std::path::Path; use crate::database::{Compression, Database, DatabaseMode}; @@ -18,13 +22,14 @@ pub struct Storage { } impl Storage { - pub fn new() -> Result { + pub fn new(base: impl AsRef) -> Result { + let base = base.as_ref(); Ok(Self { - blocks: Blocks::new()?, - states: States::new()?, - checkpoints: Checkpoints::new()?, - slot_index: SlotIndex::new()?, - state_root_index: StateRootIndex::new()?, + blocks: Blocks::new(base)?, + states: States::new(base)?, + checkpoints: Checkpoints::new(base)?, + slot_index: SlotIndex::new(base)?, + state_root_index: StateRootIndex::new(base)?, }) } @@ -139,10 +144,10 @@ struct SlotIndex(Database); struct StateRootIndex(Database); impl Blocks { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "blocks", - Path::new("./database/blocks"), + base.join("blocks"), Compression::Lz4, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -153,10 +158,10 @@ impl Blocks { } impl States { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "states", - Path::new("./database/states"), + base.join("states"), Compression::Zstd, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -167,10 +172,10 @@ impl States { } impl Checkpoints { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "checkpoints", - Path::new("./database/checkpoints"), + base.join("checkpoints"), Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -181,10 +186,10 @@ impl Checkpoints { } impl SlotIndex { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "slot_index", - Path::new("./database/slot_index"), + base.join("slot_index"), Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -195,10 +200,10 @@ impl SlotIndex { } impl StateRootIndex { - fn new() -> Result { + fn new(base: &Path) -> Result { let db = Database::persistent( "state_root_index", - Path::new("./database/state_root_index"), + base.join("state_root_index"), Compression::None, ByteSize::gib(2), DatabaseMode::ReadWrite, @@ -207,3 +212,337 @@ impl StateRootIndex { Ok(Self(db)) } } + +#[cfg(test)] +mod tests { + use super::*; + + use containers::BlockBody; + use ssz::SszWrite; + use tempfile::TempDir; + + // Each test gets its own persistent `Storage` rooted in a fresh temp dir, so + // tests are isolated, parallel-safe, and cleaned up on drop. This uses the + // real `Storage::new`, so it exercises the real libmdbx path and the real + // per-DB codecs — the only difference from production is the base directory. + // + // The returned `TempDir` MUST be kept alive for as long as the `Storage` is + // used: dropping it deletes the on-disk database files. + fn storage() -> (TempDir, Storage) { + let dir = tempfile::tempdir().expect("failed to create temp dir"); + let storage = Storage::new(dir.path()).expect("failed to open storage"); + (dir, storage) + } + + fn h256(byte: u8) -> H256 { + H256::from_slice(&[byte; 32]) + } + + fn sample_block(seed: u8) -> Block { + Block { + slot: Slot(u64::from(seed)), + proposer_index: u64::from(seed), + parent_root: h256(seed), + state_root: h256(seed.wrapping_add(1)), + body: BlockBody::default(), + } + } + + // `Block` and `State` do not implement `PartialEq`, so compare their SSZ + // encodings — the exact bytes the database stores and returns. + fn assert_ssz_eq(left: &T, right: &T) { + assert_eq!( + left.to_ssz().expect("left should serialize"), + right.to_ssz().expect("right should serialize"), + ); + } + + // --- blocks ----------------------------------------------------------- + + #[test] + fn get_block_returns_none_when_absent() { + let (_dir, storage) = storage(); + assert!(storage.get_block(h256(9)).unwrap().is_none()); + } + + #[test] + fn put_then_get_block_roundtrips() { + let (_dir, storage) = storage(); + let root = h256(1); + let block = sample_block(1); + + storage.put_block(block.clone(), root).unwrap(); + + let read = storage + .get_block(root) + .unwrap() + .expect("block should exist"); + assert_ssz_eq(&block, &read); + } + + #[test] + fn put_and_get_multiple_blocks() { + let (_dir, storage) = storage(); + let blocks: Vec<(H256, Block)> = (0..8).map(|i| (h256(i), sample_block(i))).collect(); + + for (root, block) in &blocks { + storage.put_block(block.clone(), *root).unwrap(); + } + + for (root, block) in &blocks { + let read = storage + .get_block(*root) + .unwrap() + .expect("block should exist"); + assert_ssz_eq(block, &read); + } + } + + #[test] + fn put_block_overwrites_existing_root() { + let (_dir, storage) = storage(); + let root = h256(3); + + storage.put_block(sample_block(3), root).unwrap(); + let updated = sample_block(7); + storage.put_block(updated.clone(), root).unwrap(); + + let read = storage + .get_block(root) + .unwrap() + .expect("block should exist"); + assert_ssz_eq(&updated, &read); + } + + // --- states ----------------------------------------------------------- + + #[test] + fn get_state_returns_none_when_absent() { + let (_dir, storage) = storage(); + assert!(storage.get_state(h256(9)).unwrap().is_none()); + } + + #[test] + fn put_then_get_state_roundtrips() { + let (_dir, storage) = storage(); + let root = h256(2); + let state = State::generate_genesis(1_234, 3); + + storage.put_state(state.clone(), root).unwrap(); + + let read = storage + .get_state(root) + .unwrap() + .expect("state should exist"); + assert_ssz_eq(&state, &read); + } + + #[test] + fn put_and_get_multiple_states() { + let (_dir, storage) = storage(); + let states: Vec<(H256, State)> = (0..4) + .map(|i| (h256(i), State::generate_genesis(u64::from(i), u64::from(i)))) + .collect(); + + for (root, state) in &states { + storage.put_state(state.clone(), *root).unwrap(); + } + + for (root, state) in &states { + let read = storage + .get_state(*root) + .unwrap() + .expect("state should exist"); + assert_ssz_eq(state, &read); + } + } + + // --- checkpoints (justified / finalized / head / genesis_time) --------- + + #[test] + fn justified_checkpoint_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_justified_checkpoint().unwrap().is_none()); + + let checkpoint = Checkpoint { + root: h256(4), + slot: Slot(10), + }; + storage + .put_justified_checkpoint(checkpoint.clone()) + .unwrap(); + + assert_eq!( + storage.get_justified_checkpoint().unwrap().unwrap(), + checkpoint + ); + } + + #[test] + fn finalized_checkpoint_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_finalized_checkpoint().unwrap().is_none()); + + let checkpoint = Checkpoint { + root: h256(5), + slot: Slot(20), + }; + storage + .put_finalized_checkpoint(checkpoint.clone()) + .unwrap(); + + assert_eq!( + storage.get_finalized_checkpoint().unwrap().unwrap(), + checkpoint + ); + } + + #[test] + fn head_root_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_head_root().unwrap().is_none()); + + let root = h256(6); + storage.put_head_root(root).unwrap(); + + assert_eq!(storage.get_head_root().unwrap().unwrap(), root); + } + + #[test] + fn genesis_time_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_genesis_time().unwrap().is_none()); + + storage.put_genesis_time(1_700_000_000).unwrap(); + + assert_eq!(storage.get_genesis_time().unwrap().unwrap(), 1_700_000_000); + } + + // All four values above share the single `checkpoints` database under + // distinct string keys; verify they never clobber one another. + #[test] + fn checkpoints_database_keys_do_not_collide() { + let (_dir, storage) = storage(); + let justified = Checkpoint { + root: h256(1), + slot: Slot(1), + }; + let finalized = Checkpoint { + root: h256(2), + slot: Slot(2), + }; + let head = h256(3); + let genesis_time = 42; + + storage.put_justified_checkpoint(justified.clone()).unwrap(); + storage.put_finalized_checkpoint(finalized.clone()).unwrap(); + storage.put_head_root(head).unwrap(); + storage.put_genesis_time(genesis_time).unwrap(); + + assert_eq!( + storage.get_justified_checkpoint().unwrap().unwrap(), + justified + ); + assert_eq!( + storage.get_finalized_checkpoint().unwrap().unwrap(), + finalized + ); + assert_eq!(storage.get_head_root().unwrap().unwrap(), head); + assert_eq!(storage.get_genesis_time().unwrap().unwrap(), genesis_time); + } + + // --- slot_index ------------------------------------------------------- + + #[test] + fn block_root_by_slot_returns_none_when_absent() { + let (_dir, storage) = storage(); + assert!(storage.get_block_root_by_slot(Slot(0)).unwrap().is_none()); + } + + #[test] + fn block_root_by_slot_roundtrips() { + let (_dir, storage) = storage(); + let root = h256(6); + storage.put_block_root_by_slot(Slot(42), root).unwrap(); + + assert_eq!( + storage.get_block_root_by_slot(Slot(42)).unwrap().unwrap(), + root + ); + } + + #[test] + fn block_root_by_slot_handles_multiple() { + let (_dir, storage) = storage(); + for i in 0..10 { + storage + .put_block_root_by_slot(Slot(u64::from(i)), h256(i)) + .unwrap(); + } + + for i in 0..10 { + assert_eq!( + storage + .get_block_root_by_slot(Slot(u64::from(i))) + .unwrap() + .unwrap(), + h256(i), + ); + } + } + + // --- state_root_index ------------------------------------------------- + + #[test] + fn block_root_by_state_root_returns_none_when_absent() { + let (_dir, storage) = storage(); + assert!( + storage + .get_block_root_by_state_root(h256(7)) + .unwrap() + .is_none() + ); + } + + #[test] + fn block_root_by_state_root_roundtrips() { + let (_dir, storage) = storage(); + let state_root = h256(7); + let block_root = h256(8); + storage + .put_block_root_by_state_root(state_root, block_root) + .unwrap(); + + assert_eq!( + storage + .get_block_root_by_state_root(state_root) + .unwrap() + .unwrap(), + block_root, + ); + } + + #[test] + fn block_root_by_state_root_handles_multiple() { + let (_dir, storage) = storage(); + let pairs: Vec<(H256, H256)> = (0..10) + .map(|i| (h256(i), h256(i.wrapping_add(100)))) + .collect(); + + for (state_root, block_root) in &pairs { + storage + .put_block_root_by_state_root(*state_root, *block_root) + .unwrap(); + } + + for (state_root, block_root) in &pairs { + assert_eq!( + storage + .get_block_root_by_state_root(*state_root) + .unwrap() + .unwrap(), + *block_root, + ); + } + } +} From 25902c62bca1fdee9c216ae08a7cc826558d85e3 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 3 Jul 2026 18:27:43 +0530 Subject: [PATCH 09/20] chore: combine database and storage --- lean_client/storage/src/database.rs | 997 ---------------------------- lean_client/storage/src/lib.rs | 299 +++++---- 2 files changed, 170 insertions(+), 1126 deletions(-) delete mode 100644 lean_client/storage/src/database.rs diff --git a/lean_client/storage/src/database.rs b/lean_client/storage/src/database.rs deleted file mode 100644 index b8671dd6..00000000 --- a/lean_client/storage/src/database.rs +++ /dev/null @@ -1,997 +0,0 @@ -use core::ops::{Range, RangeFrom, RangeToInclusive}; -use std::path::Path; -use std::{ - borrow::Cow, - sync::{Arc, Mutex}, -}; - -use anyhow::Result; -use bytesize::ByteSize; -use futures::channel::mpsc::UnboundedSender; -use im::OrdMap; -use itertools::Either; -use libmdbx::{DatabaseFlags, Environment, Geometry, ObjectLength, Stat, WriteFlags}; -use tap::Pipe as _; -use thiserror::Error; -use unwrap_none::UnwrapNone as _; - -const GROWTH_STEP: ByteSize = ByteSize::mib(256); - -const MAX_NAMED_DATABASES: usize = 10; - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum Compression { - None, - #[default] - Lz4, - Zstd, -} - -impl Compression { - fn compress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), - Self::Zstd => Ok(zstd::encode_all(data, 3)?), - } - } - - fn decompress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), - Self::Zstd => Ok(zstd::decode_all(data)?), - } - } - - fn decompress_pair(self, (key, compressed_value): (K, Cow<[u8]>)) -> Result<(K, Vec)> { - let value = self.decompress(&compressed_value)?; - Ok((key, value)) - } -} - -pub trait PrefixableKey { - const PREFIX: &'static str; - - #[must_use] - fn has_prefix(bytes: &[u8]) -> bool { - bytes.starts_with(Self::PREFIX.as_bytes()) - } -} - -#[derive(Debug)] -pub enum RestartMessage { - StorageMapFull(libmdbx::Error), -} - -impl RestartMessage { - pub fn send(self, tx: &UnboundedSender) { - // Nothing to do if the send fails: the receiver has been dropped. - drop(tx.unbounded_send(self)); - } -} - -#[derive(Clone, Copy)] -pub enum DatabaseMode { - ReadOnly, - ReadWrite, -} - -impl DatabaseMode { - #[must_use] - pub const fn is_read_only(self) -> bool { - matches!(self, Self::ReadOnly) - } - - #[must_use] - pub const fn mode_permissions(self) -> u16 { - match self { - // - // The UNIX permissions to set on created files. Zero value means to open existing, but do not create. - Self::ReadOnly => 0, - Self::ReadWrite => 0o600, - } - } - - #[must_use] - #[cfg(target_os = "linux")] - pub fn permissions(self) -> u32 { - self.mode_permissions().into() - } - - #[must_use] - #[cfg(not(target_os = "linux"))] - pub const fn permissions(self) -> u16 { - self.mode_permissions() - } -} - -pub struct Database { - kind: DatabaseKind, - compression: Compression, -} - -impl Database { - pub fn persistent( - name: &str, - directory: impl AsRef, - compression: Compression, - max_size: ByteSize, - mode: DatabaseMode, - restart_tx: Option>, - ) -> Result { - // If a database with the legacy name exists, keep using it. - // Otherwise, create a new database with the specified name. - // This check will not force existing users to resync. - let legacy_name = directory.as_ref().to_str().ok_or(Error)?; - - if !mode.is_read_only() { - fs_err::create_dir_all(&directory)?; - } - - // TODO(Grandine Team): The call to `set_max_dbs` and `MAX_NAMED_DATABASES` should be - // unnecessary if the default database is used. - let environment = Environment::builder() - .set_max_dbs(MAX_NAMED_DATABASES) - .set_geometry(Geometry { - size: Some(..usize::try_from(max_size.as_u64())?), - growth_step: Some(isize::try_from(GROWTH_STEP.as_u64())?), - shrink_threshold: None, - page_size: None, - }) - .open_with_permissions(directory.as_ref(), mode.permissions())?; - - let transaction = environment.begin_rw_txn()?; - let existing_db = transaction.open_db(Some(legacy_name)); - - let database_name = if existing_db.is_err() { - if !mode.is_read_only() { - transaction.create_db(Some(name), DatabaseFlags::default())?; - } - - name - } else { - legacy_name - } - .to_owned(); - - transaction.commit()?; - - Ok(Self { - kind: DatabaseKind::Persistent { - database_name, - environment, - restart_tx, - }, - compression, - }) - } - - #[must_use] - pub fn in_memory() -> Self { - Self { - kind: DatabaseKind::InMemory { - map: Mutex::default(), - }, - compression: Compression::Zstd, - } - } - - pub fn delete(&self, key: impl AsRef<[u8]>) -> Result<()> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_rw_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - if cursor.set::<()>(key.as_ref())?.is_some() { - cursor.del(WriteFlags::default())?; - transaction.commit()?; - } - } - DatabaseKind::InMemory { map } => { - map.lock() - .expect("in-memory database mutex is poisoned") - .remove(key.as_ref()); - } - } - - Ok(()) - } - - pub fn delete_batch(&self, keys: impl IntoIterator>) -> Result<()> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_rw_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - for key in keys { - if cursor.set::<()>(key.as_ref())?.is_some() { - cursor.del(WriteFlags::default())?; - } - } - - transaction.commit()?; - } - DatabaseKind::InMemory { map } => { - let mut map = map.lock().expect("in-memory database mutex is poisoned"); - - for key in keys { - map.remove(key.as_ref()); - } - } - } - - Ok(()) - } - - pub fn delete_range(&self, range: Range>) -> Result<()> { - let start = range.start.as_ref(); - let end = range.end.as_ref(); - - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_rw_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - let Some((mut key, ())) = cursor.set_range::, _>(start)? else { - return Ok(()); - }; - - while *key < *end { - cursor.del(WriteFlags::default())?; - match cursor.next::, _>()? { - Some((new_key, ())) => key = new_key, - None => break, - } - } - - transaction.commit()?; - } - DatabaseKind::InMemory { map } => { - // Update the map atomically for consistency with `Database::put_batch`. - // This should only make a difference if the method panics between mutations. - // The mutex will be left poisoned either way. - let mut map = map.lock().expect("in-memory database mutex is poisoned"); - let mut new_map = map.clone(); - - let end_pair = map.get_key_value(end); - let (below, _) = new_map.split(start); - let (_, above) = new_map.split(end); - - new_map = below.union(above); - - if let Some((key, value)) = end_pair { - new_map - .insert(key.clone(), value.clone()) - .expect_none("end_pair should have been discarded by OrdMap::split"); - } - - *map = new_map; - } - } - - Ok(()) - } - - pub fn contains_key(&self, key: impl AsRef<[u8]>) -> Result { - let contains_key = match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - transaction - .get::<()>(database.dbi(), key.as_ref())? - .is_some() - } - DatabaseKind::InMemory { map } => map - .lock() - .expect("in-memory database mutex is poisoned") - .contains_key(key.as_ref()), - }; - - Ok(contains_key) - } - - pub fn get(&self, key: impl AsRef<[u8]>) -> Result>> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - transaction - .get::>(database.dbi(), key.as_ref())? - .map(|compressed| self.compression.decompress(&compressed)) - } - DatabaseKind::InMemory { map } => map - .lock() - .expect("in-memory database mutex is poisoned") - .get(key.as_ref()) - .map(|compressed| self.compression.decompress(compressed)), - } - .transpose() - } - - pub fn db_stats(&self) -> Result> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - Some(transaction.db_stat(&database)?) - } - DatabaseKind::InMemory { map: _ } => None, - } - .pipe(Ok) - } - - pub fn iterate_all_keys_with_lengths( - &self, - ) -> Result, usize)>>> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - core::iter::from_fn(move || cursor.next().transpose()) - .map(|result| { - let (key, ObjectLength(length)) = result?; - Ok((key, length)) - }) - .pipe(Either::Left) - } - DatabaseKind::InMemory { map } => { - let map = map.lock().expect("in-memory database mutex is poisoned"); - - let it = map - .clone() - .into_iter() - .map(|(key, value)| Ok((Cow::Owned(key.to_vec()), value.len()))); - - it.pipe(Either::Right) - } - } - .pipe(Ok) - } - - #[expect(clippy::type_complexity)] - pub fn iterator_ascending( - &self, - range: RangeFrom>, - ) -> Result, Vec)>>> { - let start = range.start.as_ref(); - - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - cursor - .set_range(start) - .transpose() - .into_iter() - .chain(core::iter::from_fn(move || cursor.next().transpose())) - .map(|result| self.compression.decompress_pair(result?)) - .pipe(Either::Left) - } - DatabaseKind::InMemory { map } => { - let map = map.lock().expect("in-memory database mutex is poisoned"); - let start_pair = map.get_key_value(start); - let (_, mut above) = map.split(start); - - if let Some((key, value)) = start_pair { - above - .insert(key.clone(), value.clone()) - .expect_none("start_pair should have been discarded by OrdMap::split"); - } - - let it = above.into_iter().map(|(key, value)| { - Ok(( - Cow::Owned(key.to_vec()), - self.compression.decompress(value.as_ref())?, - )) - }); - - it.pipe(Either::Right) - } - } - .pipe(Ok) - } - - #[expect(clippy::type_complexity)] - pub fn iterator_descending( - &self, - range: RangeToInclusive>, - ) -> Result, Vec)>>> { - let end = range.end.as_ref(); - - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - cursor - .set_key(end) - .transpose() - .into_iter() - .chain(core::iter::from_fn(move || cursor.prev().transpose())) - .map(|result| self.compression.decompress_pair(result?)) - .pipe(Either::Left) - } - DatabaseKind::InMemory { map } => { - let map = map.lock().expect("in-memory database mutex is poisoned"); - let end_pair = map.get_key_value(end); - let (mut below, _) = map.split(end); - - if let Some((key, value)) = end_pair { - below - .insert(key.clone(), value.clone()) - .expect_none("end_pair should have been discarded by OrdMap::split"); - } - - let it = below.into_iter().rev().map(|(key, value)| { - Ok(( - Cow::Owned(key.to_vec()), - self.compression.decompress(value.as_ref())?, - )) - }); - - it.pipe(Either::Right) - } - } - .pipe(Ok) - } - - pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> { - self.put_batch(core::iter::once((key, value))) - } - - pub fn put_batch( - &self, - pairs: impl IntoIterator, impl AsRef<[u8]>)>, - ) -> Result<()> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx, - } => { - let transaction = environment.begin_rw_txn()?; - let database = transaction.open_db(Some(database_name))?; - - for (key, value) in pairs { - let key = key.as_ref(); - let compressed = self.compression.compress(value.as_ref())?; - transaction - .put(database.dbi(), key, compressed, WriteFlags::default()) - .map_err(|error| handle_write_error(error, restart_tx.as_ref()))?; - } - - transaction - .commit() - .map_err(|error| handle_write_error(error, restart_tx.as_ref()))?; - } - DatabaseKind::InMemory { map } => { - let mut map = map.lock().expect("in-memory database mutex is poisoned"); - let mut new_map = map.clone(); - - for (key, value) in pairs { - let key = key.as_ref().into(); - let compressed = self.compression.compress(value.as_ref())?.into(); - new_map.insert(key, compressed); - } - - *map = new_map; - } - } - - Ok(()) - } - - /// Returns the first key-value pair whose key is less than or equal to `key`. - /// - /// Behaves like [`im::OrdMap::get_prev`]. - /// - /// [`im::OrdMap::get_prev`]: https://docs.rs/im/15.1.0/im/ordmap/struct.OrdMap.html#method.get_prev - pub fn prev(&self, key: impl AsRef<[u8]>) -> Result, Vec)>> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - cursor - .set_key(key.as_ref()) - .transpose() - .or_else(|| cursor.prev().transpose()) - .transpose()? - .map(|pair| self.compression.decompress_pair(pair)) - } - DatabaseKind::InMemory { map } => map - .lock() - .expect("in-memory database mutex is poisoned") - .get_prev(key.as_ref()) - .map(|(key, value)| Ok((key.to_vec(), self.compression.decompress(value)?))), - } - .transpose() - } - - /// Returns the first key-value pair whose key is greater than or equal to `key`. - /// - /// Behaves like [`im::OrdMap::get_next`]. - /// - /// [`im::OrdMap::get_next`]: https://docs.rs/im/15.1.0/im/ordmap/struct.OrdMap.html#method.get_next - pub fn next(&self, key: impl AsRef<[u8]>) -> Result, Vec)>> { - match self.kind() { - DatabaseKind::Persistent { - database_name, - environment, - restart_tx: _, - } => { - let transaction = environment.begin_ro_txn()?; - let database = transaction.open_db(Some(database_name))?; - - let mut cursor = transaction.cursor(&database)?; - - cursor - .set_range(key.as_ref())? - .map(|pair| self.compression.decompress_pair(pair)) - } - DatabaseKind::InMemory { map } => map - .lock() - .expect("in-memory database mutex is poisoned") - .get_next(key.as_ref()) - .map(|(key, value)| Ok((key.to_vec(), self.compression.decompress(value)?))), - } - .transpose() - } - - const fn kind(&self) -> &DatabaseKind { - &self.kind - } -} - -impl From for Database { - fn from(map: InMemoryMap) -> Self { - Self { - kind: DatabaseKind::InMemory { - map: Mutex::new(map), - }, - compression: Compression::Zstd, - } - } -} - -enum DatabaseKind { - Persistent { - // TODO(Grandine Team): It should be possible to remove `database_name` by using the default - // database (`None`), but that would probably force users to resync. - database_name: String, - environment: Environment, - restart_tx: Option>, - }, - InMemory { - // Various methods of `OrdMap` and `Database` clone the elements of this map, - // so they should be cheaply cloneable. This disqualifies `Vec` and `Box<[u8]>`. - // - // Various methods of `Database` return keys in the form of `Vec` or `Cow<[u8]>`. - // Converting between them and `Arc<[u8]>` is costly due to the reference count before data. - // Returning `Arc<[u8]>` from the methods would require a conversion in the persistent case - // because `libmdbx` cannot decode directly into `std::sync::Arc` or `triomphe::Arc`. - // - // `Bytes` can be cheaply converted to and from `Vec` if its capacity equals its length, - // but `Database` cannot benefit from that with its current API. - // Returning a `Vec` or `Cow` requires copying due to shared ownership. - // Writing requires copying due to the signature of `Database::put`. - // - // Some versions of `libmdbx` (including the one from `reth-libmdbx`) can decode into - // `lifetimed_bytes::Bytes`, which functions like `Cow<[u8]>`, but with the internal - // representation of `Bytes`. `lifetimed_bytes::Bytes` is necessarily distinct from - // `bytes::Bytes`, which makes it harder to use. - map: Mutex, - }, -} - -#[derive(Debug, Error)] -#[error("database directory path should be a valid Unicode string")] -struct Error; - -pub type InMemoryMap = OrdMap, Arc<[u8]>>; - -fn handle_write_error( - error: libmdbx::Error, - restart_tx: Option<&UnboundedSender>, -) -> libmdbx::Error { - if error == libmdbx::Error::MapFull { - if let Some(restart_tx) = restart_tx { - RestartMessage::StorageMapFull(error).send(restart_tx); - } - } - - error -} - -#[cfg(test)] -mod tests { - use tempfile::TempDir; - use test_case::test_case; - - use super::*; - - type Constructor = fn() -> Result; - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_delete(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.delete("C")?; - database.delete("D")?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("A", "1"), ("B", "2"), ("E", "5")], - )?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_delete_batch(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.delete_batch(["A", "C", "D"])?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("B", "2"), ("E", "5")], - )?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_delete_range_inclusive_exclusive(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.delete_range("B".."C")?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("A", "1"), ("C", "3"), ("E", "5")], - )?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_delete_range_between(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.delete_range("D".."F")?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("A", "1"), ("B", "2"), ("C", "3")], - )?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_contains_key(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert!(database.contains_key("A")?); - assert!(database.contains_key("B")?); - assert!(database.contains_key("C")?); - assert!(!database.contains_key("D")?); - assert!(database.contains_key("E")?); - assert!(!database.contains_key("F")?); - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_iterator_ascending(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert_pairs_eq( - database.iterator_ascending("0"..)?, - [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")], - )?; - - assert_pairs_eq( - database.iterator_ascending("A"..)?, - [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")], - )?; - - assert_pairs_eq( - database.iterator_ascending("B"..)?, - [("B", "2"), ("C", "3"), ("E", "5")], - )?; - - assert_pairs_eq( - database.iterator_ascending("C"..)?, - [("C", "3"), ("E", "5")], - )?; - - assert_pairs_eq(database.iterator_ascending("D"..)?, [("E", "5")])?; - assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; - assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_iterator_descending(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert_pairs_eq( - database.iterator_descending(..="F")?, - [("E", "5"), ("C", "3"), ("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq( - database.iterator_descending(..="E")?, - [("E", "5"), ("C", "3"), ("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq( - database.iterator_descending(..="D")?, - [("C", "3"), ("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq( - database.iterator_descending(..="C")?, - [("C", "3"), ("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq( - database.iterator_descending(..="B")?, - [("B", "2"), ("A", "1")], - )?; - - assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; - assert_pairs_eq(database.iterator_descending(..="0")?, [])?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_all_keys_iterator_with_lengths(constructor: Constructor) -> Result<()> { - let database = constructor()?; - let values = database - .iterate_all_keys_with_lengths()? - .map(|result| { - let (key, length) = result?; - let key_string = core::str::from_utf8(key.as_ref())?; - Ok((key_string.to_owned(), length)) - }) - .collect::>>()?; - - let compressed_len = Compression::Zstd.compress(b"A")?.len(); - - let expected = [ - ("A".to_owned(), compressed_len), - ("B".to_owned(), compressed_len), - ("C".to_owned(), compressed_len), - ("E".to_owned(), compressed_len), - ]; - - assert_eq!(values, expected); - - Ok(()) - } - - // This covers a bug we introduced and fixed while implementing in-memory mode. - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_iterators_do_not_modify_the_database(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; - assert_pairs_eq(database.iterator_ascending("E"..)?, [("E", "5")])?; - - assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; - assert_pairs_eq(database.iterator_ascending("F"..)?, [])?; - - assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; - assert_pairs_eq(database.iterator_descending(..="A")?, [("A", "1")])?; - - assert_pairs_eq(database.iterator_descending(..="0")?, [])?; - assert_pairs_eq(database.iterator_descending(..="0")?, [])?; - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_multiple_of_the_same_key(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - database.put_batch(vec![("A", "1"), ("A", "2"), ("A", "3")])?; - - assert_eq!(database.get("A")?, Some(to_bytes("3"))); - - Ok(()) - } - - // ```text - // 0 A B C D E F - // │ │ ├─┘ ├─┘ - // A B C E - // ``` - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_prev(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert!("0" < "A"); - - assert_eq!(database.prev("0")?, None); - assert_eq!(database.prev("A")?, Some(to_bytes_pair(("A", "1")))); - assert_eq!(database.prev("B")?, Some(to_bytes_pair(("B", "2")))); - assert_eq!(database.prev("C")?, Some(to_bytes_pair(("C", "3")))); - assert_eq!(database.prev("D")?, Some(to_bytes_pair(("C", "3")))); - assert_eq!(database.prev("E")?, Some(to_bytes_pair(("E", "5")))); - assert_eq!(database.prev("F")?, Some(to_bytes_pair(("E", "5")))); - - Ok(()) - } - - // ```text - // 0 A B C D E F - // └─┤ │ │ └─┤ - // A B C E - // ``` - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_next(constructor: Constructor) -> Result<()> { - let database = constructor()?; - - assert!("0" < "A"); - - assert_eq!(database.next("0")?, Some(to_bytes_pair(("A", "1")))); - assert_eq!(database.next("A")?, Some(to_bytes_pair(("A", "1")))); - assert_eq!(database.next("B")?, Some(to_bytes_pair(("B", "2")))); - assert_eq!(database.next("C")?, Some(to_bytes_pair(("C", "3")))); - assert_eq!(database.next("D")?, Some(to_bytes_pair(("E", "5")))); - assert_eq!(database.next("E")?, Some(to_bytes_pair(("E", "5")))); - assert_eq!(database.next("F")?, None); - - Ok(()) - } - - #[test_case(build_persistent_database)] - #[test_case(build_in_memory_database)] - fn test_isolation(constructor: Constructor) -> Result<()> { - let database = constructor()?; - let iterator = database.iterator_ascending("A"..)?; - - database.delete_range("A".."F")?; - - assert_pairs_eq(iterator, [("A", "1"), ("B", "2"), ("C", "3"), ("E", "5")])?; - - Ok(()) - } - - fn build_persistent_database() -> Result { - let database = Database::persistent( - "test_db", - TempDir::new()?, - Compression::Zstd, - ByteSize::mib(1), - DatabaseMode::ReadWrite, - None, - )?; - - populate_database(&database)?; - Ok(database) - } - - fn build_in_memory_database() -> Result { - let database = Database::in_memory(); - populate_database(&database)?; - Ok(database) - } - - fn populate_database(database: &Database) -> Result<()> { - // This indirectly tests `Database::put` and `Database::put_batch`. - database.put_batch(vec![("A", "1"), ("B", "2"), ("C", "3")])?; - database.put("E", "5")?; - Ok(()) - } - - fn assert_pairs_eq<'strings>( - actual_pairs: impl IntoIterator, impl AsRef<[u8]>)>>, - expected_pairs: impl IntoIterator, - ) -> Result<()> { - let actual_pairs = to_string_pairs(actual_pairs)?; - let expected_pairs = to_string_pairs(expected_pairs.into_iter().map(Ok))?; - - assert_eq!(actual_pairs, expected_pairs); - - Ok(()) - } - - fn to_string_pairs( - pairs: impl IntoIterator, impl AsRef<[u8]>)>>, - ) -> Result> { - pairs - .into_iter() - .map(|result| { - let (key, value) = result?; - let key_string = core::str::from_utf8(key.as_ref())?; - let value_string = core::str::from_utf8(value.as_ref())?; - Ok((key_string.to_owned(), value_string.to_owned())) - }) - .collect() - } - - fn to_bytes_pair((key, value): (&str, &str)) -> (Vec, Vec) { - (to_bytes(key), to_bytes(value)) - } - - fn to_bytes(string: &str) -> Vec { - string.as_bytes().to_vec() - } -} diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 5333a297..d870883d 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,40 +1,176 @@ -mod database; +use std::{ + borrow::Cow, + path::Path, + sync::Arc, +}; -// Dev-dependency reserved for parametrized tests we haven't written yet. -#[cfg(test)] -use ::test_case as _; - -use std::path::Path; - -use crate::database::{Compression, Database, DatabaseMode}; use anyhow::Result; use bytesize::ByteSize; use containers::{Block, Checkpoint, Slot, State}; use ssz::{H256, SszReadDefault as _, SszWrite as _}; +use libmdbx::{ + DatabaseFlags, Environment, Geometry, RW, Transaction, WriteFlags, +}; + pub struct Storage { - blocks: Blocks, - states: States, - checkpoints: Checkpoints, - slot_index: SlotIndex, - state_root_index: StateRootIndex, + environment: Arc, + blocks: Database, + states: Database, + checkpoints: Database, + slot_index: Database, + state_root_index: Database, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + None, + #[default] + Lz4, + Zstd, +} + +impl Compression { + fn compress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), + Self::Zstd => Ok(zstd::encode_all(data, 3)?), + } + } + + fn decompress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), + Self::Zstd => Ok(zstd::decode_all(data)?), + } + } +} + +#[derive(Clone, Copy)] +pub enum DatabaseMode { + ReadOnly, + ReadWrite, +} + +impl DatabaseMode { + #[must_use] + pub const fn is_read_only(self) -> bool { + matches!(self, Self::ReadOnly) + } + + #[must_use] + pub const fn mode_permissions(self) -> u16 { + match self { + Self::ReadOnly => 0, + Self::ReadWrite => 0o600, + } + } + + #[must_use] + #[cfg(target_os = "linux")] + pub fn permissions(self) -> u32 { + self.mode_permissions().into() + } + + #[must_use] + #[cfg(not(target_os = "linux"))] + pub const fn permissions(self) -> u16 { + self.mode_permissions() + } +} + +pub struct Database { + environment: Arc, + name: String, + compression: Compression, +} + +impl Database { + pub fn new(env: Arc, name: &str, compression: Compression) -> Result { + Ok(Self { + environment: env, + name: name.to_owned(), + compression, + }) + } + + pub fn get(&self, key: impl AsRef<[u8]>) -> Result>> { + let txn = self.environment.begin_ro_txn()?; + let db = txn.open_db(Some(&self.name))?; + + txn.get::>(db.dbi(), key.as_ref())? + .map(|compressed| self.compression.decompress(&compressed)) + .transpose() + } + + pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> { + let txn = self.environment.begin_rw_txn()?; + self.put_in(&txn, key, value)?; + txn.commit()?; + Ok(()) + } + + fn put_in( + &self, + txn: &Transaction, + key: impl AsRef<[u8]>, + value: impl AsRef<[u8]>, + ) -> Result<()> { + let db = txn.open_db(Some(&self.name))?; + let compressed = self.compression.compress(value.as_ref())?; + txn.put(db.dbi(), key.as_ref(), compressed, WriteFlags::default())?; + Ok(()) + } } impl Storage { pub fn new(base: impl AsRef) -> Result { let base = base.as_ref(); + fs_err::create_dir_all(base)?; + + let environment = Arc::new( + Environment::builder() + .set_max_dbs(5) + .set_geometry(Geometry { + size: Some(..usize::try_from(ByteSize::gib(2).as_u64())?), + growth_step: Some(isize::try_from(ByteSize::mib(256).as_u64())?), + shrink_threshold: None, + page_size: None, + }) + .open_with_permissions(base, DatabaseMode::ReadWrite.permissions())?, + ); + + let txn = environment.begin_rw_txn()?; + for name in [ + "blocks", + "states", + "checkpoints", + "slot_index", + "state_root_index", + ] { + txn.create_db(Some(name), DatabaseFlags::default())?; + } + txn.commit()?; + Ok(Self { - blocks: Blocks::new(base)?, - states: States::new(base)?, - checkpoints: Checkpoints::new(base)?, - slot_index: SlotIndex::new(base)?, - state_root_index: StateRootIndex::new(base)?, + blocks: Database::new(environment.clone(), "blocks", Compression::Lz4)?, + states: Database::new(environment.clone(), "states", Compression::Zstd)?, + checkpoints: Database::new(environment.clone(), "checkpoints", Compression::None)?, + slot_index: Database::new(environment.clone(), "slot_index", Compression::None)?, + state_root_index: Database::new( + environment.clone(), + "state_root_index", + Compression::None, + )?, + environment, }) } pub fn get_block(&self, root: H256) -> Result> { - match self.blocks.0.get(root)? { + match self.blocks.get(root)? { Some(block_bytes) => Ok(Some(Block::from_ssz_default(&block_bytes)?)), None => Ok(None), } @@ -42,12 +178,12 @@ impl Storage { pub fn put_block(&self, block: Block, root: H256) -> Result<()> { let block_bytes = block.to_ssz()?; - self.blocks.0.put(root, block_bytes)?; + self.blocks.put(root, block_bytes)?; Ok(()) } pub fn get_state(&self, root: H256) -> Result> { - match self.states.0.get(root)? { + match self.states.get(root)? { Some(state_bytes) => Ok(Some(State::from_ssz_default(&state_bytes)?)), None => Ok(None), } @@ -55,12 +191,12 @@ impl Storage { pub fn put_state(&self, state: State, root: H256) -> Result<()> { let state_bytes = state.to_ssz()?; - self.states.0.put(root, state_bytes)?; + self.states.put(root, state_bytes)?; Ok(()) } pub fn get_justified_checkpoint(&self) -> Result> { - match self.checkpoints.0.get("justified")? { + match self.checkpoints.get("justified")? { Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), None => Ok(None), } @@ -68,12 +204,12 @@ impl Storage { pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { let checkpoint_bytes = checkpoint.to_ssz()?; - self.checkpoints.0.put("justified", checkpoint_bytes)?; + self.checkpoints.put("justified", checkpoint_bytes)?; Ok(()) } pub fn get_finalized_checkpoint(&self) -> Result> { - match self.checkpoints.0.get("finalized")? { + match self.checkpoints.get("finalized")? { Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), None => Ok(None), } @@ -81,25 +217,25 @@ impl Storage { pub fn put_finalized_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { let checkpoint_bytes = checkpoint.to_ssz()?; - self.checkpoints.0.put("finalized", checkpoint_bytes)?; + self.checkpoints.put("finalized", checkpoint_bytes)?; Ok(()) } pub fn get_head_root(&self) -> Result> { - match self.checkpoints.0.get("head")? { + match self.checkpoints.get("head")? { Some(head_root_bytes) => Ok(Some(H256::from_slice(&head_root_bytes))), None => Ok(None), } } pub fn put_head_root(&self, root: H256) -> Result<()> { - self.checkpoints.0.put("head", root)?; + self.checkpoints.put("head", root)?; Ok(()) } pub fn get_block_root_by_slot(&self, slot: Slot) -> Result> { let slot_bytes = slot.0.to_be_bytes(); - match self.slot_index.0.get(slot_bytes)? { + match self.slot_index.get(slot_bytes)? { Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), None => Ok(None), } @@ -107,24 +243,24 @@ impl Storage { pub fn put_block_root_by_slot(&self, slot: Slot, root: H256) -> Result<()> { let slot_bytes = slot.0.to_be_bytes(); - self.slot_index.0.put(slot_bytes, root)?; + self.slot_index.put(slot_bytes, root)?; Ok(()) } pub fn get_block_root_by_state_root(&self, state_root: H256) -> Result> { - match self.state_root_index.0.get(state_root)? { + match self.state_root_index.get(state_root)? { Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), None => Ok(None), } } pub fn put_block_root_by_state_root(&self, state_root: H256, block_root: H256) -> Result<()> { - self.state_root_index.0.put(state_root, block_root)?; + self.state_root_index.put(state_root, block_root)?; Ok(()) } pub fn get_genesis_time(&self) -> Result> { - match self.checkpoints.0.get("genesis_time")? { + match self.checkpoints.get("genesis_time")? { Some(genesis_time_bytes) => Ok(Some(u64::from_ssz_default(&genesis_time_bytes)?)), None => Ok(None), } @@ -132,87 +268,11 @@ impl Storage { pub fn put_genesis_time(&self, genesis_time: u64) -> Result<()> { let genesis_time_bytes = genesis_time.to_ssz()?; - self.checkpoints.0.put("genesis_time", genesis_time_bytes)?; + self.checkpoints.put("genesis_time", genesis_time_bytes)?; Ok(()) } } -struct Blocks(Database); -struct States(Database); -struct Checkpoints(Database); -struct SlotIndex(Database); -struct StateRootIndex(Database); - -impl Blocks { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "blocks", - base.join("blocks"), - Compression::Lz4, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - -impl States { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "states", - base.join("states"), - Compression::Zstd, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - -impl Checkpoints { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "checkpoints", - base.join("checkpoints"), - Compression::None, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - -impl SlotIndex { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "slot_index", - base.join("slot_index"), - Compression::None, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - -impl StateRootIndex { - fn new(base: &Path) -> Result { - let db = Database::persistent( - "state_root_index", - base.join("state_root_index"), - Compression::None, - ByteSize::gib(2), - DatabaseMode::ReadWrite, - None, - )?; - Ok(Self(db)) - } -} - #[cfg(test)] mod tests { use super::*; @@ -221,13 +281,6 @@ mod tests { use ssz::SszWrite; use tempfile::TempDir; - // Each test gets its own persistent `Storage` rooted in a fresh temp dir, so - // tests are isolated, parallel-safe, and cleaned up on drop. This uses the - // real `Storage::new`, so it exercises the real libmdbx path and the real - // per-DB codecs — the only difference from production is the base directory. - // - // The returned `TempDir` MUST be kept alive for as long as the `Storage` is - // used: dropping it deletes the on-disk database files. fn storage() -> (TempDir, Storage) { let dir = tempfile::tempdir().expect("failed to create temp dir"); let storage = Storage::new(dir.path()).expect("failed to open storage"); @@ -248,8 +301,6 @@ mod tests { } } - // `Block` and `State` do not implement `PartialEq`, so compare their SSZ - // encodings — the exact bytes the database stores and returns. fn assert_ssz_eq(left: &T, right: &T) { assert_eq!( left.to_ssz().expect("left should serialize"), @@ -257,8 +308,6 @@ mod tests { ); } - // --- blocks ----------------------------------------------------------- - #[test] fn get_block_returns_none_when_absent() { let (_dir, storage) = storage(); @@ -314,8 +363,6 @@ mod tests { assert_ssz_eq(&updated, &read); } - // --- states ----------------------------------------------------------- - #[test] fn get_state_returns_none_when_absent() { let (_dir, storage) = storage(); @@ -357,8 +404,6 @@ mod tests { } } - // --- checkpoints (justified / finalized / head / genesis_time) --------- - #[test] fn justified_checkpoint_roundtrips() { let (_dir, storage) = storage(); @@ -418,8 +463,6 @@ mod tests { assert_eq!(storage.get_genesis_time().unwrap().unwrap(), 1_700_000_000); } - // All four values above share the single `checkpoints` database under - // distinct string keys; verify they never clobber one another. #[test] fn checkpoints_database_keys_do_not_collide() { let (_dir, storage) = storage(); @@ -451,7 +494,6 @@ mod tests { assert_eq!(storage.get_genesis_time().unwrap().unwrap(), genesis_time); } - // --- slot_index ------------------------------------------------------- #[test] fn block_root_by_slot_returns_none_when_absent() { @@ -491,7 +533,6 @@ mod tests { } } - // --- state_root_index ------------------------------------------------- #[test] fn block_root_by_state_root_returns_none_when_absent() { From eaffc221d6cc7f26d923dea19b369cded0769ebc Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 3 Jul 2026 22:38:49 +0530 Subject: [PATCH 10/20] feat: fully spec-compliant storage API --- lean_client/storage/Cargo.toml | 7 - lean_client/storage/src/lib.rs | 690 +++++++++++++++++++++++++-------- 2 files changed, 530 insertions(+), 167 deletions(-) diff --git a/lean_client/storage/Cargo.toml b/lean_client/storage/Cargo.toml index ba850c47..2a05808a 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/storage/Cargo.toml @@ -7,20 +7,13 @@ anyhow = { workspace = true } bytesize = { workspace = true } containers = { workspace = true } fs-err = { workspace = true } -futures = { workspace = true } -im = { workspace = true } -itertools = { workspace = true } libmdbx = { workspace = true } lz4_flex = { workspace = true } ssz = { workspace = true } -tap = { workspace = true } -thiserror = { workspace = true } -unwrap_none = { workspace = true } zstd = { workspace = true } [dev-dependencies] tempfile = { workspace = true } -test-case = { workspace = true } [lints] workspace = true diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index d870883d..cba84a52 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,8 +1,4 @@ -use std::{ - borrow::Cow, - path::Path, - sync::Arc, -}; +use std::{borrow::Cow, cell::RefCell, collections::HashSet, path::Path, sync::Arc}; use anyhow::Result; use bytesize::ByteSize; @@ -10,120 +6,31 @@ use bytesize::ByteSize; use containers::{Block, Checkpoint, Slot, State}; use ssz::{H256, SszReadDefault as _, SszWrite as _}; -use libmdbx::{ - DatabaseFlags, Environment, Geometry, RW, Transaction, WriteFlags, -}; +use libmdbx::{DatabaseFlags, Environment, Geometry, RW, Transaction, TransactionKind, WriteFlags}; + +const BLOCKS_TABLE_NAME: &str = "blocks"; +const STATES_TABLE_NAME: &str = "states"; +const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; +const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; +const STATE_ROOT_INDEX_TABLE_NAME: &str = "state_root_index"; +const BLOCKS_BY_SLOT_TABLE_NAME: &str = "blocks_by_slot"; +const STATES_BY_SLOT_TABLE_NAME: &str = "states_by_slot"; + +const JUSTIFIED: u8 = 0; +const FINALIZED: u8 = 1; +const HEAD: u8 = 2; +const GENESIS_TIME: u8 = 3; pub struct Storage { environment: Arc, + transaction: RefCell>>, blocks: Database, states: Database, checkpoints: Database, slot_index: Database, state_root_index: Database, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum Compression { - None, - #[default] - Lz4, - Zstd, -} - -impl Compression { - fn compress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), - Self::Zstd => Ok(zstd::encode_all(data, 3)?), - } - } - - fn decompress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), - Self::Zstd => Ok(zstd::decode_all(data)?), - } - } -} - -#[derive(Clone, Copy)] -pub enum DatabaseMode { - ReadOnly, - ReadWrite, -} - -impl DatabaseMode { - #[must_use] - pub const fn is_read_only(self) -> bool { - matches!(self, Self::ReadOnly) - } - - #[must_use] - pub const fn mode_permissions(self) -> u16 { - match self { - Self::ReadOnly => 0, - Self::ReadWrite => 0o600, - } - } - - #[must_use] - #[cfg(target_os = "linux")] - pub fn permissions(self) -> u32 { - self.mode_permissions().into() - } - - #[must_use] - #[cfg(not(target_os = "linux"))] - pub const fn permissions(self) -> u16 { - self.mode_permissions() - } -} - -pub struct Database { - environment: Arc, - name: String, - compression: Compression, -} - -impl Database { - pub fn new(env: Arc, name: &str, compression: Compression) -> Result { - Ok(Self { - environment: env, - name: name.to_owned(), - compression, - }) - } - - pub fn get(&self, key: impl AsRef<[u8]>) -> Result>> { - let txn = self.environment.begin_ro_txn()?; - let db = txn.open_db(Some(&self.name))?; - - txn.get::>(db.dbi(), key.as_ref())? - .map(|compressed| self.compression.decompress(&compressed)) - .transpose() - } - - pub fn put(&self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Result<()> { - let txn = self.environment.begin_rw_txn()?; - self.put_in(&txn, key, value)?; - txn.commit()?; - Ok(()) - } - - fn put_in( - &self, - txn: &Transaction, - key: impl AsRef<[u8]>, - value: impl AsRef<[u8]>, - ) -> Result<()> { - let db = txn.open_db(Some(&self.name))?; - let compressed = self.compression.compress(value.as_ref())?; - txn.put(db.dbi(), key.as_ref(), compressed, WriteFlags::default())?; - Ok(()) - } + blocks_by_slot: Database, + states_by_slot: Database, } impl Storage { @@ -133,7 +40,7 @@ impl Storage { let environment = Arc::new( Environment::builder() - .set_max_dbs(5) + .set_max_dbs(7) .set_geometry(Geometry { size: Some(..usize::try_from(ByteSize::gib(2).as_u64())?), growth_step: Some(isize::try_from(ByteSize::mib(256).as_u64())?), @@ -145,132 +52,383 @@ impl Storage { let txn = environment.begin_rw_txn()?; for name in [ - "blocks", - "states", - "checkpoints", - "slot_index", - "state_root_index", + BLOCKS_TABLE_NAME, + STATES_TABLE_NAME, + CHECKPOINTS_TABLE_NAME, + SLOT_INDEX_TABLE_NAME, + STATE_ROOT_INDEX_TABLE_NAME, + BLOCKS_BY_SLOT_TABLE_NAME, + STATES_BY_SLOT_TABLE_NAME, ] { txn.create_db(Some(name), DatabaseFlags::default())?; } txn.commit()?; Ok(Self { - blocks: Database::new(environment.clone(), "blocks", Compression::Lz4)?, - states: Database::new(environment.clone(), "states", Compression::Zstd)?, - checkpoints: Database::new(environment.clone(), "checkpoints", Compression::None)?, - slot_index: Database::new(environment.clone(), "slot_index", Compression::None)?, - state_root_index: Database::new( - environment.clone(), - "state_root_index", - Compression::None, - )?, + blocks: Database::new(BLOCKS_TABLE_NAME, Compression::Lz4), + states: Database::new(STATES_TABLE_NAME, Compression::Zstd), + checkpoints: Database::new(CHECKPOINTS_TABLE_NAME, Compression::None), + slot_index: Database::new(SLOT_INDEX_TABLE_NAME, Compression::None), + state_root_index: Database::new(STATE_ROOT_INDEX_TABLE_NAME, Compression::None), + blocks_by_slot: Database::new(BLOCKS_BY_SLOT_TABLE_NAME, Compression::None), + states_by_slot: Database::new(STATES_BY_SLOT_TABLE_NAME, Compression::None), environment, + transaction: RefCell::new(None), }) } + pub fn batch_write(&self, f: impl FnOnce() -> Result<()>) -> Result<()> { + let txn = self.environment.begin_rw_txn()?; + *self.transaction.borrow_mut() = Some(txn); + + let result = f(); + + let txn = self + .transaction + .borrow_mut() + .take() + .expect("batch_write installed the ambient transaction"); + + match result { + Ok(()) => { + txn.commit()?; + Ok(()) + } + Err(error) => { + drop(txn); + Err(error) + } + } + } + + fn write(&self, f: impl FnOnce(&Transaction) -> Result) -> Result { + if let Some(txn) = self.transaction.borrow().as_ref() { + return f(txn); + } + let txn = self.environment.begin_rw_txn()?; + let result = f(&txn)?; + txn.commit()?; + Ok(result) + } + + fn read_bytes(&self, db: &Database, key: impl AsRef<[u8]>) -> Result>> { + if let Some(txn) = self.transaction.borrow().as_ref() { + return db.get(txn, key); + } + let txn = self.environment.begin_ro_txn()?; + let bytes = db.get(&txn, key)?; + txn.commit()?; + Ok(bytes) + } + pub fn get_block(&self, root: H256) -> Result> { - match self.blocks.get(root)? { - Some(block_bytes) => Ok(Some(Block::from_ssz_default(&block_bytes)?)), + match self.read_bytes(&self.blocks, root)? { + Some(bytes) => Ok(Some(Block::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_block(&self, block: Block, root: H256) -> Result<()> { let block_bytes = block.to_ssz()?; - self.blocks.put(root, block_bytes)?; - Ok(()) + let index_key = slot_root_key(block.slot, root); + self.write(|txn| { + self.blocks.put(txn, root, block_bytes)?; + self.blocks_by_slot.put(txn, index_key, b"")?; + Ok(()) + }) } pub fn get_state(&self, root: H256) -> Result> { - match self.states.get(root)? { - Some(state_bytes) => Ok(Some(State::from_ssz_default(&state_bytes)?)), + match self.read_bytes(&self.states, root)? { + Some(bytes) => Ok(Some(State::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_state(&self, state: State, root: H256) -> Result<()> { let state_bytes = state.to_ssz()?; - self.states.put(root, state_bytes)?; - Ok(()) + let index_key = slot_root_key(state.slot, root); + self.write(|txn| { + self.states.put(txn, root, state_bytes)?; + self.states_by_slot.put(txn, index_key, b"")?; + Ok(()) + }) } pub fn get_justified_checkpoint(&self) -> Result> { - match self.checkpoints.get("justified")? { - Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), + match self.read_bytes(&self.checkpoints, [JUSTIFIED])? { + Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { let checkpoint_bytes = checkpoint.to_ssz()?; - self.checkpoints.put("justified", checkpoint_bytes)?; - Ok(()) + self.write(|txn| self.checkpoints.put(txn, [JUSTIFIED], checkpoint_bytes)) } pub fn get_finalized_checkpoint(&self) -> Result> { - match self.checkpoints.get("finalized")? { - Some(checkpoint_bytes) => Ok(Some(Checkpoint::from_ssz_default(&checkpoint_bytes)?)), + match self.read_bytes(&self.checkpoints, [FINALIZED])? { + Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_finalized_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { let checkpoint_bytes = checkpoint.to_ssz()?; - self.checkpoints.put("finalized", checkpoint_bytes)?; - Ok(()) + self.write(|txn| self.checkpoints.put(txn, [FINALIZED], checkpoint_bytes)) } pub fn get_head_root(&self) -> Result> { - match self.checkpoints.get("head")? { - Some(head_root_bytes) => Ok(Some(H256::from_slice(&head_root_bytes))), + match self.read_bytes(&self.checkpoints, [HEAD])? { + Some(bytes) => Ok(Some(H256::from_slice(&bytes))), None => Ok(None), } } pub fn put_head_root(&self, root: H256) -> Result<()> { - self.checkpoints.put("head", root)?; - Ok(()) + self.write(|txn| self.checkpoints.put(txn, [HEAD], root)) } pub fn get_block_root_by_slot(&self, slot: Slot) -> Result> { - let slot_bytes = slot.0.to_be_bytes(); - match self.slot_index.get(slot_bytes)? { - Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), + match self.read_bytes(&self.slot_index, slot.0.to_be_bytes())? { + Some(bytes) => Ok(Some(H256::from_slice(&bytes))), None => Ok(None), } } pub fn put_block_root_by_slot(&self, slot: Slot, root: H256) -> Result<()> { let slot_bytes = slot.0.to_be_bytes(); - self.slot_index.put(slot_bytes, root)?; - Ok(()) + self.write(|txn| self.slot_index.put(txn, slot_bytes, root)) } pub fn get_block_root_by_state_root(&self, state_root: H256) -> Result> { - match self.state_root_index.get(state_root)? { - Some(block_root_bytes) => Ok(Some(H256::from_slice(&block_root_bytes))), + match self.read_bytes(&self.state_root_index, state_root)? { + Some(bytes) => Ok(Some(H256::from_slice(&bytes))), None => Ok(None), } } pub fn put_block_root_by_state_root(&self, state_root: H256, block_root: H256) -> Result<()> { - self.state_root_index.put(state_root, block_root)?; - Ok(()) + self.write(|txn| self.state_root_index.put(txn, state_root, block_root)) } pub fn get_genesis_time(&self) -> Result> { - match self.checkpoints.get("genesis_time")? { - Some(genesis_time_bytes) => Ok(Some(u64::from_ssz_default(&genesis_time_bytes)?)), + match self.read_bytes(&self.checkpoints, [GENESIS_TIME])? { + Some(bytes) => Ok(Some(u64::from_ssz_default(&bytes)?)), None => Ok(None), } } pub fn put_genesis_time(&self, genesis_time: u64) -> Result<()> { let genesis_time_bytes = genesis_time.to_ssz()?; - self.checkpoints.put("genesis_time", genesis_time_bytes)?; + self.write(|txn| { + self.checkpoints + .put(txn, [GENESIS_TIME], genesis_time_bytes) + }) + } + + pub fn prune_before_slot(&self, slot: Slot, keep_roots: &HashSet) -> Result { + let bound = slot.0; + self.write(|txn| { + let mut block_roots: Vec = Vec::new(); + let mut block_index_keys: Vec> = Vec::new(); + self.blocks_by_slot.for_each(txn, |key, _value| { + if slot_of(key) >= bound { + return Ok(false); + } + let root = H256::from_slice(&key[8..]); + if !keep_roots.contains(&root) { + block_roots.push(root); + block_index_keys.push(key.to_vec()); + } + Ok(true) + })?; + + let mut state_roots: Vec = Vec::new(); + let mut state_index_keys: Vec> = Vec::new(); + self.states_by_slot.for_each(txn, |key, _value| { + if slot_of(key) >= bound { + return Ok(false); + } + let root = H256::from_slice(&key[8..]); + if !keep_roots.contains(&root) { + state_roots.push(root); + state_index_keys.push(key.to_vec()); + } + Ok(true) + })?; + + let mut slot_index_keys: Vec> = Vec::new(); + self.slot_index.for_each(txn, |key, value| { + if slot_of(key) >= bound { + return Ok(false); + } + let canonical_root = H256::from_slice(value); + if !keep_roots.contains(&canonical_root) { + slot_index_keys.push(key.to_vec()); + } + Ok(true) + })?; + + let mut removed = 0; + removed += self.blocks.delete_batch(txn, &block_roots)?; + removed += self.blocks_by_slot.delete_batch(txn, &block_index_keys)?; + removed += self.slot_index.delete_batch(txn, &slot_index_keys)?; + removed += self.states.delete_batch(txn, &state_roots)?; + removed += self.states_by_slot.delete_batch(txn, &state_index_keys)?; + removed += self.state_root_index.delete_batch(txn, &state_roots)?; + + Ok(removed) + }) + } +} + +fn slot_root_key(slot: Slot, root: H256) -> [u8; 40] { + let mut key = [0u8; 40]; + key[..8].copy_from_slice(&slot.0.to_be_bytes()); + key[8..].copy_from_slice(root.as_ref()); + key +} + +fn slot_of(key: &[u8]) -> u64 { + let mut slot = [0u8; 8]; + slot.copy_from_slice(&key[..8]); + u64::from_be_bytes(slot) +} + +pub struct Database { + name: String, + compression: Compression, +} + +impl Database { + pub fn new(name: &str, compression: Compression) -> Self { + Self { + name: name.to_owned(), + compression, + } + } + + pub fn get( + &self, + txn: &Transaction, + key: impl AsRef<[u8]>, + ) -> Result>> { + let db = txn.open_db(Some(&self.name))?; + + txn.get::>(db.dbi(), key.as_ref())? + .map(|compressed| self.compression.decompress(&compressed)) + .transpose() + } + + pub fn put( + &self, + txn: &Transaction, + key: impl AsRef<[u8]>, + value: impl AsRef<[u8]>, + ) -> Result<()> { + let db = txn.open_db(Some(&self.name))?; + let compressed = self.compression.compress(value.as_ref())?; + txn.put(db.dbi(), key.as_ref(), compressed, WriteFlags::default())?; Ok(()) } + + pub fn for_each( + &self, + txn: &Transaction, + mut f: impl FnMut(&[u8], &[u8]) -> Result, + ) -> Result<()> { + let db = txn.open_db(Some(&self.name))?; + let mut cursor = txn.cursor(&db)?; + + while let Some((key, value)) = cursor.next::, Cow<[u8]>>()? { + let value = self.compression.decompress(&value)?; + if !f(key.as_ref(), &value)? { + break; + } + } + + Ok(()) + } + + pub fn delete_batch( + &self, + txn: &Transaction, + keys: impl IntoIterator>, + ) -> Result { + let db = txn.open_db(Some(&self.name))?; + let mut cursor = txn.cursor(&db)?; + + let mut deleted = 0; + for key in keys { + if cursor.set::<()>(key.as_ref())?.is_some() { + cursor.del(WriteFlags::default())?; + deleted += 1; + } + } + + Ok(deleted) + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + None, + #[default] + Lz4, + Zstd, +} + +impl Compression { + fn compress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), + Self::Zstd => Ok(zstd::encode_all(data, 3)?), + } + } + + fn decompress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), + Self::Zstd => Ok(zstd::decode_all(data)?), + } + } +} + +#[derive(Clone, Copy)] +pub enum DatabaseMode { + ReadOnly, + ReadWrite, +} + +impl DatabaseMode { + #[must_use] + pub const fn is_read_only(self) -> bool { + matches!(self, Self::ReadOnly) + } + + #[must_use] + pub const fn mode_permissions(self) -> u16 { + match self { + Self::ReadOnly => 0, + Self::ReadWrite => 0o600, + } + } + + #[must_use] + #[cfg(target_os = "linux")] + pub fn permissions(self) -> u32 { + self.mode_permissions().into() + } + + #[must_use] + #[cfg(not(target_os = "linux"))] + pub const fn permissions(self) -> u16 { + self.mode_permissions() + } } #[cfg(test)] @@ -494,7 +652,6 @@ mod tests { assert_eq!(storage.get_genesis_time().unwrap().unwrap(), genesis_time); } - #[test] fn block_root_by_slot_returns_none_when_absent() { let (_dir, storage) = storage(); @@ -533,7 +690,6 @@ mod tests { } } - #[test] fn block_root_by_state_root_returns_none_when_absent() { let (_dir, storage) = storage(); @@ -586,4 +742,218 @@ mod tests { ); } } + + fn state_at_slot(slot: u64) -> State { + let mut state = State::generate_genesis(1_234, 3); + state.slot = Slot(slot); + state + } + + #[test] + fn prune_removes_blocks_below_slot_and_keeps_the_rest() { + let (_dir, storage) = storage(); + for i in 0..10u8 { + storage.put_block(sample_block(i), h256(i)).unwrap(); + } + + let removed = storage.prune_before_slot(Slot(5), &HashSet::new()).unwrap(); + assert_eq!(removed, 10); + + for i in 0..5u8 { + assert!(storage.get_block(h256(i)).unwrap().is_none()); + } + for i in 5..10u8 { + assert!(storage.get_block(h256(i)).unwrap().is_some()); + } + } + + #[test] + fn prune_keeps_frozen_block_root() { + let (_dir, storage) = storage(); + for i in 0..5u8 { + storage.put_block(sample_block(i), h256(i)).unwrap(); + } + + let mut keep = HashSet::new(); + keep.insert(h256(2)); + + storage.prune_before_slot(Slot(5), &keep).unwrap(); + + assert!(storage.get_block(h256(2)).unwrap().is_some()); + for i in [0u8, 1, 3, 4] { + assert!(storage.get_block(h256(i)).unwrap().is_none()); + } + } + + #[test] + fn prune_removes_forked_blocks_at_same_slot() { + let (_dir, storage) = storage(); + let mut fork = sample_block(2); + fork.proposer_index = 99; + storage.put_block(sample_block(2), h256(20)).unwrap(); + storage.put_block(fork, h256(21)).unwrap(); + + let removed = storage.prune_before_slot(Slot(5), &HashSet::new()).unwrap(); + + assert_eq!(removed, 4); + assert!(storage.get_block(h256(20)).unwrap().is_none()); + assert!(storage.get_block(h256(21)).unwrap().is_none()); + } + + #[test] + fn prune_cleans_slot_index_unless_frozen() { + let (_dir, storage) = storage(); + storage.put_block(sample_block(2), h256(2)).unwrap(); + storage.put_block_root_by_slot(Slot(2), h256(2)).unwrap(); + storage.put_block(sample_block(3), h256(3)).unwrap(); + storage.put_block_root_by_slot(Slot(3), h256(3)).unwrap(); + + let mut keep = HashSet::new(); + keep.insert(h256(3)); + + storage.prune_before_slot(Slot(5), &keep).unwrap(); + + assert!(storage.get_block_root_by_slot(Slot(2)).unwrap().is_none()); + assert_eq!( + storage.get_block_root_by_slot(Slot(3)).unwrap().unwrap(), + h256(3), + ); + } + + #[test] + fn prune_removes_states_below_slot_and_keeps_frozen() { + let (_dir, storage) = storage(); + storage.put_state(state_at_slot(1), h256(1)).unwrap(); + storage.put_state(state_at_slot(2), h256(2)).unwrap(); + storage + .put_block_root_by_state_root(h256(1), h256(101)) + .unwrap(); + storage + .put_block_root_by_state_root(h256(2), h256(102)) + .unwrap(); + + let mut keep = HashSet::new(); + keep.insert(h256(2)); + + storage.prune_before_slot(Slot(5), &keep).unwrap(); + + assert!(storage.get_state(h256(1)).unwrap().is_none()); + assert!(storage.get_state(h256(2)).unwrap().is_some()); + assert!( + storage + .get_block_root_by_state_root(h256(1)) + .unwrap() + .is_none() + ); + assert_eq!( + storage + .get_block_root_by_state_root(h256(2)) + .unwrap() + .unwrap(), + h256(102), + ); + } + + #[test] + fn prune_protection_is_per_entity() { + let (_dir, storage) = storage(); + storage.put_block(sample_block(1), h256(1)).unwrap(); + storage.put_state(state_at_slot(1), h256(1)).unwrap(); + + let mut keep = HashSet::new(); + keep.insert(h256(1)); + + storage.prune_before_slot(Slot(5), &keep).unwrap(); + + assert!(storage.get_block(h256(1)).unwrap().is_some()); + assert!(storage.get_state(h256(1)).unwrap().is_some()); + } + + #[test] + fn prune_below_lowest_slot_is_a_noop() { + let (_dir, storage) = storage(); + for i in 0..5u8 { + storage.put_block(sample_block(i), h256(i)).unwrap(); + } + + let removed = storage.prune_before_slot(Slot(0), &HashSet::new()).unwrap(); + assert_eq!(removed, 0); + for i in 0..5u8 { + assert!(storage.get_block(h256(i)).unwrap().is_some()); + } + } + + #[test] + fn batch_write_commits_all_on_success() { + let (_dir, storage) = storage(); + + storage + .batch_write(|| { + storage.put_block(sample_block(1), h256(1))?; + storage.put_state(state_at_slot(1), h256(2))?; + storage.put_head_root(h256(3))?; + Ok(()) + }) + .unwrap(); + + assert!(storage.get_block(h256(1)).unwrap().is_some()); + assert!(storage.get_state(h256(2)).unwrap().is_some()); + assert_eq!(storage.get_head_root().unwrap().unwrap(), h256(3)); + } + + #[test] + fn batch_write_rolls_back_every_write_on_error() { + let (_dir, storage) = storage(); + + let result = storage.batch_write(|| { + storage.put_block(sample_block(1), h256(1))?; + storage.put_state(state_at_slot(1), h256(2))?; + Err(anyhow::anyhow!("intentional failure")) + }); + + assert!(result.is_err()); + assert!(storage.get_block(h256(1)).unwrap().is_none()); + assert!(storage.get_state(h256(2)).unwrap().is_none()); + } + + #[test] + fn storage_is_usable_after_batch_write() { + let (_dir, storage) = storage(); + + storage + .batch_write(|| storage.put_block(sample_block(1), h256(1))) + .unwrap(); + + storage.put_block(sample_block(2), h256(2)).unwrap(); + + assert!(storage.get_block(h256(1)).unwrap().is_some()); + assert!(storage.get_block(h256(2)).unwrap().is_some()); + } + + #[test] + fn storage_is_usable_after_failed_batch_write() { + let (_dir, storage) = storage(); + + assert!( + storage + .batch_write(|| Err(anyhow::anyhow!("boom"))) + .is_err() + ); + + storage.put_block(sample_block(5), h256(5)).unwrap(); + assert!(storage.get_block(h256(5)).unwrap().is_some()); + } + + #[test] + fn batch_write_reads_see_pending_writes() { + let (_dir, storage) = storage(); + + storage + .batch_write(|| { + storage.put_block(sample_block(7), h256(7)).unwrap(); + assert!(storage.get_block(h256(7)).unwrap().is_some()); + Ok(()) + }) + .unwrap(); + } } From 5f1fdb88da20f6e836fe5dd5056b0763adb2fba8 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 3 Jul 2026 22:59:38 +0530 Subject: [PATCH 11/20] chore: privatized Database and removed DatabaseMode --- lean_client/Cargo.lock | 46 ------------------------------- lean_client/Cargo.toml | 4 --- lean_client/storage/src/lib.rs | 49 ++++++---------------------------- 3 files changed, 8 insertions(+), 91 deletions(-) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index dd763c61..b52568f7 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -5839,17 +5839,10 @@ dependencies = [ "bytesize", "containers", "fs-err", - "futures", - "im", - "itertools 0.14.0", "lz4_flex", "reth-libmdbx", "ssz", - "tap", "tempfile", - "test-case", - "thiserror 2.0.18", - "unwrap_none", "zstd", ] @@ -6033,39 +6026,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "test-case" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" -dependencies = [ - "test-case-macros", -] - -[[package]] -name = "test-case-core" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" -dependencies = [ - "cfg-if", - "proc-macro2 1.0.106", - "quote 1.0.45", - "syn 2.0.117", -] - -[[package]] -name = "test-case-macros" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" -dependencies = [ - "proc-macro2 1.0.106", - "quote 1.0.45", - "syn 2.0.117", - "test-case-core", -] - [[package]] name = "test-generator" version = "0.3.1" @@ -6605,12 +6565,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "unwrap_none" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "461d0c5956fcc728ecc03a3a961e4adc9a7975d86f6f8371389a289517c02ca9" - [[package]] name = "url" version = "2.5.8" diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 7746917a..ac52e68a 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -255,7 +255,6 @@ hex = "0.4.3" indexmap = "2" http-body-util = "0.1" http_api_utils = { git = "https://github.com/grandinetech/grandine", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } -im = "15" k256 = "0.13" rec_aggregation = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } backend = { git = "https://github.com/leanEthereum/leanVM.git", rev = "e2592df4e30fdddbbf8ae26a333116c68cec7026" } @@ -299,9 +298,7 @@ snap = "1.1" ssz = { git = "https://github.com/grandinetech/grandine", package = "ssz", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } ssz-types = "0.3" itertools = "0.14" -tap = "1" tempfile = "3" -test-case = "3" test-generator = "0.3.1" thiserror = "2" tikv-jemallocator = { version = "0.6", features = ["stats", "unprefixed_malloc_on_supported_platforms"] } @@ -314,7 +311,6 @@ tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } tree-hash = "0.4.0" try_from_iterator = { git = "https://github.com/grandinetech/grandine", package = "try_from_iterator", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } typenum = "1.19" -unwrap_none = "0.1" yamux = "0.12" zeroize = "1.8" zstd = "0.13.3" diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index cba84a52..19436339 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -47,7 +47,7 @@ impl Storage { shrink_threshold: None, page_size: None, }) - .open_with_permissions(base, DatabaseMode::ReadWrite.permissions())?, + .open_with_permissions(base, 0o600)?, ); let txn = environment.begin_rw_txn()?; @@ -297,20 +297,20 @@ fn slot_of(key: &[u8]) -> u64 { u64::from_be_bytes(slot) } -pub struct Database { +struct Database { name: String, compression: Compression, } impl Database { - pub fn new(name: &str, compression: Compression) -> Self { + fn new(name: &str, compression: Compression) -> Self { Self { name: name.to_owned(), compression, } } - pub fn get( + fn get( &self, txn: &Transaction, key: impl AsRef<[u8]>, @@ -322,7 +322,7 @@ impl Database { .transpose() } - pub fn put( + fn put( &self, txn: &Transaction, key: impl AsRef<[u8]>, @@ -334,7 +334,7 @@ impl Database { Ok(()) } - pub fn for_each( + fn for_each( &self, txn: &Transaction, mut f: impl FnMut(&[u8], &[u8]) -> Result, @@ -352,7 +352,7 @@ impl Database { Ok(()) } - pub fn delete_batch( + fn delete_batch( &self, txn: &Transaction, keys: impl IntoIterator>, @@ -373,7 +373,7 @@ impl Database { } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum Compression { +enum Compression { None, #[default] Lz4, @@ -398,39 +398,6 @@ impl Compression { } } -#[derive(Clone, Copy)] -pub enum DatabaseMode { - ReadOnly, - ReadWrite, -} - -impl DatabaseMode { - #[must_use] - pub const fn is_read_only(self) -> bool { - matches!(self, Self::ReadOnly) - } - - #[must_use] - pub const fn mode_permissions(self) -> u16 { - match self { - Self::ReadOnly => 0, - Self::ReadWrite => 0o600, - } - } - - #[must_use] - #[cfg(target_os = "linux")] - pub fn permissions(self) -> u32 { - self.mode_permissions().into() - } - - #[must_use] - #[cfg(not(target_os = "linux"))] - pub const fn permissions(self) -> u16 { - self.mode_permissions() - } -} - #[cfg(test)] mod tests { use super::*; From fd53cdf0f5ffc96b29968433ef5474f4501560df Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Thu, 9 Jul 2026 04:37:19 +0530 Subject: [PATCH 12/20] chore: refinement --- lean_client/Cargo.lock | 2 + lean_client/fork_choice/Cargo.toml | 1 + lean_client/fork_choice/src/handlers.rs | 7 + lean_client/fork_choice/src/store.rs | 27 ++ .../tests/fork_choice_test_vectors.rs | 15 +- .../fork_choice/tests/unit_tests/common.rs | 16 +- .../fork_choice/tests/unit_tests/validator.rs | 6 +- lean_client/http_api/Cargo.toml | 1 + lean_client/http_api/src/test_driver.rs | 12 +- lean_client/src/main.rs | 9 +- lean_client/storage/src/lib.rs | 241 +++--------------- 11 files changed, 127 insertions(+), 210 deletions(-) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index b52568f7..eaf4ec37 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -1927,6 +1927,7 @@ dependencies = [ "serde_json", "spec_test_fixtures", "ssz", + "storage", "test-generator", "tracing", "xmss", @@ -2430,6 +2431,7 @@ dependencies = [ "serde_json", "spec_test_fixtures", "ssz", + "storage", "test-generator", "tokio", "tower", diff --git a/lean_client/fork_choice/Cargo.toml b/lean_client/fork_choice/Cargo.toml index e80e2f4c..bd64043e 100644 --- a/lean_client/fork_choice/Cargo.toml +++ b/lean_client/fork_choice/Cargo.toml @@ -11,6 +11,7 @@ indexmap = { workspace = true } metrics = { workspace = true } parking_lot = { workspace = true } ssz = { workspace = true } +storage = { workspace = true } tracing = { workspace = true } xmss = { workspace = true } diff --git a/lean_client/fork_choice/src/handlers.rs b/lean_client/fork_choice/src/handlers.rs index e6fcf7fd..a4da81e6 100644 --- a/lean_client/fork_choice/src/handlers.rs +++ b/lean_client/fork_choice/src/handlers.rs @@ -727,6 +727,13 @@ pub fn apply_verified_block( store.blocks.insert(block_root, block.clone()); store.states.insert(block_root, Arc::clone(&new_state)); + + store.storage.batch_write(|| { + store.storage.put_block(block.clone(), block_root)?; + store.storage.put_state(new_state.as_ref().clone(), block_root)?; + Ok(()) + }).expect("database write failed"); + METRICS.get().map(|m| { m.grandine_store_blocks_size.set(store.blocks.len() as i64); m.grandine_store_states_size.set(store.states.len() as i64); diff --git a/lean_client/fork_choice/src/store.rs b/lean_client/fork_choice/src/store.rs index 6951b9c2..8cc3f352 100644 --- a/lean_client/fork_choice/src/store.rs +++ b/lean_client/fork_choice/src/store.rs @@ -9,6 +9,7 @@ use containers::{ use indexmap::IndexMap; use metrics::{METRICS, set_gauge_u64}; use ssz::{H256, SszHash}; +use storage::Storage; use tracing::{info, warn}; use xmss::Signature; @@ -108,6 +109,8 @@ pub struct Store { pub pending_fetch_roots: HashSet, pub log_inv_rate: usize, + + pub storage: Arc, } const JUSTIFICATION_LOOKBACK_SLOTS: u64 = 3; @@ -133,6 +136,15 @@ pub const STATES_TO_KEEP: usize = 3_000; pub const HEAD_RETENTION_SLOTS: u64 = 128; impl Store { + pub fn get_block(&self, root: H256) -> Option { + if let Some(block) = self.blocks.get(&root) { + return Some(block.clone()); + } + self.storage + .get_block(root) + .expect("failed to read block from database") + } + pub fn produce_attestation_data(&self, slot: Slot) -> Result { let head_checkpoint = Checkpoint { root: self.head, @@ -224,6 +236,7 @@ pub fn get_forkchoice_store( config: Config, is_aggregator: bool, log_inv_rate: usize, + storage: Arc, ) -> Store { // Extract the plain Block from the signed block let block = anchor_block.block.clone(); @@ -261,6 +274,14 @@ pub fn get_forkchoice_store( let latest_justified = anchor_checkpoint.clone(); let latest_finalized = anchor_checkpoint; + storage.batch_write(|| { + storage.put_block(block.clone(), block_root)?; + storage.put_state(anchor_state.clone(), block_root)?; + storage.put_finalized_checkpoint(latest_finalized.clone())?; + storage.put_head_root(block_root)?; + Ok(()) + }).expect("database write failed"); + // Store the original anchor_state - do NOT modify it // Modifying checkpoints would change its hash_tree_root(), breaking the // consistency with block.state_root @@ -294,6 +315,7 @@ pub fn get_forkchoice_store( pending_aggregated_attestations: HashMap::new(), pending_fetch_roots: HashSet::new(), log_inv_rate, + storage, } } @@ -403,6 +425,7 @@ pub fn update_head(store: &mut Store) { // Compute new head using LMD-GHOST from latest justified root let new_head = get_fork_choice_head(store, store.latest_justified.root, &latest_votes, 0); store.head = new_head; + store.storage.put_head_root(new_head).expect("failed to persist head"); if let Some(head_state) = store.states.get(&new_head) { let finalized_slot = head_state.latest_finalized.slot; @@ -423,6 +446,10 @@ pub fn update_head(store: &mut Store) { root: finalized_root, slot: finalized_slot, }; + store.storage.put_finalized_checkpoint(Checkpoint { + root: finalized_root, + slot: finalized_slot, + }).expect("failed to write to database"); store.finalized_ever_updated = true; METRICS.get().map(|m| { if let Ok(s) = i64::try_from(finalized_slot.0) { diff --git a/lean_client/fork_choice/tests/fork_choice_test_vectors.rs b/lean_client/fork_choice/tests/fork_choice_test_vectors.rs index 514e3e2b..1797b197 100644 --- a/lean_client/fork_choice/tests/fork_choice_test_vectors.rs +++ b/lean_client/fork_choice/tests/fork_choice_test_vectors.rs @@ -13,11 +13,23 @@ use spec_test_fixtures::fork_choice::{ForkChoiceStep, StoreChecks}; use serde::Deserialize; use ssz::{BitList, H256, SszHash}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::{collections::HashMap, fs::File}; use std::{panic::AssertUnwindSafe, path::Path}; +use storage::Storage; use test_generator::test_resources; use xmss::PublicKey; +/// Open a throwaway `Storage` in a unique temp directory, one libmdbx +/// environment per store so parallel test cases never share a path. +fn test_storage() -> Arc { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("lean-test-vectors-{}-{n}", std::process::id())); + Arc::new(Storage::new(dir).expect("failed to open test storage")) +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct TestCase { @@ -469,7 +481,8 @@ fn forkchoice(spec_file: &str) { body_root, }; - let mut store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1); + let mut store = + get_forkchoice_store(anchor_state, anchor_block, config, false, 1, test_storage()); let mut cache = BlockCache::new(); let mut block_labels: HashMap = HashMap::new(); diff --git a/lean_client/fork_choice/tests/unit_tests/common.rs b/lean_client/fork_choice/tests/unit_tests/common.rs index c9c71da2..8dfb7678 100644 --- a/lean_client/fork_choice/tests/unit_tests/common.rs +++ b/lean_client/fork_choice/tests/unit_tests/common.rs @@ -1,6 +1,20 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + use containers::{Block, BlockBody, Config, SignedBlock, Slot, State, Validator}; use fork_choice::store::{Store, get_forkchoice_store}; use ssz::{H256, SszHash}; +use storage::Storage; + +/// Open a throwaway `Storage` in a unique temp directory. Each store gets its +/// own libmdbx environment so tests running in parallel never open the same +/// path twice within a process. +pub fn test_storage() -> Arc { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("lean-test-{}-{n}", std::process::id())); + Arc::new(Storage::new(dir).expect("failed to open test storage")) +} pub fn create_test_store() -> Store { let config = Config { genesis_time: 1000 }; @@ -22,5 +36,5 @@ pub fn create_test_store() -> Store { proof: Default::default(), }; - get_forkchoice_store(state, signed_block, config, true, 1) + get_forkchoice_store(state, signed_block, config, true, 1, test_storage()) } diff --git a/lean_client/fork_choice/tests/unit_tests/validator.rs b/lean_client/fork_choice/tests/unit_tests/validator.rs index 97e5f37d..a6651ecf 100644 --- a/lean_client/fork_choice/tests/unit_tests/validator.rs +++ b/lean_client/fork_choice/tests/unit_tests/validator.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; -use crate::unit_tests::common::create_test_store; +use crate::unit_tests::common::{create_test_store, test_storage}; use containers::{ AggregatedSignatureProof, AggregationBits, Attestation, AttestationData, Block, BlockBody, Checkpoint, Config, MultiMessageAggregate, SignedBlock, Slot, State, Validator, @@ -122,7 +122,7 @@ fn create_test_store_with_signers() -> (Store, HashMap) { }; ( - get_forkchoice_store(state, signed_block, config, true, 1), + get_forkchoice_store(state, signed_block, config, true, 1, test_storage()), keys, ) } @@ -514,7 +514,7 @@ fn test_validator_operations_empty_store() { proof: Default::default(), }; - let mut store = get_forkchoice_store(state, signed_block, config, true, 1); + let mut store = get_forkchoice_store(state, signed_block, config, true, 1, test_storage()); // Should be able to produce block and attestation let (_root, block, _sig) = diff --git a/lean_client/http_api/Cargo.toml b/lean_client/http_api/Cargo.toml index 42627d89..2413ba92 100644 --- a/lean_client/http_api/Cargo.toml +++ b/lean_client/http_api/Cargo.toml @@ -18,6 +18,7 @@ serde = { workspace = true } serde_json = { workspace = true } spec_test_fixtures = { workspace = true } ssz = { workspace = true } +storage = { workspace = true } tokio = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } diff --git a/lean_client/http_api/src/test_driver.rs b/lean_client/http_api/src/test_driver.rs index afa6a4ab..d8616262 100644 --- a/lean_client/http_api/src/test_driver.rs +++ b/lean_client/http_api/src/test_driver.rs @@ -12,6 +12,7 @@ //! `simulators/lean/src/scenarios/spec_assets.rs`. use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use axum::{ Json, Router, body::Bytes, extract::State as AxumState, http::StatusCode, routing::post, @@ -32,6 +33,7 @@ use spec_test_fixtures::{ VerifySignaturesTestCase, }; use ssz::SszHash; +use storage::Storage; use xmss::{AggregatedSignature, Signature}; /// Shared state for test-driver routes. Carries a writable handle to the @@ -179,7 +181,15 @@ async fn init_fork_choice( genesis_time: anchor_state.config.genesis_time, }; - let new_store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1); + // Each fixture drives a fresh fork-choice store, so give it its own + // throwaway libmdbx environment in a unique temp directory. + let storage = { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("lean-test-driver-{}-{n}", std::process::id())); + Arc::new(Storage::new(dir).expect("failed to open test-driver storage")) + }; + let new_store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1, storage); *state.store.write() = new_store; *state.cache.write() = BlockCache::new(); diff --git a/lean_client/src/main.rs b/lean_client/src/main.rs index 882c2be2..fdb3b8a7 100644 --- a/lean_client/src/main.rs +++ b/lean_client/src/main.rs @@ -35,11 +35,12 @@ use networking::types::{ }; use parking_lot::{Mutex, RwLock}; use ssz::{PersistentList, SszHash, SszReadDefault as _}; -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use std::{io::IsTerminal, net::IpAddr}; +use storage::Storage; use tokio::{ sync::{Notify, mpsc, oneshot, watch}, task, @@ -376,6 +377,9 @@ struct Args { #[arg(short, long)] genesis: Option, + #[arg(long, default_value = "data")] + data_dir: PathBuf, + #[arg(long)] node_id: Option, @@ -447,6 +451,8 @@ async fn main() -> Result<()> { feature.enable(); } + let storage = Arc::new(Storage::new(args.data_dir)?); + let metrics = if args.metrics_config.enabled() { let metrics = Metrics::new()?; metrics.register_with_default_metrics()?; @@ -1020,6 +1026,7 @@ async fn main() -> Result<()> { config.clone(), args.is_aggregator, genesis_log_inv_rate as usize, + storage, ))); // Seed the block provider so we can serve the anchor block to peers via BlocksByRoot. diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 19436339..450dead0 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, cell::RefCell, collections::HashSet, path::Path, sync::Arc}; +use std::{borrow::Cow, collections::HashSet, path::Path, sync::{Arc, Mutex}}; use anyhow::Result; use bytesize::ByteSize; @@ -12,25 +12,20 @@ const BLOCKS_TABLE_NAME: &str = "blocks"; const STATES_TABLE_NAME: &str = "states"; const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; -const STATE_ROOT_INDEX_TABLE_NAME: &str = "state_root_index"; const BLOCKS_BY_SLOT_TABLE_NAME: &str = "blocks_by_slot"; -const STATES_BY_SLOT_TABLE_NAME: &str = "states_by_slot"; -const JUSTIFIED: u8 = 0; const FINALIZED: u8 = 1; const HEAD: u8 = 2; -const GENESIS_TIME: u8 = 3; +#[derive(Debug)] pub struct Storage { environment: Arc, - transaction: RefCell>>, + transaction: Mutex>>, blocks: Database, states: Database, checkpoints: Database, slot_index: Database, - state_root_index: Database, blocks_by_slot: Database, - states_by_slot: Database, } impl Storage { @@ -40,7 +35,7 @@ impl Storage { let environment = Arc::new( Environment::builder() - .set_max_dbs(7) + .set_max_dbs(5) .set_geometry(Geometry { size: Some(..usize::try_from(ByteSize::gib(2).as_u64())?), growth_step: Some(isize::try_from(ByteSize::mib(256).as_u64())?), @@ -56,9 +51,7 @@ impl Storage { STATES_TABLE_NAME, CHECKPOINTS_TABLE_NAME, SLOT_INDEX_TABLE_NAME, - STATE_ROOT_INDEX_TABLE_NAME, BLOCKS_BY_SLOT_TABLE_NAME, - STATES_BY_SLOT_TABLE_NAME, ] { txn.create_db(Some(name), DatabaseFlags::default())?; } @@ -69,23 +62,22 @@ impl Storage { states: Database::new(STATES_TABLE_NAME, Compression::Zstd), checkpoints: Database::new(CHECKPOINTS_TABLE_NAME, Compression::None), slot_index: Database::new(SLOT_INDEX_TABLE_NAME, Compression::None), - state_root_index: Database::new(STATE_ROOT_INDEX_TABLE_NAME, Compression::None), blocks_by_slot: Database::new(BLOCKS_BY_SLOT_TABLE_NAME, Compression::None), - states_by_slot: Database::new(STATES_BY_SLOT_TABLE_NAME, Compression::None), environment, - transaction: RefCell::new(None), + transaction: Mutex::new(None), }) } pub fn batch_write(&self, f: impl FnOnce() -> Result<()>) -> Result<()> { let txn = self.environment.begin_rw_txn()?; - *self.transaction.borrow_mut() = Some(txn); + *self.transaction.lock().expect("transaction mutex poisoned") = Some(txn); let result = f(); let txn = self .transaction - .borrow_mut() + .lock() + .expect("transaction mutex poisoned") .take() .expect("batch_write installed the ambient transaction"); @@ -102,9 +94,11 @@ impl Storage { } fn write(&self, f: impl FnOnce(&Transaction) -> Result) -> Result { - if let Some(txn) = self.transaction.borrow().as_ref() { + let guard = self.transaction.lock().expect("transaction mutex poisoned"); + if let Some(txn) = guard.as_ref() { return f(txn); } + drop(guard); let txn = self.environment.begin_rw_txn()?; let result = f(&txn)?; txn.commit()?; @@ -112,59 +106,44 @@ impl Storage { } fn read_bytes(&self, db: &Database, key: impl AsRef<[u8]>) -> Result>> { - if let Some(txn) = self.transaction.borrow().as_ref() { + let guard = self.transaction.lock().expect("transaction mutex poisoned"); + if let Some(txn) = guard.as_ref() { return db.get(txn, key); } + drop(guard); let txn = self.environment.begin_ro_txn()?; let bytes = db.get(&txn, key)?; txn.commit()?; Ok(bytes) } - pub fn get_block(&self, root: H256) -> Result> { - match self.read_bytes(&self.blocks, root)? { + pub fn get_block(&self, block_root: H256) -> Result> { + match self.read_bytes(&self.blocks, block_root)? { Some(bytes) => Ok(Some(Block::from_ssz_default(&bytes)?)), None => Ok(None), } } - pub fn put_block(&self, block: Block, root: H256) -> Result<()> { + pub fn put_block(&self, block: Block, block_root: H256) -> Result<()> { let block_bytes = block.to_ssz()?; - let index_key = slot_root_key(block.slot, root); + let index_key = slot_root_key(block.slot, block_root); self.write(|txn| { - self.blocks.put(txn, root, block_bytes)?; + self.blocks.put(txn, block_root, block_bytes)?; self.blocks_by_slot.put(txn, index_key, b"")?; Ok(()) }) } - pub fn get_state(&self, root: H256) -> Result> { - match self.read_bytes(&self.states, root)? { + pub fn get_state(&self, block_root: H256) -> Result> { + match self.read_bytes(&self.states, block_root)? { Some(bytes) => Ok(Some(State::from_ssz_default(&bytes)?)), None => Ok(None), } } - pub fn put_state(&self, state: State, root: H256) -> Result<()> { + pub fn put_state(&self, state: State, block_root: H256) -> Result<()> { let state_bytes = state.to_ssz()?; - let index_key = slot_root_key(state.slot, root); - self.write(|txn| { - self.states.put(txn, root, state_bytes)?; - self.states_by_slot.put(txn, index_key, b"")?; - Ok(()) - }) - } - - pub fn get_justified_checkpoint(&self) -> Result> { - match self.read_bytes(&self.checkpoints, [JUSTIFIED])? { - Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), - None => Ok(None), - } - } - - pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { - let checkpoint_bytes = checkpoint.to_ssz()?; - self.write(|txn| self.checkpoints.put(txn, [JUSTIFIED], checkpoint_bytes)) + self.write(|txn| self.states.put(txn, block_root, state_bytes)) } pub fn get_finalized_checkpoint(&self) -> Result> { @@ -202,36 +181,10 @@ impl Storage { self.write(|txn| self.slot_index.put(txn, slot_bytes, root)) } - pub fn get_block_root_by_state_root(&self, state_root: H256) -> Result> { - match self.read_bytes(&self.state_root_index, state_root)? { - Some(bytes) => Ok(Some(H256::from_slice(&bytes))), - None => Ok(None), - } - } - - pub fn put_block_root_by_state_root(&self, state_root: H256, block_root: H256) -> Result<()> { - self.write(|txn| self.state_root_index.put(txn, state_root, block_root)) - } - - pub fn get_genesis_time(&self) -> Result> { - match self.read_bytes(&self.checkpoints, [GENESIS_TIME])? { - Some(bytes) => Ok(Some(u64::from_ssz_default(&bytes)?)), - None => Ok(None), - } - } - - pub fn put_genesis_time(&self, genesis_time: u64) -> Result<()> { - let genesis_time_bytes = genesis_time.to_ssz()?; - self.write(|txn| { - self.checkpoints - .put(txn, [GENESIS_TIME], genesis_time_bytes) - }) - } - pub fn prune_before_slot(&self, slot: Slot, keep_roots: &HashSet) -> Result { let bound = slot.0; self.write(|txn| { - let mut block_roots: Vec = Vec::new(); + let mut roots: Vec = Vec::new(); let mut block_index_keys: Vec> = Vec::new(); self.blocks_by_slot.for_each(txn, |key, _value| { if slot_of(key) >= bound { @@ -239,26 +192,12 @@ impl Storage { } let root = H256::from_slice(&key[8..]); if !keep_roots.contains(&root) { - block_roots.push(root); + roots.push(root); block_index_keys.push(key.to_vec()); } Ok(true) })?; - let mut state_roots: Vec = Vec::new(); - let mut state_index_keys: Vec> = Vec::new(); - self.states_by_slot.for_each(txn, |key, _value| { - if slot_of(key) >= bound { - return Ok(false); - } - let root = H256::from_slice(&key[8..]); - if !keep_roots.contains(&root) { - state_roots.push(root); - state_index_keys.push(key.to_vec()); - } - Ok(true) - })?; - let mut slot_index_keys: Vec> = Vec::new(); self.slot_index.for_each(txn, |key, value| { if slot_of(key) >= bound { @@ -272,18 +211,22 @@ impl Storage { })?; let mut removed = 0; - removed += self.blocks.delete_batch(txn, &block_roots)?; + removed += self.blocks.delete_batch(txn, &roots)?; + removed += self.states.delete_batch(txn, &roots)?; removed += self.blocks_by_slot.delete_batch(txn, &block_index_keys)?; removed += self.slot_index.delete_batch(txn, &slot_index_keys)?; - removed += self.states.delete_batch(txn, &state_roots)?; - removed += self.states_by_slot.delete_batch(txn, &state_index_keys)?; - removed += self.state_root_index.delete_batch(txn, &state_roots)?; Ok(removed) }) } } +impl Default for Storage { + fn default() -> Self { + Self::new("./database").expect("failed to open default storage at ./database") + } +} + fn slot_root_key(slot: Slot, root: H256) -> [u8; 40] { let mut key = [0u8; 40]; key[..8].copy_from_slice(&slot.0.to_be_bytes()); @@ -297,6 +240,7 @@ fn slot_of(key: &[u8]) -> u64 { u64::from_be_bytes(slot) } +#[derive(Debug)] struct Database { name: String, compression: Compression, @@ -529,25 +473,6 @@ mod tests { } } - #[test] - fn justified_checkpoint_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_justified_checkpoint().unwrap().is_none()); - - let checkpoint = Checkpoint { - root: h256(4), - slot: Slot(10), - }; - storage - .put_justified_checkpoint(checkpoint.clone()) - .unwrap(); - - assert_eq!( - storage.get_justified_checkpoint().unwrap().unwrap(), - checkpoint - ); - } - #[test] fn finalized_checkpoint_roundtrips() { let (_dir, storage) = storage(); @@ -578,45 +503,23 @@ mod tests { assert_eq!(storage.get_head_root().unwrap().unwrap(), root); } - #[test] - fn genesis_time_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_genesis_time().unwrap().is_none()); - - storage.put_genesis_time(1_700_000_000).unwrap(); - - assert_eq!(storage.get_genesis_time().unwrap().unwrap(), 1_700_000_000); - } - #[test] fn checkpoints_database_keys_do_not_collide() { let (_dir, storage) = storage(); - let justified = Checkpoint { - root: h256(1), - slot: Slot(1), - }; let finalized = Checkpoint { root: h256(2), slot: Slot(2), }; let head = h256(3); - let genesis_time = 42; - storage.put_justified_checkpoint(justified.clone()).unwrap(); storage.put_finalized_checkpoint(finalized.clone()).unwrap(); storage.put_head_root(head).unwrap(); - storage.put_genesis_time(genesis_time).unwrap(); - assert_eq!( - storage.get_justified_checkpoint().unwrap().unwrap(), - justified - ); assert_eq!( storage.get_finalized_checkpoint().unwrap().unwrap(), finalized ); assert_eq!(storage.get_head_root().unwrap().unwrap(), head); - assert_eq!(storage.get_genesis_time().unwrap().unwrap(), genesis_time); } #[test] @@ -657,59 +560,6 @@ mod tests { } } - #[test] - fn block_root_by_state_root_returns_none_when_absent() { - let (_dir, storage) = storage(); - assert!( - storage - .get_block_root_by_state_root(h256(7)) - .unwrap() - .is_none() - ); - } - - #[test] - fn block_root_by_state_root_roundtrips() { - let (_dir, storage) = storage(); - let state_root = h256(7); - let block_root = h256(8); - storage - .put_block_root_by_state_root(state_root, block_root) - .unwrap(); - - assert_eq!( - storage - .get_block_root_by_state_root(state_root) - .unwrap() - .unwrap(), - block_root, - ); - } - - #[test] - fn block_root_by_state_root_handles_multiple() { - let (_dir, storage) = storage(); - let pairs: Vec<(H256, H256)> = (0..10) - .map(|i| (h256(i), h256(i.wrapping_add(100)))) - .collect(); - - for (state_root, block_root) in &pairs { - storage - .put_block_root_by_state_root(*state_root, *block_root) - .unwrap(); - } - - for (state_root, block_root) in &pairs { - assert_eq!( - storage - .get_block_root_by_state_root(*state_root) - .unwrap() - .unwrap(), - *block_root, - ); - } - } - fn state_at_slot(slot: u64) -> State { let mut state = State::generate_genesis(1_234, 3); state.slot = Slot(slot); @@ -790,35 +640,20 @@ mod tests { #[test] fn prune_removes_states_below_slot_and_keeps_frozen() { let (_dir, storage) = storage(); + storage.put_block(sample_block(1), h256(1)).unwrap(); storage.put_state(state_at_slot(1), h256(1)).unwrap(); + storage.put_block(sample_block(2), h256(2)).unwrap(); storage.put_state(state_at_slot(2), h256(2)).unwrap(); - storage - .put_block_root_by_state_root(h256(1), h256(101)) - .unwrap(); - storage - .put_block_root_by_state_root(h256(2), h256(102)) - .unwrap(); let mut keep = HashSet::new(); keep.insert(h256(2)); storage.prune_before_slot(Slot(5), &keep).unwrap(); + assert!(storage.get_block(h256(1)).unwrap().is_none()); assert!(storage.get_state(h256(1)).unwrap().is_none()); + assert!(storage.get_block(h256(2)).unwrap().is_some()); assert!(storage.get_state(h256(2)).unwrap().is_some()); - assert!( - storage - .get_block_root_by_state_root(h256(1)) - .unwrap() - .is_none() - ); - assert_eq!( - storage - .get_block_root_by_state_root(h256(2)) - .unwrap() - .unwrap(), - h256(102), - ); } #[test] From e5f57c93c28a6855f4cd8e657a90f5ac69fcdbdd Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Thu, 9 Jul 2026 05:00:45 +0530 Subject: [PATCH 13/20] feat: add more storage functions --- lean_client/storage/src/lib.rs | 317 ++++++++++++++++++++++++++++++++- 1 file changed, 316 insertions(+), 1 deletion(-) diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 450dead0..341065d9 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -13,9 +13,17 @@ const STATES_TABLE_NAME: &str = "states"; const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; const BLOCKS_BY_SLOT_TABLE_NAME: &str = "blocks_by_slot"; +const METADATA_TABLE_NAME: &str = "metadata"; +const JUSTIFIED: u8 = 0; const FINALIZED: u8 = 1; const HEAD: u8 = 2; +const SAFE_TARGET: u8 = 3; + +const SCHEMA_VERSION: u8 = 0; +const NETWORK_ID: u8 = 1; +const JUSTIFIED_EVER_UPDATED: u8 = 2; +const FINALIZED_EVER_UPDATED: u8 = 3; #[derive(Debug)] pub struct Storage { @@ -26,6 +34,7 @@ pub struct Storage { checkpoints: Database, slot_index: Database, blocks_by_slot: Database, + metadata: Database, } impl Storage { @@ -35,7 +44,7 @@ impl Storage { let environment = Arc::new( Environment::builder() - .set_max_dbs(5) + .set_max_dbs(6) .set_geometry(Geometry { size: Some(..usize::try_from(ByteSize::gib(2).as_u64())?), growth_step: Some(isize::try_from(ByteSize::mib(256).as_u64())?), @@ -52,6 +61,7 @@ impl Storage { CHECKPOINTS_TABLE_NAME, SLOT_INDEX_TABLE_NAME, BLOCKS_BY_SLOT_TABLE_NAME, + METADATA_TABLE_NAME, ] { txn.create_db(Some(name), DatabaseFlags::default())?; } @@ -63,6 +73,7 @@ impl Storage { checkpoints: Database::new(CHECKPOINTS_TABLE_NAME, Compression::None), slot_index: Database::new(SLOT_INDEX_TABLE_NAME, Compression::None), blocks_by_slot: Database::new(BLOCKS_BY_SLOT_TABLE_NAME, Compression::None), + metadata: Database::new(METADATA_TABLE_NAME, Compression::None), environment, transaction: Mutex::new(None), }) @@ -134,6 +145,10 @@ impl Storage { }) } + pub fn has_block(&self, block_root: H256) -> Result { + Ok(self.read_bytes(&self.blocks, block_root)?.is_some()) + } + pub fn get_state(&self, block_root: H256) -> Result> { match self.read_bytes(&self.states, block_root)? { Some(bytes) => Ok(Some(State::from_ssz_default(&bytes)?)), @@ -146,6 +161,18 @@ impl Storage { self.write(|txn| self.states.put(txn, block_root, state_bytes)) } + pub fn get_justified_checkpoint(&self) -> Result> { + match self.read_bytes(&self.checkpoints, [JUSTIFIED])? { + Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), + None => Ok(None), + } + } + + pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { + let checkpoint_bytes = checkpoint.to_ssz()?; + self.write(|txn| self.checkpoints.put(txn, [JUSTIFIED], checkpoint_bytes)) + } + pub fn get_finalized_checkpoint(&self) -> Result> { match self.read_bytes(&self.checkpoints, [FINALIZED])? { Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), @@ -169,6 +196,67 @@ impl Storage { self.write(|txn| self.checkpoints.put(txn, [HEAD], root)) } + pub fn get_safe_target(&self) -> Result> { + match self.read_bytes(&self.checkpoints, [SAFE_TARGET])? { + Some(bytes) => Ok(Some(H256::from_slice(&bytes))), + None => Ok(None), + } + } + + pub fn put_safe_target(&self, root: H256) -> Result<()> { + self.write(|txn| self.checkpoints.put(txn, [SAFE_TARGET], root)) + } + + pub fn get_schema_version(&self) -> Result> { + match self.read_bytes(&self.metadata, [SCHEMA_VERSION])? { + Some(bytes) => Ok(Some(u32::from_be_bytes(bytes.as_slice().try_into()?))), + None => Ok(None), + } + } + + pub fn put_schema_version(&self, version: u32) -> Result<()> { + self.write(|txn| self.metadata.put(txn, [SCHEMA_VERSION], version.to_be_bytes())) + } + + pub fn get_network_id(&self) -> Result> { + match self.read_bytes(&self.metadata, [NETWORK_ID])? { + Some(bytes) => Ok(Some(H256::from_slice(&bytes))), + None => Ok(None), + } + } + + pub fn put_network_id(&self, network_id: H256) -> Result<()> { + self.write(|txn| self.metadata.put(txn, [NETWORK_ID], network_id)) + } + + pub fn get_justified_ever_updated(&self) -> Result> { + match self.read_bytes(&self.metadata, [JUSTIFIED_EVER_UPDATED])? { + Some(bytes) => Ok(Some(bytes.first().copied().unwrap_or(0) != 0)), + None => Ok(None), + } + } + + pub fn put_justified_ever_updated(&self, value: bool) -> Result<()> { + self.write(|txn| { + self.metadata + .put(txn, [JUSTIFIED_EVER_UPDATED], [u8::from(value)]) + }) + } + + pub fn get_finalized_ever_updated(&self) -> Result> { + match self.read_bytes(&self.metadata, [FINALIZED_EVER_UPDATED])? { + Some(bytes) => Ok(Some(bytes.first().copied().unwrap_or(0) != 0)), + None => Ok(None), + } + } + + pub fn put_finalized_ever_updated(&self, value: bool) -> Result<()> { + self.write(|txn| { + self.metadata + .put(txn, [FINALIZED_EVER_UPDATED], [u8::from(value)]) + }) + } + pub fn get_block_root_by_slot(&self, slot: Slot) -> Result> { match self.read_bytes(&self.slot_index, slot.0.to_be_bytes())? { Some(bytes) => Ok(Some(H256::from_slice(&bytes))), @@ -219,6 +307,54 @@ impl Storage { Ok(removed) }) } + + pub fn load_since(&self, slot: Slot) -> Result> { + let bound = slot.0; + let txn = self.environment.begin_ro_txn()?; + + let mut roots: Vec = Vec::new(); + self.blocks_by_slot.for_each(&txn, |key, _value| { + if slot_of(key) >= bound { + roots.push(H256::from_slice(&key[8..])); + } + Ok(true) + })?; + + let mut entries = Vec::with_capacity(roots.len()); + for root in roots { + let block = match self.blocks.get(&txn, root)? { + Some(bytes) => Block::from_ssz_default(&bytes)?, + None => continue, + }; + let state = match self.states.get(&txn, root)? { + Some(bytes) => State::from_ssz_default(&bytes)?, + None => continue, + }; + entries.push((root, block, state)); + } + + txn.commit()?; + Ok(entries) + } + + pub fn commit_finalization( + &self, + head: H256, + finalized: Checkpoint, + safe_target: H256, + prune_before: Slot, + keep_roots: &HashSet, + ) -> Result { + let mut removed = 0; + self.batch_write(|| { + self.put_head_root(head)?; + self.put_finalized_checkpoint(finalized)?; + self.put_safe_target(safe_target)?; + removed = self.prune_before_slot(prune_before, keep_roots)?; + Ok(()) + })?; + Ok(removed) + } } impl Default for Storage { @@ -503,6 +639,185 @@ mod tests { assert_eq!(storage.get_head_root().unwrap().unwrap(), root); } + #[test] + fn safe_target_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_safe_target().unwrap().is_none()); + + let root = h256(7); + storage.put_safe_target(root).unwrap(); + + assert_eq!(storage.get_safe_target().unwrap().unwrap(), root); + } + + #[test] + fn schema_version_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_schema_version().unwrap().is_none()); + + storage.put_schema_version(3).unwrap(); + + assert_eq!(storage.get_schema_version().unwrap().unwrap(), 3); + } + + #[test] + fn network_id_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_network_id().unwrap().is_none()); + + let id = h256(9); + storage.put_network_id(id).unwrap(); + + assert_eq!(storage.get_network_id().unwrap().unwrap(), id); + } + + #[test] + fn justified_checkpoint_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_justified_checkpoint().unwrap().is_none()); + + let checkpoint = Checkpoint { + root: h256(4), + slot: Slot(10), + }; + storage + .put_justified_checkpoint(checkpoint.clone()) + .unwrap(); + + assert_eq!( + storage.get_justified_checkpoint().unwrap().unwrap(), + checkpoint + ); + } + + #[test] + fn justified_ever_updated_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_justified_ever_updated().unwrap().is_none()); + + storage.put_justified_ever_updated(true).unwrap(); + assert!(storage.get_justified_ever_updated().unwrap().unwrap()); + + storage.put_justified_ever_updated(false).unwrap(); + assert!(!storage.get_justified_ever_updated().unwrap().unwrap()); + } + + #[test] + fn finalized_ever_updated_roundtrips() { + let (_dir, storage) = storage(); + assert!(storage.get_finalized_ever_updated().unwrap().is_none()); + + storage.put_finalized_ever_updated(true).unwrap(); + assert!(storage.get_finalized_ever_updated().unwrap().unwrap()); + + storage.put_finalized_ever_updated(false).unwrap(); + assert!(!storage.get_finalized_ever_updated().unwrap().unwrap()); + } + + #[test] + fn checkpoints_and_metadata_keys_do_not_collide() { + let (_dir, storage) = storage(); + let justified = Checkpoint { + root: h256(1), + slot: Slot(1), + }; + let finalized = Checkpoint { + root: h256(2), + slot: Slot(2), + }; + + storage.put_justified_checkpoint(justified.clone()).unwrap(); + storage.put_finalized_checkpoint(finalized.clone()).unwrap(); + storage.put_head_root(h256(3)).unwrap(); + storage.put_safe_target(h256(4)).unwrap(); + storage.put_schema_version(7).unwrap(); + storage.put_network_id(h256(5)).unwrap(); + storage.put_justified_ever_updated(true).unwrap(); + storage.put_finalized_ever_updated(false).unwrap(); + + assert_eq!( + storage.get_justified_checkpoint().unwrap().unwrap(), + justified + ); + assert_eq!( + storage.get_finalized_checkpoint().unwrap().unwrap(), + finalized + ); + assert_eq!(storage.get_head_root().unwrap().unwrap(), h256(3)); + assert_eq!(storage.get_safe_target().unwrap().unwrap(), h256(4)); + assert_eq!(storage.get_schema_version().unwrap().unwrap(), 7); + assert_eq!(storage.get_network_id().unwrap().unwrap(), h256(5)); + assert!(storage.get_justified_ever_updated().unwrap().unwrap()); + assert!(!storage.get_finalized_ever_updated().unwrap().unwrap()); + } + + #[test] + fn has_block_reflects_presence() { + let (_dir, storage) = storage(); + assert!(!storage.has_block(h256(1)).unwrap()); + + storage.put_block(sample_block(1), h256(1)).unwrap(); + + assert!(storage.has_block(h256(1)).unwrap()); + assert!(!storage.has_block(h256(2)).unwrap()); + } + + #[test] + fn load_since_returns_blocks_and_states_from_slot() { + let (_dir, storage) = storage(); + for i in 0..6u8 { + storage.put_block(sample_block(i), h256(i)).unwrap(); + storage + .put_state(state_at_slot(u64::from(i)), h256(i)) + .unwrap(); + } + + let mut loaded = storage.load_since(Slot(3)).unwrap(); + loaded.sort_by_key(|(_, block, _)| block.slot.0); + + let slots: Vec = loaded.iter().map(|(_, block, _)| block.slot.0).collect(); + assert_eq!(slots, vec![3, 4, 5]); + + for (root, block, state) in loaded { + assert_eq!(root, h256(u8::try_from(block.slot.0).unwrap())); + assert_eq!(state.slot, block.slot); + } + } + + #[test] + fn commit_finalization_persists_and_prunes() { + let (_dir, storage) = storage(); + for i in 0..6u8 { + storage.put_block(sample_block(i), h256(i)).unwrap(); + storage + .put_state(state_at_slot(u64::from(i)), h256(i)) + .unwrap(); + } + + let finalized = Checkpoint { + root: h256(4), + slot: Slot(4), + }; + let mut keep = HashSet::new(); + keep.insert(h256(4)); + + let removed = storage + .commit_finalization(h256(4), finalized.clone(), h256(5), Slot(4), &keep) + .unwrap(); + + assert!(removed > 0); + assert_eq!(storage.get_head_root().unwrap().unwrap(), h256(4)); + assert_eq!( + storage.get_finalized_checkpoint().unwrap().unwrap(), + finalized + ); + assert_eq!(storage.get_safe_target().unwrap().unwrap(), h256(5)); + assert!(storage.get_block(h256(0)).unwrap().is_none()); + assert!(storage.get_state(h256(0)).unwrap().is_none()); + assert!(storage.get_block(h256(4)).unwrap().is_some()); + assert!(storage.get_block(h256(5)).unwrap().is_some()); + } + #[test] fn checkpoints_database_keys_do_not_collide() { let (_dir, storage) = storage(); From 6d8ae2fcd25e5329b32aaba8c2e203c33db203af Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Thu, 9 Jul 2026 19:48:51 +0530 Subject: [PATCH 14/20] feat: wiring up storage --- lean_client/fork_choice/src/handlers.rs | 14 +- lean_client/fork_choice/src/store.rs | 221 +++++++++++------- .../tests/unit_tests/fork_choice.rs | 42 +++- lean_client/src/main.rs | 9 + lean_client/storage/src/lib.rs | 128 +--------- 5 files changed, 199 insertions(+), 215 deletions(-) diff --git a/lean_client/fork_choice/src/handlers.rs b/lean_client/fork_choice/src/handlers.rs index a4da81e6..d57f1718 100644 --- a/lean_client/fork_choice/src/handlers.rs +++ b/lean_client/fork_choice/src/handlers.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use anyhow::{Context, Result, anyhow, bail, ensure}; use containers::{ AttestationData, Checkpoint, SignatureKey, SignedAggregatedAttestation, SignedAttestation, - SignedBlock, State, + SignedBlock, Slot, State, }; use metrics::METRICS; use parking_lot::RwLock; @@ -778,7 +778,9 @@ pub fn apply_verified_block( "Store justified checkpoint updated!" ); store.latest_justified = new_state.latest_justified.clone(); + store.storage.put_justified_checkpoint(new_state.latest_justified.clone()).expect("failed to persist justified checkpoint"); store.justified_ever_updated = true; + store.storage.put_justified_ever_updated(true).expect("failed to persist justified_ever_updated"); METRICS.get().map(|metrics| { let Some(slot) = new_state.latest_justified.slot.0.try_into().ok() else { warn!("unable to set latest_justified slot in metrics"); @@ -852,6 +854,16 @@ pub fn apply_verified_block( .saturating_sub(STATE_PRUNE_BUFFER); store.states.retain(|_, state| state.slot.0 >= keep_from); store.blocks.retain(|_, block| block.slot.0 >= keep_from); + store + .storage + .commit_finalization( + store.head, + store.latest_finalized.clone(), + store.safe_target, + Slot(keep_from), + &HashSet::new(), + ) + .expect("failed to commit finalization"); let finalized_slot = store.latest_finalized.slot.0; let adr = &store.attestation_data_by_root; diff --git a/lean_client/fork_choice/src/store.rs b/lean_client/fork_choice/src/store.rs index 8cc3f352..c4f292a1 100644 --- a/lean_client/fork_choice/src/store.rs +++ b/lean_client/fork_choice/src/store.rs @@ -136,15 +136,6 @@ pub const STATES_TO_KEEP: usize = 3_000; pub const HEAD_RETENTION_SLOTS: u64 = 128; impl Store { - pub fn get_block(&self, root: H256) -> Option { - if let Some(block) = self.blocks.get(&root) { - return Some(block.clone()); - } - self.storage - .get_block(root) - .expect("failed to read block from database") - } - pub fn produce_attestation_data(&self, slot: Slot) -> Result { let head_checkpoint = Checkpoint { root: self.head, @@ -238,84 +229,144 @@ pub fn get_forkchoice_store( log_inv_rate: usize, storage: Arc, ) -> Store { - // Extract the plain Block from the signed block - let block = anchor_block.block.clone(); - let block_slot = block.slot; - - // Compute block root differently for genesis vs checkpoint sync: - // - Genesis (slot 0): Use block.hash_tree_root() directly — block and state are consistent. - // - Checkpoint sync (slot > 0): Reconstruct BlockHeader from state.latest_block_header, - // using anchor_state.hash_tree_root() as state_root. This guarantees the root stored - // as the key in store.blocks / store.states is the canonical one committed to by the - // downloaded state, independent of what the real block's state_root field contains. - let block_root = if block_slot.0 == 0 { - block.hash_tree_root() + if let Some(head_root) = storage + .get_head_root() + .expect("failed to read head root from database") + { + let latest_finalized = storage + .get_finalized_checkpoint() + .expect("failed to read finalized checkpoint from database") + .unwrap_or_default(); + let latest_justified = storage + .get_justified_checkpoint() + .expect("failed to read justified checkpoint from database") + .unwrap_or_default(); + let safe_target = storage + .get_safe_target() + .expect("failed to read safe target from database") + .unwrap_or_default(); + let justified_ever_updated = storage + .get_justified_ever_updated() + .expect("failed to read justified_ever_updated from database") + .unwrap_or(false); + let finalized_ever_updated = storage + .get_finalized_ever_updated() + .expect("failed to read finalized_ever_updated from database") + .unwrap_or(false); + + let keep_from = latest_finalized.slot.0.saturating_sub(STATE_PRUNE_BUFFER); + let mut blocks = HashMap::new(); + let mut states = HashMap::new(); + for (root, block, state) in storage + .load_since(Slot(keep_from)) + .expect("failed to load chain from database") + { + blocks.insert(root, block); + states.insert(root, Arc::new(state)); + } + + let head = if blocks.contains_key(&head_root) { + head_root + } else { + latest_finalized.root + }; + let head_slot = blocks + .get(&head) + .map(|block| block.slot.0) + .unwrap_or(latest_finalized.slot.0); + + Store { + time: head_slot * INTERVALS_PER_SLOT, + config, + is_aggregator, + head, + safe_target, + latest_justified, + latest_finalized, + justified_ever_updated, + finalized_ever_updated, + blocks, + states, + latest_known_attestations: HashMap::new(), + latest_new_attestations: HashMap::new(), + gossip_signatures: HashMap::new(), + latest_known_aggregated_payloads: IndexMap::new(), + latest_new_aggregated_payloads: IndexMap::new(), + attestation_data_by_root: HashMap::new(), + pending_attestations: HashMap::new(), + pending_aggregated_attestations: HashMap::new(), + pending_fetch_roots: HashSet::new(), + log_inv_rate, + storage, + } } else { - let block_header = BlockHeader { - slot: anchor_state.latest_block_header.slot, - proposer_index: anchor_state.latest_block_header.proposer_index, - parent_root: anchor_state.latest_block_header.parent_root, - state_root: anchor_state.hash_tree_root(), - body_root: anchor_state.latest_block_header.body_root, + let block = anchor_block.block.clone(); + let block_slot = block.slot; + + let block_root = if block_slot.0 == 0 { + block.hash_tree_root() + } else { + let block_header = BlockHeader { + slot: anchor_state.latest_block_header.slot, + proposer_index: anchor_state.latest_block_header.proposer_index, + parent_root: anchor_state.latest_block_header.parent_root, + state_root: anchor_state.hash_tree_root(), + body_root: anchor_state.latest_block_header.body_root, + }; + block_header.hash_tree_root() }; - block_header.hash_tree_root() - }; - // Seed both checkpoints from the anchor block itself: (root=anchor_root, - // slot=anchor_slot). The store treats the anchor as the new "genesis" for - // fork choice — pre-anchor history is pruned — so the embedded checkpoints - // from the downloaded state are intentionally ignored. This keeps the - // checkpoint slot/root pair internally consistent with the block at - // anchor_root, mirroring the beacon-chain seeding convention. - let anchor_checkpoint = Checkpoint { - root: block_root, - slot: block_slot, - }; - let latest_justified = anchor_checkpoint.clone(); - let latest_finalized = anchor_checkpoint; - - storage.batch_write(|| { - storage.put_block(block.clone(), block_root)?; - storage.put_state(anchor_state.clone(), block_root)?; - storage.put_finalized_checkpoint(latest_finalized.clone())?; - storage.put_head_root(block_root)?; - Ok(()) - }).expect("database write failed"); - - // Store the original anchor_state - do NOT modify it - // Modifying checkpoints would change its hash_tree_root(), breaking the - // consistency with block.state_root - Store { - time: block_slot.0 * INTERVALS_PER_SLOT, - config, - is_aggregator, - head: block_root, - safe_target: block_root, - latest_justified, - latest_finalized, - justified_ever_updated: block_slot.0 == 0, - finalized_ever_updated: false, - blocks: { - let mut m = HashMap::new(); - m.insert(block_root, block); - m - }, - states: { - let mut m = HashMap::new(); - m.insert(block_root, Arc::new(anchor_state)); - m - }, - latest_known_attestations: HashMap::new(), - latest_new_attestations: HashMap::new(), - gossip_signatures: HashMap::new(), - latest_known_aggregated_payloads: IndexMap::new(), - latest_new_aggregated_payloads: IndexMap::new(), - attestation_data_by_root: HashMap::new(), - pending_attestations: HashMap::new(), - pending_aggregated_attestations: HashMap::new(), - pending_fetch_roots: HashSet::new(), - log_inv_rate, - storage, + let anchor_checkpoint = Checkpoint { + root: block_root, + slot: block_slot, + }; + let latest_justified = anchor_checkpoint.clone(); + let latest_finalized = anchor_checkpoint; + + storage.batch_write(|| { + storage.put_block(block.clone(), block_root)?; + storage.put_state(anchor_state.clone(), block_root)?; + storage.put_justified_checkpoint(latest_justified.clone())?; + storage.put_finalized_checkpoint(latest_finalized.clone())?; + storage.put_head_root(block_root)?; + storage.put_safe_target(block_root)?; + storage.put_justified_ever_updated(block_slot.0 == 0)?; + storage.put_finalized_ever_updated(false)?; + Ok(()) + }).expect("database write failed"); + + Store { + time: block_slot.0 * INTERVALS_PER_SLOT, + config, + is_aggregator, + head: block_root, + safe_target: block_root, + latest_justified, + latest_finalized, + justified_ever_updated: block_slot.0 == 0, + finalized_ever_updated: false, + blocks: { + let mut m = HashMap::new(); + m.insert(block_root, block); + m + }, + states: { + let mut m = HashMap::new(); + m.insert(block_root, Arc::new(anchor_state)); + m + }, + latest_known_attestations: HashMap::new(), + latest_new_attestations: HashMap::new(), + gossip_signatures: HashMap::new(), + latest_known_aggregated_payloads: IndexMap::new(), + latest_new_aggregated_payloads: IndexMap::new(), + attestation_data_by_root: HashMap::new(), + pending_attestations: HashMap::new(), + pending_aggregated_attestations: HashMap::new(), + pending_fetch_roots: HashSet::new(), + log_inv_rate, + storage, + } } } @@ -451,6 +502,7 @@ pub fn update_head(store: &mut Store) { slot: finalized_slot, }).expect("failed to write to database"); store.finalized_ever_updated = true; + store.storage.put_finalized_ever_updated(true).expect("failed to persist finalized_ever_updated"); METRICS.get().map(|m| { if let Ok(s) = i64::try_from(finalized_slot.0) { m.lean_latest_finalized_slot.set(s); @@ -592,6 +644,7 @@ pub fn update_safe_target(store: &mut Store) { // Run LMD-GHOST with 2/3 threshold to find safe target let new_safe_target = get_fork_choice_head(store, root, &attestations, min_score); store.safe_target = new_safe_target; + store.storage.put_safe_target(new_safe_target).expect("failed to persist safe target"); set_gauge_u64( |metrics| &metrics.lean_safe_target_slot, diff --git a/lean_client/fork_choice/tests/unit_tests/fork_choice.rs b/lean_client/fork_choice/tests/unit_tests/fork_choice.rs index 597f3511..67ae3718 100644 --- a/lean_client/fork_choice/tests/unit_tests/fork_choice.rs +++ b/lean_client/fork_choice/tests/unit_tests/fork_choice.rs @@ -1,6 +1,6 @@ -use super::common::create_test_store; -use containers::{Block, BlockBody, Slot}; -use fork_choice::store::get_proposal_head; +use super::common::{create_test_store, test_storage}; +use containers::{Block, BlockBody, Config, SignedBlock, Slot, State, Validator}; +use fork_choice::store::{get_forkchoice_store, get_proposal_head}; use ssz::{H256, SszHash}; #[test] @@ -53,3 +53,39 @@ fn test_get_vote_target_chain() { assert_eq!(target.slot, Slot(6)); } + +#[test] +fn get_forkchoice_store_restores_from_db() { + let storage = test_storage(); + let state = State::generate_genesis_with_validators(1000, vec![Validator::default(); 4]); + let block = Block { + slot: Slot(0), + proposer_index: 0, + parent_root: H256::default(), + state_root: state.hash_tree_root(), + body: BlockBody::default(), + }; + let signed_block = SignedBlock { + block, + proof: Default::default(), + }; + let config = Config { genesis_time: 1000 }; + + let fresh = get_forkchoice_store( + state.clone(), + signed_block.clone(), + config.clone(), + true, + 1, + storage.clone(), + ); + let head = fresh.head; + let finalized = fresh.latest_finalized.clone(); + drop(fresh); + + let restored = get_forkchoice_store(state, signed_block, config, true, 1, storage); + assert_eq!(restored.head, head); + assert!(restored.blocks.contains_key(&head)); + assert!(restored.states.contains_key(&head)); + assert_eq!(restored.latest_finalized, finalized); +} diff --git a/lean_client/src/main.rs b/lean_client/src/main.rs index fdb3b8a7..fb0365bc 100644 --- a/lean_client/src/main.rs +++ b/lean_client/src/main.rs @@ -582,6 +582,15 @@ async fn main() -> Result<()> { }, }; + let genesis_root = genesis_block.hash_tree_root(); + match storage.get_network_id()? { + Some(existing) => anyhow::ensure!( + existing == genesis_root, + "database belongs to a different network: found {existing:?}, expected {genesis_root:?}" + ), + None => storage.put_network_id(genesis_root)?, + } + let genesis_signed_block = SignedBlock { block: genesis_block, proof: MultiMessageAggregate::default(), diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 341065d9..4c509831 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -11,7 +11,6 @@ use libmdbx::{DatabaseFlags, Environment, Geometry, RW, Transaction, Transaction const BLOCKS_TABLE_NAME: &str = "blocks"; const STATES_TABLE_NAME: &str = "states"; const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; -const SLOT_INDEX_TABLE_NAME: &str = "slot_index"; const BLOCKS_BY_SLOT_TABLE_NAME: &str = "blocks_by_slot"; const METADATA_TABLE_NAME: &str = "metadata"; @@ -20,7 +19,6 @@ const FINALIZED: u8 = 1; const HEAD: u8 = 2; const SAFE_TARGET: u8 = 3; -const SCHEMA_VERSION: u8 = 0; const NETWORK_ID: u8 = 1; const JUSTIFIED_EVER_UPDATED: u8 = 2; const FINALIZED_EVER_UPDATED: u8 = 3; @@ -32,7 +30,6 @@ pub struct Storage { blocks: Database, states: Database, checkpoints: Database, - slot_index: Database, blocks_by_slot: Database, metadata: Database, } @@ -44,7 +41,7 @@ impl Storage { let environment = Arc::new( Environment::builder() - .set_max_dbs(6) + .set_max_dbs(5) .set_geometry(Geometry { size: Some(..usize::try_from(ByteSize::gib(2).as_u64())?), growth_step: Some(isize::try_from(ByteSize::mib(256).as_u64())?), @@ -59,7 +56,6 @@ impl Storage { BLOCKS_TABLE_NAME, STATES_TABLE_NAME, CHECKPOINTS_TABLE_NAME, - SLOT_INDEX_TABLE_NAME, BLOCKS_BY_SLOT_TABLE_NAME, METADATA_TABLE_NAME, ] { @@ -71,7 +67,6 @@ impl Storage { blocks: Database::new(BLOCKS_TABLE_NAME, Compression::Lz4), states: Database::new(STATES_TABLE_NAME, Compression::Zstd), checkpoints: Database::new(CHECKPOINTS_TABLE_NAME, Compression::None), - slot_index: Database::new(SLOT_INDEX_TABLE_NAME, Compression::None), blocks_by_slot: Database::new(BLOCKS_BY_SLOT_TABLE_NAME, Compression::None), metadata: Database::new(METADATA_TABLE_NAME, Compression::None), environment, @@ -145,10 +140,6 @@ impl Storage { }) } - pub fn has_block(&self, block_root: H256) -> Result { - Ok(self.read_bytes(&self.blocks, block_root)?.is_some()) - } - pub fn get_state(&self, block_root: H256) -> Result> { match self.read_bytes(&self.states, block_root)? { Some(bytes) => Ok(Some(State::from_ssz_default(&bytes)?)), @@ -207,17 +198,6 @@ impl Storage { self.write(|txn| self.checkpoints.put(txn, [SAFE_TARGET], root)) } - pub fn get_schema_version(&self) -> Result> { - match self.read_bytes(&self.metadata, [SCHEMA_VERSION])? { - Some(bytes) => Ok(Some(u32::from_be_bytes(bytes.as_slice().try_into()?))), - None => Ok(None), - } - } - - pub fn put_schema_version(&self, version: u32) -> Result<()> { - self.write(|txn| self.metadata.put(txn, [SCHEMA_VERSION], version.to_be_bytes())) - } - pub fn get_network_id(&self) -> Result> { match self.read_bytes(&self.metadata, [NETWORK_ID])? { Some(bytes) => Ok(Some(H256::from_slice(&bytes))), @@ -257,18 +237,6 @@ impl Storage { }) } - pub fn get_block_root_by_slot(&self, slot: Slot) -> Result> { - match self.read_bytes(&self.slot_index, slot.0.to_be_bytes())? { - Some(bytes) => Ok(Some(H256::from_slice(&bytes))), - None => Ok(None), - } - } - - pub fn put_block_root_by_slot(&self, slot: Slot, root: H256) -> Result<()> { - let slot_bytes = slot.0.to_be_bytes(); - self.write(|txn| self.slot_index.put(txn, slot_bytes, root)) - } - pub fn prune_before_slot(&self, slot: Slot, keep_roots: &HashSet) -> Result { let bound = slot.0; self.write(|txn| { @@ -286,23 +254,10 @@ impl Storage { Ok(true) })?; - let mut slot_index_keys: Vec> = Vec::new(); - self.slot_index.for_each(txn, |key, value| { - if slot_of(key) >= bound { - return Ok(false); - } - let canonical_root = H256::from_slice(value); - if !keep_roots.contains(&canonical_root) { - slot_index_keys.push(key.to_vec()); - } - Ok(true) - })?; - let mut removed = 0; removed += self.blocks.delete_batch(txn, &roots)?; removed += self.states.delete_batch(txn, &roots)?; removed += self.blocks_by_slot.delete_batch(txn, &block_index_keys)?; - removed += self.slot_index.delete_batch(txn, &slot_index_keys)?; Ok(removed) }) @@ -650,16 +605,6 @@ mod tests { assert_eq!(storage.get_safe_target().unwrap().unwrap(), root); } - #[test] - fn schema_version_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_schema_version().unwrap().is_none()); - - storage.put_schema_version(3).unwrap(); - - assert_eq!(storage.get_schema_version().unwrap().unwrap(), 3); - } - #[test] fn network_id_roundtrips() { let (_dir, storage) = storage(); @@ -730,7 +675,6 @@ mod tests { storage.put_finalized_checkpoint(finalized.clone()).unwrap(); storage.put_head_root(h256(3)).unwrap(); storage.put_safe_target(h256(4)).unwrap(); - storage.put_schema_version(7).unwrap(); storage.put_network_id(h256(5)).unwrap(); storage.put_justified_ever_updated(true).unwrap(); storage.put_finalized_ever_updated(false).unwrap(); @@ -745,23 +689,11 @@ mod tests { ); assert_eq!(storage.get_head_root().unwrap().unwrap(), h256(3)); assert_eq!(storage.get_safe_target().unwrap().unwrap(), h256(4)); - assert_eq!(storage.get_schema_version().unwrap().unwrap(), 7); assert_eq!(storage.get_network_id().unwrap().unwrap(), h256(5)); assert!(storage.get_justified_ever_updated().unwrap().unwrap()); assert!(!storage.get_finalized_ever_updated().unwrap().unwrap()); } - #[test] - fn has_block_reflects_presence() { - let (_dir, storage) = storage(); - assert!(!storage.has_block(h256(1)).unwrap()); - - storage.put_block(sample_block(1), h256(1)).unwrap(); - - assert!(storage.has_block(h256(1)).unwrap()); - assert!(!storage.has_block(h256(2)).unwrap()); - } - #[test] fn load_since_returns_blocks_and_states_from_slot() { let (_dir, storage) = storage(); @@ -837,44 +769,6 @@ mod tests { assert_eq!(storage.get_head_root().unwrap().unwrap(), head); } - #[test] - fn block_root_by_slot_returns_none_when_absent() { - let (_dir, storage) = storage(); - assert!(storage.get_block_root_by_slot(Slot(0)).unwrap().is_none()); - } - - #[test] - fn block_root_by_slot_roundtrips() { - let (_dir, storage) = storage(); - let root = h256(6); - storage.put_block_root_by_slot(Slot(42), root).unwrap(); - - assert_eq!( - storage.get_block_root_by_slot(Slot(42)).unwrap().unwrap(), - root - ); - } - - #[test] - fn block_root_by_slot_handles_multiple() { - let (_dir, storage) = storage(); - for i in 0..10 { - storage - .put_block_root_by_slot(Slot(u64::from(i)), h256(i)) - .unwrap(); - } - - for i in 0..10 { - assert_eq!( - storage - .get_block_root_by_slot(Slot(u64::from(i))) - .unwrap() - .unwrap(), - h256(i), - ); - } - } - fn state_at_slot(slot: u64) -> State { let mut state = State::generate_genesis(1_234, 3); state.slot = Slot(slot); @@ -932,26 +826,6 @@ mod tests { assert!(storage.get_block(h256(21)).unwrap().is_none()); } - #[test] - fn prune_cleans_slot_index_unless_frozen() { - let (_dir, storage) = storage(); - storage.put_block(sample_block(2), h256(2)).unwrap(); - storage.put_block_root_by_slot(Slot(2), h256(2)).unwrap(); - storage.put_block(sample_block(3), h256(3)).unwrap(); - storage.put_block_root_by_slot(Slot(3), h256(3)).unwrap(); - - let mut keep = HashSet::new(); - keep.insert(h256(3)); - - storage.prune_before_slot(Slot(5), &keep).unwrap(); - - assert!(storage.get_block_root_by_slot(Slot(2)).unwrap().is_none()); - assert_eq!( - storage.get_block_root_by_slot(Slot(3)).unwrap().unwrap(), - h256(3), - ); - } - #[test] fn prune_removes_states_below_slot_and_keeps_frozen() { let (_dir, storage) = storage(); From 4fc822816fb7ca5916eeacd8d4e0f9c52d051d47 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 10 Jul 2026 21:51:33 +0530 Subject: [PATCH 15/20] feat: new db on wrong network panic --- .gitignore | 1 + lean_client/fork_choice/src/handlers.rs | 26 +++++++++---- lean_client/fork_choice/src/store.rs | 50 ++++++++++++++++--------- lean_client/src/main.rs | 25 +++++++++---- lean_client/storage/src/lib.rs | 9 ++++- 5 files changed, 75 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 0ed89914..c2754f87 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ lean_client/target/* lean_client/tests/mainnet lean_client/bin lean_client/spec/ +lean_client/**/data target/* *.local* diff --git a/lean_client/fork_choice/src/handlers.rs b/lean_client/fork_choice/src/handlers.rs index d57f1718..73f6182d 100644 --- a/lean_client/fork_choice/src/handlers.rs +++ b/lean_client/fork_choice/src/handlers.rs @@ -727,12 +727,16 @@ pub fn apply_verified_block( store.blocks.insert(block_root, block.clone()); store.states.insert(block_root, Arc::clone(&new_state)); - - store.storage.batch_write(|| { - store.storage.put_block(block.clone(), block_root)?; - store.storage.put_state(new_state.as_ref().clone(), block_root)?; - Ok(()) - }).expect("database write failed"); + store + .storage + .batch_write(|| { + store.storage.put_block(block.clone(), block_root)?; + store + .storage + .put_state(new_state.as_ref().clone(), block_root)?; + Ok(()) + }) + .expect("database write failed"); METRICS.get().map(|m| { m.grandine_store_blocks_size.set(store.blocks.len() as i64); @@ -778,9 +782,15 @@ pub fn apply_verified_block( "Store justified checkpoint updated!" ); store.latest_justified = new_state.latest_justified.clone(); - store.storage.put_justified_checkpoint(new_state.latest_justified.clone()).expect("failed to persist justified checkpoint"); + store + .storage + .put_justified_checkpoint(new_state.latest_justified.clone()) + .expect("failed to persist justified checkpoint"); store.justified_ever_updated = true; - store.storage.put_justified_ever_updated(true).expect("failed to persist justified_ever_updated"); + store + .storage + .put_justified_ever_updated(true) + .expect("failed to persist justified_ever_updated"); METRICS.get().map(|metrics| { let Some(slot) = new_state.latest_justified.slot.0.try_into().ok() else { warn!("unable to set latest_justified slot in metrics"); diff --git a/lean_client/fork_choice/src/store.rs b/lean_client/fork_choice/src/store.rs index c4f292a1..d89dcc17 100644 --- a/lean_client/fork_choice/src/store.rs +++ b/lean_client/fork_choice/src/store.rs @@ -323,17 +323,19 @@ pub fn get_forkchoice_store( let latest_justified = anchor_checkpoint.clone(); let latest_finalized = anchor_checkpoint; - storage.batch_write(|| { - storage.put_block(block.clone(), block_root)?; - storage.put_state(anchor_state.clone(), block_root)?; - storage.put_justified_checkpoint(latest_justified.clone())?; - storage.put_finalized_checkpoint(latest_finalized.clone())?; - storage.put_head_root(block_root)?; - storage.put_safe_target(block_root)?; - storage.put_justified_ever_updated(block_slot.0 == 0)?; - storage.put_finalized_ever_updated(false)?; - Ok(()) - }).expect("database write failed"); + storage + .batch_write(|| { + storage.put_block(block.clone(), block_root)?; + storage.put_state(anchor_state.clone(), block_root)?; + storage.put_justified_checkpoint(latest_justified.clone())?; + storage.put_finalized_checkpoint(latest_finalized.clone())?; + storage.put_head_root(block_root)?; + storage.put_safe_target(block_root)?; + storage.put_justified_ever_updated(block_slot.0 == 0)?; + storage.put_finalized_ever_updated(false)?; + Ok(()) + }) + .expect("database write failed"); Store { time: block_slot.0 * INTERVALS_PER_SLOT, @@ -476,7 +478,10 @@ pub fn update_head(store: &mut Store) { // Compute new head using LMD-GHOST from latest justified root let new_head = get_fork_choice_head(store, store.latest_justified.root, &latest_votes, 0); store.head = new_head; - store.storage.put_head_root(new_head).expect("failed to persist head"); + store + .storage + .put_head_root(new_head) + .expect("failed to persist head"); if let Some(head_state) = store.states.get(&new_head) { let finalized_slot = head_state.latest_finalized.slot; @@ -497,12 +502,18 @@ pub fn update_head(store: &mut Store) { root: finalized_root, slot: finalized_slot, }; - store.storage.put_finalized_checkpoint(Checkpoint { - root: finalized_root, - slot: finalized_slot, - }).expect("failed to write to database"); + store + .storage + .put_finalized_checkpoint(Checkpoint { + root: finalized_root, + slot: finalized_slot, + }) + .expect("failed to write to database"); store.finalized_ever_updated = true; - store.storage.put_finalized_ever_updated(true).expect("failed to persist finalized_ever_updated"); + store + .storage + .put_finalized_ever_updated(true) + .expect("failed to persist finalized_ever_updated"); METRICS.get().map(|m| { if let Ok(s) = i64::try_from(finalized_slot.0) { m.lean_latest_finalized_slot.set(s); @@ -644,7 +655,10 @@ pub fn update_safe_target(store: &mut Store) { // Run LMD-GHOST with 2/3 threshold to find safe target let new_safe_target = get_fork_choice_head(store, root, &attestations, min_score); store.safe_target = new_safe_target; - store.storage.put_safe_target(new_safe_target).expect("failed to persist safe target"); + store + .storage + .put_safe_target(new_safe_target) + .expect("failed to persist safe target"); set_gauge_u64( |metrics| &metrics.lean_safe_target_slot, diff --git a/lean_client/src/main.rs b/lean_client/src/main.rs index 413c3c6d..79b65412 100644 --- a/lean_client/src/main.rs +++ b/lean_client/src/main.rs @@ -36,10 +36,10 @@ use networking::types::{ }; use parking_lot::{Mutex, RwLock}; use ssz::{PersistentList, SszHash, SszReadDefault as _}; -use std::{collections::HashMap, path::PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; +use std::{collections::HashMap, path::PathBuf}; use std::{io::IsTerminal, net::IpAddr}; use storage::Storage; use tokio::{ @@ -500,7 +500,7 @@ async fn main() -> Result<()> { feature.enable(); } - let storage = Arc::new(Storage::new(args.data_dir)?); + let mut storage = Arc::new(Storage::new(&args.data_dir)?); let metrics = if args.metrics_config.enabled() { let metrics = Metrics::new()?; @@ -632,12 +632,21 @@ async fn main() -> Result<()> { }; let genesis_root = genesis_block.hash_tree_root(); - match storage.get_network_id()? { - Some(existing) => anyhow::ensure!( - existing == genesis_root, - "database belongs to a different network: found {existing:?}, expected {genesis_root:?}" - ), - None => storage.put_network_id(genesis_root)?, + if let Some(existing) = storage.get_network_id()? { + if existing != genesis_root { + let suffix = hex::encode(&genesis_root.as_bytes()[..3]); + let mut new_dir = args.data_dir.clone().into_os_string(); + new_dir.push("_"); + new_dir.push(&suffix); + let new_dir = PathBuf::from(new_dir); + info!("database belongs to a different network; switching to {new_dir:?}"); + storage = Arc::new(Storage::new(&new_dir)?); + if storage.get_network_id()?.is_none() { + storage.put_network_id(genesis_root)?; + } + } + } else { + storage.put_network_id(genesis_root)?; } let genesis_signed_block = SignedBlock { diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs index 4c509831..200e671a 100644 --- a/lean_client/storage/src/lib.rs +++ b/lean_client/storage/src/lib.rs @@ -1,4 +1,9 @@ -use std::{borrow::Cow, collections::HashSet, path::Path, sync::{Arc, Mutex}}; +use std::{ + borrow::Cow, + collections::HashSet, + path::Path, + sync::{Arc, Mutex}, +}; use anyhow::Result; use bytesize::ByteSize; @@ -314,7 +319,7 @@ impl Storage { impl Default for Storage { fn default() -> Self { - Self::new("./database").expect("failed to open default storage at ./database") + Self::new("./data").expect("failed to open default storage at ./data") } } From 9ed2d6f96cf279adeb910246a071ce837c889d69 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Fri, 10 Jul 2026 21:54:55 +0530 Subject: [PATCH 16/20] chore: gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c2754f87..f3882695 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ lean_client/target/* lean_client/tests/mainnet lean_client/bin lean_client/spec/ -lean_client/**/data +**/data target/* *.local* From 8394e4ac12d68206d1549d6dcb1abc6f265655c2 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Sat, 11 Jul 2026 21:31:26 +0530 Subject: [PATCH 17/20] tidying up --- lean_client/fork_choice/src/store.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lean_client/fork_choice/src/store.rs b/lean_client/fork_choice/src/store.rs index d89dcc17..ce3db02c 100644 --- a/lean_client/fork_choice/src/store.rs +++ b/lean_client/fork_choice/src/store.rs @@ -300,9 +300,16 @@ pub fn get_forkchoice_store( storage, } } else { + // Extract the plain Block from the signed block let block = anchor_block.block.clone(); let block_slot = block.slot; + // Compute block root differently for genesis vs checkpoint sync: + // - Genesis (slot 0): Use block.hash_tree_root() directly — block and state are consistent. + // - Checkpoint sync (slot > 0): Reconstruct BlockHeader from state.latest_block_header, + // using anchor_state.hash_tree_root() as state_root. This guarantees the root stored + // as the key in store.blocks / store.states is the canonical one committed to by the + // downloaded state, independent of what the real block's state_root field contains. let block_root = if block_slot.0 == 0 { block.hash_tree_root() } else { @@ -316,6 +323,12 @@ pub fn get_forkchoice_store( block_header.hash_tree_root() }; + // Seed both checkpoints from the anchor block itself: (root=anchor_root, + // slot=anchor_slot). The store treats the anchor as the new "genesis" for + // fork choice — pre-anchor history is pruned — so the embedded checkpoints + // from the downloaded state are intentionally ignored. This keeps the + // checkpoint slot/root pair internally consistent with the block at + // anchor_root, mirroring the beacon-chain seeding convention. let anchor_checkpoint = Checkpoint { root: block_root, slot: block_slot, @@ -337,6 +350,9 @@ pub fn get_forkchoice_store( }) .expect("database write failed"); + // Store the original anchor_state - do NOT modify it + // Modifying checkpoints would change its hash_tree_root(), breaking the + // consistency with block.state_root Store { time: block_slot.0 * INTERVALS_PER_SLOT, config, From 3a4c25c06a1f8f21f2d9a43a7a2cecaced513c53 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Tue, 21 Jul 2026 20:57:58 +0530 Subject: [PATCH 18/20] diffs removed --- .gitignore | 1 - lean_client/Cargo.lock | 2 - lean_client/fork_choice/Cargo.toml | 1 - lean_client/fork_choice/src/handlers.rs | 31 +-- lean_client/fork_choice/src/store.rs | 242 +++++------------- .../tests/fork_choice_test_vectors.rs | 15 +- .../fork_choice/tests/unit_tests/common.rs | 16 +- .../tests/unit_tests/fork_choice.rs | 42 +-- .../fork_choice/tests/unit_tests/validator.rs | 6 +- lean_client/http_api/Cargo.toml | 1 - lean_client/http_api/src/test_driver.rs | 12 +- lean_client/src/main.rs | 27 +- 12 files changed, 77 insertions(+), 319 deletions(-) diff --git a/.gitignore b/.gitignore index f3882695..0ed89914 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ lean_client/target/* lean_client/tests/mainnet lean_client/bin lean_client/spec/ -**/data target/* *.local* diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 71dd1b87..c85730f3 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -1927,7 +1927,6 @@ dependencies = [ "serde_json", "spec_test_fixtures", "ssz", - "storage", "test-generator", "tracing", "xmss", @@ -2431,7 +2430,6 @@ dependencies = [ "serde_json", "spec_test_fixtures", "ssz", - "storage", "test-generator", "tokio", "tower", diff --git a/lean_client/fork_choice/Cargo.toml b/lean_client/fork_choice/Cargo.toml index bd64043e..e80e2f4c 100644 --- a/lean_client/fork_choice/Cargo.toml +++ b/lean_client/fork_choice/Cargo.toml @@ -11,7 +11,6 @@ indexmap = { workspace = true } metrics = { workspace = true } parking_lot = { workspace = true } ssz = { workspace = true } -storage = { workspace = true } tracing = { workspace = true } xmss = { workspace = true } diff --git a/lean_client/fork_choice/src/handlers.rs b/lean_client/fork_choice/src/handlers.rs index 73f6182d..e6fcf7fd 100644 --- a/lean_client/fork_choice/src/handlers.rs +++ b/lean_client/fork_choice/src/handlers.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use anyhow::{Context, Result, anyhow, bail, ensure}; use containers::{ AttestationData, Checkpoint, SignatureKey, SignedAggregatedAttestation, SignedAttestation, - SignedBlock, Slot, State, + SignedBlock, State, }; use metrics::METRICS; use parking_lot::RwLock; @@ -727,17 +727,6 @@ pub fn apply_verified_block( store.blocks.insert(block_root, block.clone()); store.states.insert(block_root, Arc::clone(&new_state)); - store - .storage - .batch_write(|| { - store.storage.put_block(block.clone(), block_root)?; - store - .storage - .put_state(new_state.as_ref().clone(), block_root)?; - Ok(()) - }) - .expect("database write failed"); - METRICS.get().map(|m| { m.grandine_store_blocks_size.set(store.blocks.len() as i64); m.grandine_store_states_size.set(store.states.len() as i64); @@ -782,15 +771,7 @@ pub fn apply_verified_block( "Store justified checkpoint updated!" ); store.latest_justified = new_state.latest_justified.clone(); - store - .storage - .put_justified_checkpoint(new_state.latest_justified.clone()) - .expect("failed to persist justified checkpoint"); store.justified_ever_updated = true; - store - .storage - .put_justified_ever_updated(true) - .expect("failed to persist justified_ever_updated"); METRICS.get().map(|metrics| { let Some(slot) = new_state.latest_justified.slot.0.try_into().ok() else { warn!("unable to set latest_justified slot in metrics"); @@ -864,16 +845,6 @@ pub fn apply_verified_block( .saturating_sub(STATE_PRUNE_BUFFER); store.states.retain(|_, state| state.slot.0 >= keep_from); store.blocks.retain(|_, block| block.slot.0 >= keep_from); - store - .storage - .commit_finalization( - store.head, - store.latest_finalized.clone(), - store.safe_target, - Slot(keep_from), - &HashSet::new(), - ) - .expect("failed to commit finalization"); let finalized_slot = store.latest_finalized.slot.0; let adr = &store.attestation_data_by_root; diff --git a/lean_client/fork_choice/src/store.rs b/lean_client/fork_choice/src/store.rs index ce3db02c..6951b9c2 100644 --- a/lean_client/fork_choice/src/store.rs +++ b/lean_client/fork_choice/src/store.rs @@ -9,7 +9,6 @@ use containers::{ use indexmap::IndexMap; use metrics::{METRICS, set_gauge_u64}; use ssz::{H256, SszHash}; -use storage::Storage; use tracing::{info, warn}; use xmss::Signature; @@ -109,8 +108,6 @@ pub struct Store { pub pending_fetch_roots: HashSet, pub log_inv_rate: usize, - - pub storage: Arc, } const JUSTIFICATION_LOOKBACK_SLOTS: u64 = 3; @@ -227,164 +224,76 @@ pub fn get_forkchoice_store( config: Config, is_aggregator: bool, log_inv_rate: usize, - storage: Arc, ) -> Store { - if let Some(head_root) = storage - .get_head_root() - .expect("failed to read head root from database") - { - let latest_finalized = storage - .get_finalized_checkpoint() - .expect("failed to read finalized checkpoint from database") - .unwrap_or_default(); - let latest_justified = storage - .get_justified_checkpoint() - .expect("failed to read justified checkpoint from database") - .unwrap_or_default(); - let safe_target = storage - .get_safe_target() - .expect("failed to read safe target from database") - .unwrap_or_default(); - let justified_ever_updated = storage - .get_justified_ever_updated() - .expect("failed to read justified_ever_updated from database") - .unwrap_or(false); - let finalized_ever_updated = storage - .get_finalized_ever_updated() - .expect("failed to read finalized_ever_updated from database") - .unwrap_or(false); - - let keep_from = latest_finalized.slot.0.saturating_sub(STATE_PRUNE_BUFFER); - let mut blocks = HashMap::new(); - let mut states = HashMap::new(); - for (root, block, state) in storage - .load_since(Slot(keep_from)) - .expect("failed to load chain from database") - { - blocks.insert(root, block); - states.insert(root, Arc::new(state)); - } - - let head = if blocks.contains_key(&head_root) { - head_root - } else { - latest_finalized.root - }; - let head_slot = blocks - .get(&head) - .map(|block| block.slot.0) - .unwrap_or(latest_finalized.slot.0); - - Store { - time: head_slot * INTERVALS_PER_SLOT, - config, - is_aggregator, - head, - safe_target, - latest_justified, - latest_finalized, - justified_ever_updated, - finalized_ever_updated, - blocks, - states, - latest_known_attestations: HashMap::new(), - latest_new_attestations: HashMap::new(), - gossip_signatures: HashMap::new(), - latest_known_aggregated_payloads: IndexMap::new(), - latest_new_aggregated_payloads: IndexMap::new(), - attestation_data_by_root: HashMap::new(), - pending_attestations: HashMap::new(), - pending_aggregated_attestations: HashMap::new(), - pending_fetch_roots: HashSet::new(), - log_inv_rate, - storage, - } + // Extract the plain Block from the signed block + let block = anchor_block.block.clone(); + let block_slot = block.slot; + + // Compute block root differently for genesis vs checkpoint sync: + // - Genesis (slot 0): Use block.hash_tree_root() directly — block and state are consistent. + // - Checkpoint sync (slot > 0): Reconstruct BlockHeader from state.latest_block_header, + // using anchor_state.hash_tree_root() as state_root. This guarantees the root stored + // as the key in store.blocks / store.states is the canonical one committed to by the + // downloaded state, independent of what the real block's state_root field contains. + let block_root = if block_slot.0 == 0 { + block.hash_tree_root() } else { - // Extract the plain Block from the signed block - let block = anchor_block.block.clone(); - let block_slot = block.slot; - - // Compute block root differently for genesis vs checkpoint sync: - // - Genesis (slot 0): Use block.hash_tree_root() directly — block and state are consistent. - // - Checkpoint sync (slot > 0): Reconstruct BlockHeader from state.latest_block_header, - // using anchor_state.hash_tree_root() as state_root. This guarantees the root stored - // as the key in store.blocks / store.states is the canonical one committed to by the - // downloaded state, independent of what the real block's state_root field contains. - let block_root = if block_slot.0 == 0 { - block.hash_tree_root() - } else { - let block_header = BlockHeader { - slot: anchor_state.latest_block_header.slot, - proposer_index: anchor_state.latest_block_header.proposer_index, - parent_root: anchor_state.latest_block_header.parent_root, - state_root: anchor_state.hash_tree_root(), - body_root: anchor_state.latest_block_header.body_root, - }; - block_header.hash_tree_root() + let block_header = BlockHeader { + slot: anchor_state.latest_block_header.slot, + proposer_index: anchor_state.latest_block_header.proposer_index, + parent_root: anchor_state.latest_block_header.parent_root, + state_root: anchor_state.hash_tree_root(), + body_root: anchor_state.latest_block_header.body_root, }; + block_header.hash_tree_root() + }; - // Seed both checkpoints from the anchor block itself: (root=anchor_root, - // slot=anchor_slot). The store treats the anchor as the new "genesis" for - // fork choice — pre-anchor history is pruned — so the embedded checkpoints - // from the downloaded state are intentionally ignored. This keeps the - // checkpoint slot/root pair internally consistent with the block at - // anchor_root, mirroring the beacon-chain seeding convention. - let anchor_checkpoint = Checkpoint { - root: block_root, - slot: block_slot, - }; - let latest_justified = anchor_checkpoint.clone(); - let latest_finalized = anchor_checkpoint; - - storage - .batch_write(|| { - storage.put_block(block.clone(), block_root)?; - storage.put_state(anchor_state.clone(), block_root)?; - storage.put_justified_checkpoint(latest_justified.clone())?; - storage.put_finalized_checkpoint(latest_finalized.clone())?; - storage.put_head_root(block_root)?; - storage.put_safe_target(block_root)?; - storage.put_justified_ever_updated(block_slot.0 == 0)?; - storage.put_finalized_ever_updated(false)?; - Ok(()) - }) - .expect("database write failed"); - - // Store the original anchor_state - do NOT modify it - // Modifying checkpoints would change its hash_tree_root(), breaking the - // consistency with block.state_root - Store { - time: block_slot.0 * INTERVALS_PER_SLOT, - config, - is_aggregator, - head: block_root, - safe_target: block_root, - latest_justified, - latest_finalized, - justified_ever_updated: block_slot.0 == 0, - finalized_ever_updated: false, - blocks: { - let mut m = HashMap::new(); - m.insert(block_root, block); - m - }, - states: { - let mut m = HashMap::new(); - m.insert(block_root, Arc::new(anchor_state)); - m - }, - latest_known_attestations: HashMap::new(), - latest_new_attestations: HashMap::new(), - gossip_signatures: HashMap::new(), - latest_known_aggregated_payloads: IndexMap::new(), - latest_new_aggregated_payloads: IndexMap::new(), - attestation_data_by_root: HashMap::new(), - pending_attestations: HashMap::new(), - pending_aggregated_attestations: HashMap::new(), - pending_fetch_roots: HashSet::new(), - log_inv_rate, - storage, - } + // Seed both checkpoints from the anchor block itself: (root=anchor_root, + // slot=anchor_slot). The store treats the anchor as the new "genesis" for + // fork choice — pre-anchor history is pruned — so the embedded checkpoints + // from the downloaded state are intentionally ignored. This keeps the + // checkpoint slot/root pair internally consistent with the block at + // anchor_root, mirroring the beacon-chain seeding convention. + let anchor_checkpoint = Checkpoint { + root: block_root, + slot: block_slot, + }; + let latest_justified = anchor_checkpoint.clone(); + let latest_finalized = anchor_checkpoint; + + // Store the original anchor_state - do NOT modify it + // Modifying checkpoints would change its hash_tree_root(), breaking the + // consistency with block.state_root + Store { + time: block_slot.0 * INTERVALS_PER_SLOT, + config, + is_aggregator, + head: block_root, + safe_target: block_root, + latest_justified, + latest_finalized, + justified_ever_updated: block_slot.0 == 0, + finalized_ever_updated: false, + blocks: { + let mut m = HashMap::new(); + m.insert(block_root, block); + m + }, + states: { + let mut m = HashMap::new(); + m.insert(block_root, Arc::new(anchor_state)); + m + }, + latest_known_attestations: HashMap::new(), + latest_new_attestations: HashMap::new(), + gossip_signatures: HashMap::new(), + latest_known_aggregated_payloads: IndexMap::new(), + latest_new_aggregated_payloads: IndexMap::new(), + attestation_data_by_root: HashMap::new(), + pending_attestations: HashMap::new(), + pending_aggregated_attestations: HashMap::new(), + pending_fetch_roots: HashSet::new(), + log_inv_rate, } } @@ -494,10 +403,6 @@ pub fn update_head(store: &mut Store) { // Compute new head using LMD-GHOST from latest justified root let new_head = get_fork_choice_head(store, store.latest_justified.root, &latest_votes, 0); store.head = new_head; - store - .storage - .put_head_root(new_head) - .expect("failed to persist head"); if let Some(head_state) = store.states.get(&new_head) { let finalized_slot = head_state.latest_finalized.slot; @@ -518,18 +423,7 @@ pub fn update_head(store: &mut Store) { root: finalized_root, slot: finalized_slot, }; - store - .storage - .put_finalized_checkpoint(Checkpoint { - root: finalized_root, - slot: finalized_slot, - }) - .expect("failed to write to database"); store.finalized_ever_updated = true; - store - .storage - .put_finalized_ever_updated(true) - .expect("failed to persist finalized_ever_updated"); METRICS.get().map(|m| { if let Ok(s) = i64::try_from(finalized_slot.0) { m.lean_latest_finalized_slot.set(s); @@ -671,10 +565,6 @@ pub fn update_safe_target(store: &mut Store) { // Run LMD-GHOST with 2/3 threshold to find safe target let new_safe_target = get_fork_choice_head(store, root, &attestations, min_score); store.safe_target = new_safe_target; - store - .storage - .put_safe_target(new_safe_target) - .expect("failed to persist safe target"); set_gauge_u64( |metrics| &metrics.lean_safe_target_slot, diff --git a/lean_client/fork_choice/tests/fork_choice_test_vectors.rs b/lean_client/fork_choice/tests/fork_choice_test_vectors.rs index 1797b197..514e3e2b 100644 --- a/lean_client/fork_choice/tests/fork_choice_test_vectors.rs +++ b/lean_client/fork_choice/tests/fork_choice_test_vectors.rs @@ -13,23 +13,11 @@ use spec_test_fixtures::fork_choice::{ForkChoiceStep, StoreChecks}; use serde::Deserialize; use ssz::{BitList, H256, SszHash}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; use std::{collections::HashMap, fs::File}; use std::{panic::AssertUnwindSafe, path::Path}; -use storage::Storage; use test_generator::test_resources; use xmss::PublicKey; -/// Open a throwaway `Storage` in a unique temp directory, one libmdbx -/// environment per store so parallel test cases never share a path. -fn test_storage() -> Arc { - static COUNTER: AtomicU64 = AtomicU64::new(0); - let n = COUNTER.fetch_add(1, Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!("lean-test-vectors-{}-{n}", std::process::id())); - Arc::new(Storage::new(dir).expect("failed to open test storage")) -} - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct TestCase { @@ -481,8 +469,7 @@ fn forkchoice(spec_file: &str) { body_root, }; - let mut store = - get_forkchoice_store(anchor_state, anchor_block, config, false, 1, test_storage()); + let mut store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1); let mut cache = BlockCache::new(); let mut block_labels: HashMap = HashMap::new(); diff --git a/lean_client/fork_choice/tests/unit_tests/common.rs b/lean_client/fork_choice/tests/unit_tests/common.rs index 8dfb7678..c9c71da2 100644 --- a/lean_client/fork_choice/tests/unit_tests/common.rs +++ b/lean_client/fork_choice/tests/unit_tests/common.rs @@ -1,20 +1,6 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; - use containers::{Block, BlockBody, Config, SignedBlock, Slot, State, Validator}; use fork_choice::store::{Store, get_forkchoice_store}; use ssz::{H256, SszHash}; -use storage::Storage; - -/// Open a throwaway `Storage` in a unique temp directory. Each store gets its -/// own libmdbx environment so tests running in parallel never open the same -/// path twice within a process. -pub fn test_storage() -> Arc { - static COUNTER: AtomicU64 = AtomicU64::new(0); - let n = COUNTER.fetch_add(1, Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!("lean-test-{}-{n}", std::process::id())); - Arc::new(Storage::new(dir).expect("failed to open test storage")) -} pub fn create_test_store() -> Store { let config = Config { genesis_time: 1000 }; @@ -36,5 +22,5 @@ pub fn create_test_store() -> Store { proof: Default::default(), }; - get_forkchoice_store(state, signed_block, config, true, 1, test_storage()) + get_forkchoice_store(state, signed_block, config, true, 1) } diff --git a/lean_client/fork_choice/tests/unit_tests/fork_choice.rs b/lean_client/fork_choice/tests/unit_tests/fork_choice.rs index 67ae3718..597f3511 100644 --- a/lean_client/fork_choice/tests/unit_tests/fork_choice.rs +++ b/lean_client/fork_choice/tests/unit_tests/fork_choice.rs @@ -1,6 +1,6 @@ -use super::common::{create_test_store, test_storage}; -use containers::{Block, BlockBody, Config, SignedBlock, Slot, State, Validator}; -use fork_choice::store::{get_forkchoice_store, get_proposal_head}; +use super::common::create_test_store; +use containers::{Block, BlockBody, Slot}; +use fork_choice::store::get_proposal_head; use ssz::{H256, SszHash}; #[test] @@ -53,39 +53,3 @@ fn test_get_vote_target_chain() { assert_eq!(target.slot, Slot(6)); } - -#[test] -fn get_forkchoice_store_restores_from_db() { - let storage = test_storage(); - let state = State::generate_genesis_with_validators(1000, vec![Validator::default(); 4]); - let block = Block { - slot: Slot(0), - proposer_index: 0, - parent_root: H256::default(), - state_root: state.hash_tree_root(), - body: BlockBody::default(), - }; - let signed_block = SignedBlock { - block, - proof: Default::default(), - }; - let config = Config { genesis_time: 1000 }; - - let fresh = get_forkchoice_store( - state.clone(), - signed_block.clone(), - config.clone(), - true, - 1, - storage.clone(), - ); - let head = fresh.head; - let finalized = fresh.latest_finalized.clone(); - drop(fresh); - - let restored = get_forkchoice_store(state, signed_block, config, true, 1, storage); - assert_eq!(restored.head, head); - assert!(restored.blocks.contains_key(&head)); - assert!(restored.states.contains_key(&head)); - assert_eq!(restored.latest_finalized, finalized); -} diff --git a/lean_client/fork_choice/tests/unit_tests/validator.rs b/lean_client/fork_choice/tests/unit_tests/validator.rs index a6651ecf..97e5f37d 100644 --- a/lean_client/fork_choice/tests/unit_tests/validator.rs +++ b/lean_client/fork_choice/tests/unit_tests/validator.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; -use crate::unit_tests::common::{create_test_store, test_storage}; +use crate::unit_tests::common::create_test_store; use containers::{ AggregatedSignatureProof, AggregationBits, Attestation, AttestationData, Block, BlockBody, Checkpoint, Config, MultiMessageAggregate, SignedBlock, Slot, State, Validator, @@ -122,7 +122,7 @@ fn create_test_store_with_signers() -> (Store, HashMap) { }; ( - get_forkchoice_store(state, signed_block, config, true, 1, test_storage()), + get_forkchoice_store(state, signed_block, config, true, 1), keys, ) } @@ -514,7 +514,7 @@ fn test_validator_operations_empty_store() { proof: Default::default(), }; - let mut store = get_forkchoice_store(state, signed_block, config, true, 1, test_storage()); + let mut store = get_forkchoice_store(state, signed_block, config, true, 1); // Should be able to produce block and attestation let (_root, block, _sig) = diff --git a/lean_client/http_api/Cargo.toml b/lean_client/http_api/Cargo.toml index 2413ba92..42627d89 100644 --- a/lean_client/http_api/Cargo.toml +++ b/lean_client/http_api/Cargo.toml @@ -18,7 +18,6 @@ serde = { workspace = true } serde_json = { workspace = true } spec_test_fixtures = { workspace = true } ssz = { workspace = true } -storage = { workspace = true } tokio = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } diff --git a/lean_client/http_api/src/test_driver.rs b/lean_client/http_api/src/test_driver.rs index d8616262..afa6a4ab 100644 --- a/lean_client/http_api/src/test_driver.rs +++ b/lean_client/http_api/src/test_driver.rs @@ -12,7 +12,6 @@ //! `simulators/lean/src/scenarios/spec_assets.rs`. use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; use axum::{ Json, Router, body::Bytes, extract::State as AxumState, http::StatusCode, routing::post, @@ -33,7 +32,6 @@ use spec_test_fixtures::{ VerifySignaturesTestCase, }; use ssz::SszHash; -use storage::Storage; use xmss::{AggregatedSignature, Signature}; /// Shared state for test-driver routes. Carries a writable handle to the @@ -181,15 +179,7 @@ async fn init_fork_choice( genesis_time: anchor_state.config.genesis_time, }; - // Each fixture drives a fresh fork-choice store, so give it its own - // throwaway libmdbx environment in a unique temp directory. - let storage = { - static COUNTER: AtomicU64 = AtomicU64::new(0); - let n = COUNTER.fetch_add(1, Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!("lean-test-driver-{}-{n}", std::process::id())); - Arc::new(Storage::new(dir).expect("failed to open test-driver storage")) - }; - let new_store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1, storage); + let new_store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1); *state.store.write() = new_store; *state.cache.write() = BlockCache::new(); diff --git a/lean_client/src/main.rs b/lean_client/src/main.rs index 79b65412..de675c45 100644 --- a/lean_client/src/main.rs +++ b/lean_client/src/main.rs @@ -36,12 +36,11 @@ use networking::types::{ }; use parking_lot::{Mutex, RwLock}; use ssz::{PersistentList, SszHash, SszReadDefault as _}; +use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; -use std::{collections::HashMap, path::PathBuf}; use std::{io::IsTerminal, net::IpAddr}; -use storage::Storage; use tokio::{ sync::{Notify, mpsc, oneshot, watch}, task, @@ -378,9 +377,6 @@ struct Args { #[arg(short, long)] genesis: Option, - #[arg(long, default_value = "data")] - data_dir: PathBuf, - #[arg(long)] node_id: Option, @@ -500,8 +496,6 @@ async fn main() -> Result<()> { feature.enable(); } - let mut storage = Arc::new(Storage::new(&args.data_dir)?); - let metrics = if args.metrics_config.enabled() { let metrics = Metrics::new()?; metrics.register_with_default_metrics()?; @@ -631,24 +625,6 @@ async fn main() -> Result<()> { }, }; - let genesis_root = genesis_block.hash_tree_root(); - if let Some(existing) = storage.get_network_id()? { - if existing != genesis_root { - let suffix = hex::encode(&genesis_root.as_bytes()[..3]); - let mut new_dir = args.data_dir.clone().into_os_string(); - new_dir.push("_"); - new_dir.push(&suffix); - let new_dir = PathBuf::from(new_dir); - info!("database belongs to a different network; switching to {new_dir:?}"); - storage = Arc::new(Storage::new(&new_dir)?); - if storage.get_network_id()?.is_none() { - storage.put_network_id(genesis_root)?; - } - } - } else { - storage.put_network_id(genesis_root)?; - } - let genesis_signed_block = SignedBlock { block: genesis_block, proof: MultiMessageAggregate::default(), @@ -1093,7 +1069,6 @@ async fn main() -> Result<()> { config.clone(), args.is_aggregator, genesis_log_inv_rate as usize, - storage, ))); // Seed the block provider so we can serve the anchor block to peers via BlocksByRoot. From 630b7a9c6cf4492f7579c651c46550968491add6 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Wed, 22 Jul 2026 02:32:31 +0530 Subject: [PATCH 19/20] db makeover --- lean_client/Cargo.lock | 48 +- lean_client/Cargo.toml | 9 +- lean_client/{storage => database}/Cargo.toml | 7 +- lean_client/database/src/lib.rs | 281 ++++++ lean_client/fork_choice/Cargo.toml | 1 + lean_client/fork_choice/src/store.rs | 5 +- .../fork_choice/tests/unit_tests/validator.rs | 14 +- lean_client/http_api/tests/api_endpoint.rs | 21 +- lean_client/storage/src/lib.rs | 955 ------------------ 9 files changed, 338 insertions(+), 1003 deletions(-) rename lean_client/{storage => database}/Cargo.toml (66%) create mode 100644 lean_client/database/src/lib.rs delete mode 100644 lean_client/storage/src/lib.rs diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index c85730f3..38f6ba4b 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -816,15 +816,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bytesize" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d" -dependencies = [ - "serde_core", -] - [[package]] name = "cc" version = "1.2.15" @@ -1345,6 +1336,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "database" +version = "0.0.0" +dependencies = [ + "anyhow", + "containers", + "lz4_flex", + "reth-libmdbx", + "ssz", + "zstd", +] + [[package]] name = "dedicated_executor" version = "0.1.0" @@ -1917,6 +1920,7 @@ dependencies = [ "anyhow", "bls", "containers", + "database", "env-config", "indexmap 2.14.0", "metrics", @@ -1941,15 +1945,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs-err" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" -dependencies = [ - "autocfg", -] - [[package]] name = "funty" version = "2.0.0" @@ -2996,6 +2991,7 @@ dependencies = [ "chain", "clap", "containers", + "database", "dedicated_executor", "ethereum-types", "features", @@ -3010,7 +3006,6 @@ dependencies = [ "parking_lot", "reqwest", "ssz", - "storage", "tikv-jemallocator", "tokio", "tracing", @@ -5831,21 +5826,6 @@ dependencies = [ "triomphe", ] -[[package]] -name = "storage" -version = "0.0.0" -dependencies = [ - "anyhow", - "bytesize", - "containers", - "fs-err", - "lz4_flex", - "reth-libmdbx", - "ssz", - "tempfile", - "zstd", -] - [[package]] name = "strength_reduce" version = "0.2.4" diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 2776424f..aa95194d 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -2,13 +2,13 @@ members = [ "chain", "containers", + "database", "env-config", "fork_choice", "http_api", "metrics", "networking", - "spec_test_fixtures", - "storage", + "spec_test_fixtures", "validator", "xmss", ] @@ -226,13 +226,13 @@ private_intra_doc_links = 'allow' [workspace.dependencies] chain = { path = "./chain" } containers = { path = "./containers" } +database = { path = "./database" } env-config = { path = "./env-config" } fork_choice = { path = "./fork_choice" } http_api = { path = "./http_api" } metrics = { path = "./metrics" } networking = { path = "./networking" } spec_test_fixtures = { path = "./spec_test_fixtures" } -storage = { path = "./storage" } validator = { path = "./validator" } xmss = { path = "./xmss" } @@ -243,7 +243,6 @@ bitvec = "1.0.1" bls = { git = "https://github.com/grandinetech/grandine", package = "bls", features = ["blst"], rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } bytesize = { version = '2', features = ['serde'] } clap = { version = "4", features = ["derive"] } -database = { git = "https://github.com/grandinetech/grandine", package = "database", rev = "64afdee3c6be79fceffb66933dcb69a943f3f1ae" } derive_more = "2.1.1" discv5 = "0.10.2" enr = { version = "0.13", features = ["k256"] } @@ -328,6 +327,7 @@ bls = { workspace = true } chain = { workspace = true } clap = { workspace = true } containers = { workspace = true } +database = { workspace = true } dedicated_executor = { workspace = true } ethereum-types = { workspace = true } features = { workspace = true } @@ -341,7 +341,6 @@ networking = { workspace = true } num_cpus = { workspace = true } parking_lot = { workspace = true } ssz = { workspace = true } -storage = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/lean_client/storage/Cargo.toml b/lean_client/database/Cargo.toml similarity index 66% rename from lean_client/storage/Cargo.toml rename to lean_client/database/Cargo.toml index 2a05808a..729b8efd 100644 --- a/lean_client/storage/Cargo.toml +++ b/lean_client/database/Cargo.toml @@ -1,19 +1,14 @@ [package] -name = "storage" +name = "database" edition = { workspace = true } [dependencies] anyhow = { workspace = true } -bytesize = { workspace = true } containers = { workspace = true } -fs-err = { workspace = true } libmdbx = { workspace = true } lz4_flex = { workspace = true } ssz = { workspace = true } zstd = { workspace = true } -[dev-dependencies] -tempfile = { workspace = true } - [lints] workspace = true diff --git a/lean_client/database/src/lib.rs b/lean_client/database/src/lib.rs new file mode 100644 index 00000000..23721dec --- /dev/null +++ b/lean_client/database/src/lib.rs @@ -0,0 +1,281 @@ +use std::{ + borrow::Cow, + marker::PhantomData, + ops::{Bound, RangeBounds}, + path::PathBuf, + sync::Arc, +}; + +use anyhow::{Ok, Result}; + +use containers::Slot; +use ssz::{H256, SszReadDefault, SszWrite}; + +use libmdbx::{ + DatabaseFlags, Environment, EnvironmentFlags, EnvironmentKind, Geometry, Mode, SyncMode, + WriteFlags, +}; + +const GIB: usize = 1 << 30; +const MIB: usize = 1 << 20; + +const BLOCKS_TABLE_NAME: &str = "blocks"; +const GENESIS_STATE_TABLE_NAME: &str = "genesis_state"; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum Compression { + None, + #[default] + Lz4, + Zstd, +} + +impl Compression { + fn compress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), + Self::Zstd => Ok(zstd::encode_all(data, 3)?), + } + } + + fn decompress(self, data: &[u8]) -> Result> { + match self { + Self::None => Ok(data.to_vec()), + Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), + Self::Zstd => Ok(zstd::decode_all(data)?), + } + } +} + +pub struct EnvironmentBuilder { + path: PathBuf, + max_size: usize, +} + +impl EnvironmentBuilder { + pub fn new(path: impl Into) -> Self { + Self { + path: path.into(), + max_size: 128 * GIB, + } + } + + pub fn max_size(mut self, bytes: usize) -> Self { + self.max_size = bytes; + self + } + + pub fn build(&self) -> Result> { + std::fs::create_dir_all(&self.path)?; + + let env = Environment::builder() + .set_max_dbs(2) + .set_kind(EnvironmentKind::WriteMap) + .set_flags(EnvironmentFlags { + mode: Mode::ReadWrite { + sync_mode: SyncMode::SafeNoSync, + }, + no_meminit: true, + coalesce: true, + liforeclaim: true, + ..Default::default() + }) + .set_geometry(Geometry { + size: Some(0..self.max_size), + growth_step: Some((256 * MIB) as isize), + shrink_threshold: None, + page_size: None, + }) + .open(&self.path)?; + + Ok(Arc::new(env)) + } +} + +struct Database { + env: Arc, + name: &'static str, + compression: Compression, + _marker: PhantomData V>, +} + +impl Database { + pub fn new(env: Arc, name: &'static str, compression: Compression) -> Result { + let txn = env.begin_rw_txn()?; + txn.create_db(Some(name), DatabaseFlags::default())?; + txn.commit()?; + Ok(Self { + env, + name, + compression, + _marker: PhantomData, + }) + } + + pub fn get(&self, key: &K) -> Result> { + let txn = self.env.begin_ro_txn()?; + let db = txn.open_db(Some(self.name))?; + + let key_bytes = key.encode(); + let Some(compressed) = txn.get::>(db.dbi(), &key_bytes)? else { + return Ok(None); + }; + + let bytes = self.compression.decompress(&compressed)?; + Ok(Some(V::decode(&bytes)?)) + } + + pub fn put(&self, key: &K, value: &V) -> Result<()> { + let txn = self.env.begin_rw_txn()?; + let db = txn.open_db(Some(self.name))?; + + let compressed = self.compression.compress(&value.encode()?)?; + txn.put(db.dbi(), key.encode(), compressed, WriteFlags::default())?; + + txn.commit()?; + Ok(()) + } + + pub fn range(&self, bounds: impl RangeBounds

) -> Result> { + let txn = self.env.begin_ro_txn()?; + let db = txn.open_db(Some(self.name))?; + let mut cursor = txn.cursor(&db)?; + + let start = bounds.start_bound().map(|p| p.encode()); + let end = bounds.end_bound().map(|p| p.encode()); + + let mut entry = match &start { + Bound::Included(s) | Bound::Excluded(s) => { + cursor.set_range::, Cow<[u8]>>(s)? + } + Bound::Unbounded => cursor.first::, Cow<[u8]>>()?, + }; + + let mut out = Vec::new(); + while let Some((key, value)) = entry { + if let Bound::Excluded(s) = &start { + if key.len() >= s.len() && &key[..s.len()] == s.as_slice() { + entry = cursor.next::, Cow<[u8]>>()?; + continue; + } + } + let stop = match &end { + Bound::Included(e) => key.len() >= e.len() && &key[..e.len()] > e.as_slice(), + Bound::Excluded(e) => key.len() >= e.len() && &key[..e.len()] >= e.as_slice(), + Bound::Unbounded => false, + }; + if stop { + break; + } + + out.push(V::decode(&self.compression.decompress(&value)?)?); + entry = cursor.next::, Cow<[u8]>>()?; + } + Ok(out) + } + + pub fn delete_range(&self, bounds: impl RangeBounds

) -> Result { + let txn = self.env.begin_rw_txn()?; + let db = txn.open_db(Some(self.name))?; + + let start = bounds.start_bound().map(|p| p.encode()); + let end = bounds.end_bound().map(|p| p.encode()); + + let mut keys: Vec> = Vec::new(); + { + let mut cursor = txn.cursor(&db)?; + let mut entry = match &start { + Bound::Included(s) | Bound::Excluded(s) => cursor.set_range::, ()>(s)?, + Bound::Unbounded => cursor.first::, ()>()?, + }; + while let Some((key, ())) = entry { + if let Bound::Excluded(s) = &start { + if key.len() >= s.len() && &key[..s.len()] == s.as_slice() { + entry = cursor.next::, ()>()?; + continue; + } + } + let stop = match &end { + Bound::Included(e) => key.len() >= e.len() && &key[..e.len()] > e.as_slice(), + Bound::Excluded(e) => key.len() >= e.len() && &key[..e.len()] >= e.as_slice(), + Bound::Unbounded => false, + }; + if stop { + break; + } + + keys.push(key.into_owned()); + entry = cursor.next::, ()>()?; + } + } + + let mut cursor = txn.cursor(&db)?; + let mut deleted = 0; + for key in &keys { + if cursor.set::<()>(key)?.is_some() { + cursor.del(WriteFlags::default())?; + deleted += 1; + } + } + + txn.commit()?; + Ok(deleted) + } +} + +pub trait Key { + fn encode(&self) -> Vec; +} + +pub trait KeyPrefix { + fn encode(&self) -> Vec; +} + +pub trait Value: Sized { + fn encode(&self) -> Result>; + fn decode(bytes: &[u8]) -> Result; +} + +impl Value for T { + fn encode(&self) -> Result> { + Ok(self.to_ssz()?) + } + + fn decode(bytes: &[u8]) -> Result { + Ok(Self::from_ssz_default(bytes)?) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BlockKey { + pub slot: Slot, + pub root: H256, +} + +impl Key for BlockKey { + fn encode(&self) -> Vec { + let mut b = Vec::with_capacity(40); + b.extend_from_slice(&self.slot.0.to_be_bytes()); + b.extend_from_slice(self.root.as_ref()); + b + } +} + +#[derive(Clone, Copy, Debug)] +pub struct BySlot(pub Slot); + +impl KeyPrefix for BySlot { + fn encode(&self) -> Vec { + self.0.0.to_be_bytes().to_vec() + } +} + +#[derive(Clone, Copy, Debug)] +pub struct GenesisKey; + +impl Key for GenesisKey { + fn encode(&self) -> Vec { + vec![0] + } +} \ No newline at end of file diff --git a/lean_client/fork_choice/Cargo.toml b/lean_client/fork_choice/Cargo.toml index e80e2f4c..2b550ab1 100644 --- a/lean_client/fork_choice/Cargo.toml +++ b/lean_client/fork_choice/Cargo.toml @@ -6,6 +6,7 @@ edition = { workspace = true } anyhow = { workspace = true } bls = { workspace = true } containers = { workspace = true } +database = { workspace = true } env-config = { workspace = true } indexmap = { workspace = true } metrics = { workspace = true } diff --git a/lean_client/fork_choice/src/store.rs b/lean_client/fork_choice/src/store.rs index 6951b9c2..100d4a4c 100644 --- a/lean_client/fork_choice/src/store.rs +++ b/lean_client/fork_choice/src/store.rs @@ -6,6 +6,7 @@ use containers::{ AggregatedSignatureProof, AttestationData, Block, BlockHeader, Checkpoint, Config, SignatureKey, SignedAggregatedAttestation, SignedAttestation, SignedBlock, Slot, State, }; +use database::Database; use indexmap::IndexMap; use metrics::{METRICS, set_gauge_u64}; use ssz::{H256, SszHash}; @@ -26,7 +27,7 @@ pub const GOSSIP_DISPARITY_INTERVALS: u64 = 1; /// Forkchoice store tracking chain state and validator attestations -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct Store { pub time: Interval, @@ -108,6 +109,8 @@ pub struct Store { pub pending_fetch_roots: HashSet, pub log_inv_rate: usize, + + // pub database: Arc, } const JUSTIFICATION_LOOKBACK_SLOTS: u64 = 3; diff --git a/lean_client/fork_choice/tests/unit_tests/validator.rs b/lean_client/fork_choice/tests/unit_tests/validator.rs index 97e5f37d..948a2977 100644 --- a/lean_client/fork_choice/tests/unit_tests/validator.rs +++ b/lean_client/fork_choice/tests/unit_tests/validator.rs @@ -557,13 +557,25 @@ fn test_produce_block_missing_parent_state() { let store = Store { time: 100, config: Config { genesis_time: 1000 }, + is_aggregator: true, head: H256::from_slice(&[0xab; 32]), safe_target: H256::from_slice(&[0xab; 32]), latest_justified: checkpoint.clone(), latest_finalized: checkpoint, + justified_ever_updated: false, + finalized_ever_updated: false, blocks: Default::default(), states: Default::default(), - ..Default::default() + latest_known_attestations: Default::default(), + latest_new_attestations: Default::default(), + gossip_signatures: Default::default(), + latest_known_aggregated_payloads: Default::default(), + latest_new_aggregated_payloads: Default::default(), + attestation_data_by_root: Default::default(), + pending_attestations: Default::default(), + pending_aggregated_attestations: Default::default(), + pending_fetch_roots: Default::default(), + log_inv_rate: 1, }; let mut s = store; diff --git a/lean_client/http_api/tests/api_endpoint.rs b/lean_client/http_api/tests/api_endpoint.rs index ebc07dde..bc52959b 100644 --- a/lean_client/http_api/tests/api_endpoint.rs +++ b/lean_client/http_api/tests/api_endpoint.rs @@ -69,8 +69,27 @@ fn api_endpoint(spec_file: &str) { // `store.is_aggregator`; no chain state is needed. let initial_is_aggregator = case.initial_is_aggregator.unwrap_or(false); let store: SharedStore = Arc::new(RwLock::new(Store { + time: Default::default(), + config: Default::default(), is_aggregator: initial_is_aggregator, - ..Default::default() + head: Default::default(), + safe_target: Default::default(), + latest_justified: Default::default(), + latest_finalized: Default::default(), + justified_ever_updated: false, + finalized_ever_updated: false, + blocks: Default::default(), + states: Default::default(), + latest_known_attestations: Default::default(), + latest_new_attestations: Default::default(), + gossip_signatures: Default::default(), + latest_known_aggregated_payloads: Default::default(), + latest_new_aggregated_payloads: Default::default(), + attestation_data_by_root: Default::default(), + pending_attestations: Default::default(), + pending_aggregated_attestations: Default::default(), + pending_fetch_roots: Default::default(), + log_inv_rate: Default::default(), })); let controller = Some(Arc::new(AggregatorController::new(store.clone(), None))); diff --git a/lean_client/storage/src/lib.rs b/lean_client/storage/src/lib.rs deleted file mode 100644 index 200e671a..00000000 --- a/lean_client/storage/src/lib.rs +++ /dev/null @@ -1,955 +0,0 @@ -use std::{ - borrow::Cow, - collections::HashSet, - path::Path, - sync::{Arc, Mutex}, -}; - -use anyhow::Result; -use bytesize::ByteSize; - -use containers::{Block, Checkpoint, Slot, State}; -use ssz::{H256, SszReadDefault as _, SszWrite as _}; - -use libmdbx::{DatabaseFlags, Environment, Geometry, RW, Transaction, TransactionKind, WriteFlags}; - -const BLOCKS_TABLE_NAME: &str = "blocks"; -const STATES_TABLE_NAME: &str = "states"; -const CHECKPOINTS_TABLE_NAME: &str = "checkpoints"; -const BLOCKS_BY_SLOT_TABLE_NAME: &str = "blocks_by_slot"; -const METADATA_TABLE_NAME: &str = "metadata"; - -const JUSTIFIED: u8 = 0; -const FINALIZED: u8 = 1; -const HEAD: u8 = 2; -const SAFE_TARGET: u8 = 3; - -const NETWORK_ID: u8 = 1; -const JUSTIFIED_EVER_UPDATED: u8 = 2; -const FINALIZED_EVER_UPDATED: u8 = 3; - -#[derive(Debug)] -pub struct Storage { - environment: Arc, - transaction: Mutex>>, - blocks: Database, - states: Database, - checkpoints: Database, - blocks_by_slot: Database, - metadata: Database, -} - -impl Storage { - pub fn new(base: impl AsRef) -> Result { - let base = base.as_ref(); - fs_err::create_dir_all(base)?; - - let environment = Arc::new( - Environment::builder() - .set_max_dbs(5) - .set_geometry(Geometry { - size: Some(..usize::try_from(ByteSize::gib(2).as_u64())?), - growth_step: Some(isize::try_from(ByteSize::mib(256).as_u64())?), - shrink_threshold: None, - page_size: None, - }) - .open_with_permissions(base, 0o600)?, - ); - - let txn = environment.begin_rw_txn()?; - for name in [ - BLOCKS_TABLE_NAME, - STATES_TABLE_NAME, - CHECKPOINTS_TABLE_NAME, - BLOCKS_BY_SLOT_TABLE_NAME, - METADATA_TABLE_NAME, - ] { - txn.create_db(Some(name), DatabaseFlags::default())?; - } - txn.commit()?; - - Ok(Self { - blocks: Database::new(BLOCKS_TABLE_NAME, Compression::Lz4), - states: Database::new(STATES_TABLE_NAME, Compression::Zstd), - checkpoints: Database::new(CHECKPOINTS_TABLE_NAME, Compression::None), - blocks_by_slot: Database::new(BLOCKS_BY_SLOT_TABLE_NAME, Compression::None), - metadata: Database::new(METADATA_TABLE_NAME, Compression::None), - environment, - transaction: Mutex::new(None), - }) - } - - pub fn batch_write(&self, f: impl FnOnce() -> Result<()>) -> Result<()> { - let txn = self.environment.begin_rw_txn()?; - *self.transaction.lock().expect("transaction mutex poisoned") = Some(txn); - - let result = f(); - - let txn = self - .transaction - .lock() - .expect("transaction mutex poisoned") - .take() - .expect("batch_write installed the ambient transaction"); - - match result { - Ok(()) => { - txn.commit()?; - Ok(()) - } - Err(error) => { - drop(txn); - Err(error) - } - } - } - - fn write(&self, f: impl FnOnce(&Transaction) -> Result) -> Result { - let guard = self.transaction.lock().expect("transaction mutex poisoned"); - if let Some(txn) = guard.as_ref() { - return f(txn); - } - drop(guard); - let txn = self.environment.begin_rw_txn()?; - let result = f(&txn)?; - txn.commit()?; - Ok(result) - } - - fn read_bytes(&self, db: &Database, key: impl AsRef<[u8]>) -> Result>> { - let guard = self.transaction.lock().expect("transaction mutex poisoned"); - if let Some(txn) = guard.as_ref() { - return db.get(txn, key); - } - drop(guard); - let txn = self.environment.begin_ro_txn()?; - let bytes = db.get(&txn, key)?; - txn.commit()?; - Ok(bytes) - } - - pub fn get_block(&self, block_root: H256) -> Result> { - match self.read_bytes(&self.blocks, block_root)? { - Some(bytes) => Ok(Some(Block::from_ssz_default(&bytes)?)), - None => Ok(None), - } - } - - pub fn put_block(&self, block: Block, block_root: H256) -> Result<()> { - let block_bytes = block.to_ssz()?; - let index_key = slot_root_key(block.slot, block_root); - self.write(|txn| { - self.blocks.put(txn, block_root, block_bytes)?; - self.blocks_by_slot.put(txn, index_key, b"")?; - Ok(()) - }) - } - - pub fn get_state(&self, block_root: H256) -> Result> { - match self.read_bytes(&self.states, block_root)? { - Some(bytes) => Ok(Some(State::from_ssz_default(&bytes)?)), - None => Ok(None), - } - } - - pub fn put_state(&self, state: State, block_root: H256) -> Result<()> { - let state_bytes = state.to_ssz()?; - self.write(|txn| self.states.put(txn, block_root, state_bytes)) - } - - pub fn get_justified_checkpoint(&self) -> Result> { - match self.read_bytes(&self.checkpoints, [JUSTIFIED])? { - Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), - None => Ok(None), - } - } - - pub fn put_justified_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { - let checkpoint_bytes = checkpoint.to_ssz()?; - self.write(|txn| self.checkpoints.put(txn, [JUSTIFIED], checkpoint_bytes)) - } - - pub fn get_finalized_checkpoint(&self) -> Result> { - match self.read_bytes(&self.checkpoints, [FINALIZED])? { - Some(bytes) => Ok(Some(Checkpoint::from_ssz_default(&bytes)?)), - None => Ok(None), - } - } - - pub fn put_finalized_checkpoint(&self, checkpoint: Checkpoint) -> Result<()> { - let checkpoint_bytes = checkpoint.to_ssz()?; - self.write(|txn| self.checkpoints.put(txn, [FINALIZED], checkpoint_bytes)) - } - - pub fn get_head_root(&self) -> Result> { - match self.read_bytes(&self.checkpoints, [HEAD])? { - Some(bytes) => Ok(Some(H256::from_slice(&bytes))), - None => Ok(None), - } - } - - pub fn put_head_root(&self, root: H256) -> Result<()> { - self.write(|txn| self.checkpoints.put(txn, [HEAD], root)) - } - - pub fn get_safe_target(&self) -> Result> { - match self.read_bytes(&self.checkpoints, [SAFE_TARGET])? { - Some(bytes) => Ok(Some(H256::from_slice(&bytes))), - None => Ok(None), - } - } - - pub fn put_safe_target(&self, root: H256) -> Result<()> { - self.write(|txn| self.checkpoints.put(txn, [SAFE_TARGET], root)) - } - - pub fn get_network_id(&self) -> Result> { - match self.read_bytes(&self.metadata, [NETWORK_ID])? { - Some(bytes) => Ok(Some(H256::from_slice(&bytes))), - None => Ok(None), - } - } - - pub fn put_network_id(&self, network_id: H256) -> Result<()> { - self.write(|txn| self.metadata.put(txn, [NETWORK_ID], network_id)) - } - - pub fn get_justified_ever_updated(&self) -> Result> { - match self.read_bytes(&self.metadata, [JUSTIFIED_EVER_UPDATED])? { - Some(bytes) => Ok(Some(bytes.first().copied().unwrap_or(0) != 0)), - None => Ok(None), - } - } - - pub fn put_justified_ever_updated(&self, value: bool) -> Result<()> { - self.write(|txn| { - self.metadata - .put(txn, [JUSTIFIED_EVER_UPDATED], [u8::from(value)]) - }) - } - - pub fn get_finalized_ever_updated(&self) -> Result> { - match self.read_bytes(&self.metadata, [FINALIZED_EVER_UPDATED])? { - Some(bytes) => Ok(Some(bytes.first().copied().unwrap_or(0) != 0)), - None => Ok(None), - } - } - - pub fn put_finalized_ever_updated(&self, value: bool) -> Result<()> { - self.write(|txn| { - self.metadata - .put(txn, [FINALIZED_EVER_UPDATED], [u8::from(value)]) - }) - } - - pub fn prune_before_slot(&self, slot: Slot, keep_roots: &HashSet) -> Result { - let bound = slot.0; - self.write(|txn| { - let mut roots: Vec = Vec::new(); - let mut block_index_keys: Vec> = Vec::new(); - self.blocks_by_slot.for_each(txn, |key, _value| { - if slot_of(key) >= bound { - return Ok(false); - } - let root = H256::from_slice(&key[8..]); - if !keep_roots.contains(&root) { - roots.push(root); - block_index_keys.push(key.to_vec()); - } - Ok(true) - })?; - - let mut removed = 0; - removed += self.blocks.delete_batch(txn, &roots)?; - removed += self.states.delete_batch(txn, &roots)?; - removed += self.blocks_by_slot.delete_batch(txn, &block_index_keys)?; - - Ok(removed) - }) - } - - pub fn load_since(&self, slot: Slot) -> Result> { - let bound = slot.0; - let txn = self.environment.begin_ro_txn()?; - - let mut roots: Vec = Vec::new(); - self.blocks_by_slot.for_each(&txn, |key, _value| { - if slot_of(key) >= bound { - roots.push(H256::from_slice(&key[8..])); - } - Ok(true) - })?; - - let mut entries = Vec::with_capacity(roots.len()); - for root in roots { - let block = match self.blocks.get(&txn, root)? { - Some(bytes) => Block::from_ssz_default(&bytes)?, - None => continue, - }; - let state = match self.states.get(&txn, root)? { - Some(bytes) => State::from_ssz_default(&bytes)?, - None => continue, - }; - entries.push((root, block, state)); - } - - txn.commit()?; - Ok(entries) - } - - pub fn commit_finalization( - &self, - head: H256, - finalized: Checkpoint, - safe_target: H256, - prune_before: Slot, - keep_roots: &HashSet, - ) -> Result { - let mut removed = 0; - self.batch_write(|| { - self.put_head_root(head)?; - self.put_finalized_checkpoint(finalized)?; - self.put_safe_target(safe_target)?; - removed = self.prune_before_slot(prune_before, keep_roots)?; - Ok(()) - })?; - Ok(removed) - } -} - -impl Default for Storage { - fn default() -> Self { - Self::new("./data").expect("failed to open default storage at ./data") - } -} - -fn slot_root_key(slot: Slot, root: H256) -> [u8; 40] { - let mut key = [0u8; 40]; - key[..8].copy_from_slice(&slot.0.to_be_bytes()); - key[8..].copy_from_slice(root.as_ref()); - key -} - -fn slot_of(key: &[u8]) -> u64 { - let mut slot = [0u8; 8]; - slot.copy_from_slice(&key[..8]); - u64::from_be_bytes(slot) -} - -#[derive(Debug)] -struct Database { - name: String, - compression: Compression, -} - -impl Database { - fn new(name: &str, compression: Compression) -> Self { - Self { - name: name.to_owned(), - compression, - } - } - - fn get( - &self, - txn: &Transaction, - key: impl AsRef<[u8]>, - ) -> Result>> { - let db = txn.open_db(Some(&self.name))?; - - txn.get::>(db.dbi(), key.as_ref())? - .map(|compressed| self.compression.decompress(&compressed)) - .transpose() - } - - fn put( - &self, - txn: &Transaction, - key: impl AsRef<[u8]>, - value: impl AsRef<[u8]>, - ) -> Result<()> { - let db = txn.open_db(Some(&self.name))?; - let compressed = self.compression.compress(value.as_ref())?; - txn.put(db.dbi(), key.as_ref(), compressed, WriteFlags::default())?; - Ok(()) - } - - fn for_each( - &self, - txn: &Transaction, - mut f: impl FnMut(&[u8], &[u8]) -> Result, - ) -> Result<()> { - let db = txn.open_db(Some(&self.name))?; - let mut cursor = txn.cursor(&db)?; - - while let Some((key, value)) = cursor.next::, Cow<[u8]>>()? { - let value = self.compression.decompress(&value)?; - if !f(key.as_ref(), &value)? { - break; - } - } - - Ok(()) - } - - fn delete_batch( - &self, - txn: &Transaction, - keys: impl IntoIterator>, - ) -> Result { - let db = txn.open_db(Some(&self.name))?; - let mut cursor = txn.cursor(&db)?; - - let mut deleted = 0; - for key in keys { - if cursor.set::<()>(key.as_ref())?.is_some() { - cursor.del(WriteFlags::default())?; - deleted += 1; - } - } - - Ok(deleted) - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -enum Compression { - None, - #[default] - Lz4, - Zstd, -} - -impl Compression { - fn compress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), - Self::Zstd => Ok(zstd::encode_all(data, 3)?), - } - } - - fn decompress(self, data: &[u8]) -> Result> { - match self { - Self::None => Ok(data.to_vec()), - Self::Lz4 => Ok(lz4_flex::decompress_size_prepended(data)?), - Self::Zstd => Ok(zstd::decode_all(data)?), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use containers::BlockBody; - use ssz::SszWrite; - use tempfile::TempDir; - - fn storage() -> (TempDir, Storage) { - let dir = tempfile::tempdir().expect("failed to create temp dir"); - let storage = Storage::new(dir.path()).expect("failed to open storage"); - (dir, storage) - } - - fn h256(byte: u8) -> H256 { - H256::from_slice(&[byte; 32]) - } - - fn sample_block(seed: u8) -> Block { - Block { - slot: Slot(u64::from(seed)), - proposer_index: u64::from(seed), - parent_root: h256(seed), - state_root: h256(seed.wrapping_add(1)), - body: BlockBody::default(), - } - } - - fn assert_ssz_eq(left: &T, right: &T) { - assert_eq!( - left.to_ssz().expect("left should serialize"), - right.to_ssz().expect("right should serialize"), - ); - } - - #[test] - fn get_block_returns_none_when_absent() { - let (_dir, storage) = storage(); - assert!(storage.get_block(h256(9)).unwrap().is_none()); - } - - #[test] - fn put_then_get_block_roundtrips() { - let (_dir, storage) = storage(); - let root = h256(1); - let block = sample_block(1); - - storage.put_block(block.clone(), root).unwrap(); - - let read = storage - .get_block(root) - .unwrap() - .expect("block should exist"); - assert_ssz_eq(&block, &read); - } - - #[test] - fn put_and_get_multiple_blocks() { - let (_dir, storage) = storage(); - let blocks: Vec<(H256, Block)> = (0..8).map(|i| (h256(i), sample_block(i))).collect(); - - for (root, block) in &blocks { - storage.put_block(block.clone(), *root).unwrap(); - } - - for (root, block) in &blocks { - let read = storage - .get_block(*root) - .unwrap() - .expect("block should exist"); - assert_ssz_eq(block, &read); - } - } - - #[test] - fn put_block_overwrites_existing_root() { - let (_dir, storage) = storage(); - let root = h256(3); - - storage.put_block(sample_block(3), root).unwrap(); - let updated = sample_block(7); - storage.put_block(updated.clone(), root).unwrap(); - - let read = storage - .get_block(root) - .unwrap() - .expect("block should exist"); - assert_ssz_eq(&updated, &read); - } - - #[test] - fn get_state_returns_none_when_absent() { - let (_dir, storage) = storage(); - assert!(storage.get_state(h256(9)).unwrap().is_none()); - } - - #[test] - fn put_then_get_state_roundtrips() { - let (_dir, storage) = storage(); - let root = h256(2); - let state = State::generate_genesis(1_234, 3); - - storage.put_state(state.clone(), root).unwrap(); - - let read = storage - .get_state(root) - .unwrap() - .expect("state should exist"); - assert_ssz_eq(&state, &read); - } - - #[test] - fn put_and_get_multiple_states() { - let (_dir, storage) = storage(); - let states: Vec<(H256, State)> = (0..4) - .map(|i| (h256(i), State::generate_genesis(u64::from(i), u64::from(i)))) - .collect(); - - for (root, state) in &states { - storage.put_state(state.clone(), *root).unwrap(); - } - - for (root, state) in &states { - let read = storage - .get_state(*root) - .unwrap() - .expect("state should exist"); - assert_ssz_eq(state, &read); - } - } - - #[test] - fn finalized_checkpoint_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_finalized_checkpoint().unwrap().is_none()); - - let checkpoint = Checkpoint { - root: h256(5), - slot: Slot(20), - }; - storage - .put_finalized_checkpoint(checkpoint.clone()) - .unwrap(); - - assert_eq!( - storage.get_finalized_checkpoint().unwrap().unwrap(), - checkpoint - ); - } - - #[test] - fn head_root_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_head_root().unwrap().is_none()); - - let root = h256(6); - storage.put_head_root(root).unwrap(); - - assert_eq!(storage.get_head_root().unwrap().unwrap(), root); - } - - #[test] - fn safe_target_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_safe_target().unwrap().is_none()); - - let root = h256(7); - storage.put_safe_target(root).unwrap(); - - assert_eq!(storage.get_safe_target().unwrap().unwrap(), root); - } - - #[test] - fn network_id_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_network_id().unwrap().is_none()); - - let id = h256(9); - storage.put_network_id(id).unwrap(); - - assert_eq!(storage.get_network_id().unwrap().unwrap(), id); - } - - #[test] - fn justified_checkpoint_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_justified_checkpoint().unwrap().is_none()); - - let checkpoint = Checkpoint { - root: h256(4), - slot: Slot(10), - }; - storage - .put_justified_checkpoint(checkpoint.clone()) - .unwrap(); - - assert_eq!( - storage.get_justified_checkpoint().unwrap().unwrap(), - checkpoint - ); - } - - #[test] - fn justified_ever_updated_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_justified_ever_updated().unwrap().is_none()); - - storage.put_justified_ever_updated(true).unwrap(); - assert!(storage.get_justified_ever_updated().unwrap().unwrap()); - - storage.put_justified_ever_updated(false).unwrap(); - assert!(!storage.get_justified_ever_updated().unwrap().unwrap()); - } - - #[test] - fn finalized_ever_updated_roundtrips() { - let (_dir, storage) = storage(); - assert!(storage.get_finalized_ever_updated().unwrap().is_none()); - - storage.put_finalized_ever_updated(true).unwrap(); - assert!(storage.get_finalized_ever_updated().unwrap().unwrap()); - - storage.put_finalized_ever_updated(false).unwrap(); - assert!(!storage.get_finalized_ever_updated().unwrap().unwrap()); - } - - #[test] - fn checkpoints_and_metadata_keys_do_not_collide() { - let (_dir, storage) = storage(); - let justified = Checkpoint { - root: h256(1), - slot: Slot(1), - }; - let finalized = Checkpoint { - root: h256(2), - slot: Slot(2), - }; - - storage.put_justified_checkpoint(justified.clone()).unwrap(); - storage.put_finalized_checkpoint(finalized.clone()).unwrap(); - storage.put_head_root(h256(3)).unwrap(); - storage.put_safe_target(h256(4)).unwrap(); - storage.put_network_id(h256(5)).unwrap(); - storage.put_justified_ever_updated(true).unwrap(); - storage.put_finalized_ever_updated(false).unwrap(); - - assert_eq!( - storage.get_justified_checkpoint().unwrap().unwrap(), - justified - ); - assert_eq!( - storage.get_finalized_checkpoint().unwrap().unwrap(), - finalized - ); - assert_eq!(storage.get_head_root().unwrap().unwrap(), h256(3)); - assert_eq!(storage.get_safe_target().unwrap().unwrap(), h256(4)); - assert_eq!(storage.get_network_id().unwrap().unwrap(), h256(5)); - assert!(storage.get_justified_ever_updated().unwrap().unwrap()); - assert!(!storage.get_finalized_ever_updated().unwrap().unwrap()); - } - - #[test] - fn load_since_returns_blocks_and_states_from_slot() { - let (_dir, storage) = storage(); - for i in 0..6u8 { - storage.put_block(sample_block(i), h256(i)).unwrap(); - storage - .put_state(state_at_slot(u64::from(i)), h256(i)) - .unwrap(); - } - - let mut loaded = storage.load_since(Slot(3)).unwrap(); - loaded.sort_by_key(|(_, block, _)| block.slot.0); - - let slots: Vec = loaded.iter().map(|(_, block, _)| block.slot.0).collect(); - assert_eq!(slots, vec![3, 4, 5]); - - for (root, block, state) in loaded { - assert_eq!(root, h256(u8::try_from(block.slot.0).unwrap())); - assert_eq!(state.slot, block.slot); - } - } - - #[test] - fn commit_finalization_persists_and_prunes() { - let (_dir, storage) = storage(); - for i in 0..6u8 { - storage.put_block(sample_block(i), h256(i)).unwrap(); - storage - .put_state(state_at_slot(u64::from(i)), h256(i)) - .unwrap(); - } - - let finalized = Checkpoint { - root: h256(4), - slot: Slot(4), - }; - let mut keep = HashSet::new(); - keep.insert(h256(4)); - - let removed = storage - .commit_finalization(h256(4), finalized.clone(), h256(5), Slot(4), &keep) - .unwrap(); - - assert!(removed > 0); - assert_eq!(storage.get_head_root().unwrap().unwrap(), h256(4)); - assert_eq!( - storage.get_finalized_checkpoint().unwrap().unwrap(), - finalized - ); - assert_eq!(storage.get_safe_target().unwrap().unwrap(), h256(5)); - assert!(storage.get_block(h256(0)).unwrap().is_none()); - assert!(storage.get_state(h256(0)).unwrap().is_none()); - assert!(storage.get_block(h256(4)).unwrap().is_some()); - assert!(storage.get_block(h256(5)).unwrap().is_some()); - } - - #[test] - fn checkpoints_database_keys_do_not_collide() { - let (_dir, storage) = storage(); - let finalized = Checkpoint { - root: h256(2), - slot: Slot(2), - }; - let head = h256(3); - - storage.put_finalized_checkpoint(finalized.clone()).unwrap(); - storage.put_head_root(head).unwrap(); - - assert_eq!( - storage.get_finalized_checkpoint().unwrap().unwrap(), - finalized - ); - assert_eq!(storage.get_head_root().unwrap().unwrap(), head); - } - - fn state_at_slot(slot: u64) -> State { - let mut state = State::generate_genesis(1_234, 3); - state.slot = Slot(slot); - state - } - - #[test] - fn prune_removes_blocks_below_slot_and_keeps_the_rest() { - let (_dir, storage) = storage(); - for i in 0..10u8 { - storage.put_block(sample_block(i), h256(i)).unwrap(); - } - - let removed = storage.prune_before_slot(Slot(5), &HashSet::new()).unwrap(); - assert_eq!(removed, 10); - - for i in 0..5u8 { - assert!(storage.get_block(h256(i)).unwrap().is_none()); - } - for i in 5..10u8 { - assert!(storage.get_block(h256(i)).unwrap().is_some()); - } - } - - #[test] - fn prune_keeps_frozen_block_root() { - let (_dir, storage) = storage(); - for i in 0..5u8 { - storage.put_block(sample_block(i), h256(i)).unwrap(); - } - - let mut keep = HashSet::new(); - keep.insert(h256(2)); - - storage.prune_before_slot(Slot(5), &keep).unwrap(); - - assert!(storage.get_block(h256(2)).unwrap().is_some()); - for i in [0u8, 1, 3, 4] { - assert!(storage.get_block(h256(i)).unwrap().is_none()); - } - } - - #[test] - fn prune_removes_forked_blocks_at_same_slot() { - let (_dir, storage) = storage(); - let mut fork = sample_block(2); - fork.proposer_index = 99; - storage.put_block(sample_block(2), h256(20)).unwrap(); - storage.put_block(fork, h256(21)).unwrap(); - - let removed = storage.prune_before_slot(Slot(5), &HashSet::new()).unwrap(); - - assert_eq!(removed, 4); - assert!(storage.get_block(h256(20)).unwrap().is_none()); - assert!(storage.get_block(h256(21)).unwrap().is_none()); - } - - #[test] - fn prune_removes_states_below_slot_and_keeps_frozen() { - let (_dir, storage) = storage(); - storage.put_block(sample_block(1), h256(1)).unwrap(); - storage.put_state(state_at_slot(1), h256(1)).unwrap(); - storage.put_block(sample_block(2), h256(2)).unwrap(); - storage.put_state(state_at_slot(2), h256(2)).unwrap(); - - let mut keep = HashSet::new(); - keep.insert(h256(2)); - - storage.prune_before_slot(Slot(5), &keep).unwrap(); - - assert!(storage.get_block(h256(1)).unwrap().is_none()); - assert!(storage.get_state(h256(1)).unwrap().is_none()); - assert!(storage.get_block(h256(2)).unwrap().is_some()); - assert!(storage.get_state(h256(2)).unwrap().is_some()); - } - - #[test] - fn prune_protection_is_per_entity() { - let (_dir, storage) = storage(); - storage.put_block(sample_block(1), h256(1)).unwrap(); - storage.put_state(state_at_slot(1), h256(1)).unwrap(); - - let mut keep = HashSet::new(); - keep.insert(h256(1)); - - storage.prune_before_slot(Slot(5), &keep).unwrap(); - - assert!(storage.get_block(h256(1)).unwrap().is_some()); - assert!(storage.get_state(h256(1)).unwrap().is_some()); - } - - #[test] - fn prune_below_lowest_slot_is_a_noop() { - let (_dir, storage) = storage(); - for i in 0..5u8 { - storage.put_block(sample_block(i), h256(i)).unwrap(); - } - - let removed = storage.prune_before_slot(Slot(0), &HashSet::new()).unwrap(); - assert_eq!(removed, 0); - for i in 0..5u8 { - assert!(storage.get_block(h256(i)).unwrap().is_some()); - } - } - - #[test] - fn batch_write_commits_all_on_success() { - let (_dir, storage) = storage(); - - storage - .batch_write(|| { - storage.put_block(sample_block(1), h256(1))?; - storage.put_state(state_at_slot(1), h256(2))?; - storage.put_head_root(h256(3))?; - Ok(()) - }) - .unwrap(); - - assert!(storage.get_block(h256(1)).unwrap().is_some()); - assert!(storage.get_state(h256(2)).unwrap().is_some()); - assert_eq!(storage.get_head_root().unwrap().unwrap(), h256(3)); - } - - #[test] - fn batch_write_rolls_back_every_write_on_error() { - let (_dir, storage) = storage(); - - let result = storage.batch_write(|| { - storage.put_block(sample_block(1), h256(1))?; - storage.put_state(state_at_slot(1), h256(2))?; - Err(anyhow::anyhow!("intentional failure")) - }); - - assert!(result.is_err()); - assert!(storage.get_block(h256(1)).unwrap().is_none()); - assert!(storage.get_state(h256(2)).unwrap().is_none()); - } - - #[test] - fn storage_is_usable_after_batch_write() { - let (_dir, storage) = storage(); - - storage - .batch_write(|| storage.put_block(sample_block(1), h256(1))) - .unwrap(); - - storage.put_block(sample_block(2), h256(2)).unwrap(); - - assert!(storage.get_block(h256(1)).unwrap().is_some()); - assert!(storage.get_block(h256(2)).unwrap().is_some()); - } - - #[test] - fn storage_is_usable_after_failed_batch_write() { - let (_dir, storage) = storage(); - - assert!( - storage - .batch_write(|| Err(anyhow::anyhow!("boom"))) - .is_err() - ); - - storage.put_block(sample_block(5), h256(5)).unwrap(); - assert!(storage.get_block(h256(5)).unwrap().is_some()); - } - - #[test] - fn batch_write_reads_see_pending_writes() { - let (_dir, storage) = storage(); - - storage - .batch_write(|| { - storage.put_block(sample_block(7), h256(7)).unwrap(); - assert!(storage.get_block(h256(7)).unwrap().is_some()); - Ok(()) - }) - .unwrap(); - } -} From 3fd2338338c32f5213630becefca983a3d494654 Mon Sep 17 00:00:00 2001 From: Sagar Rana Date: Thu, 23 Jul 2026 01:52:14 +0530 Subject: [PATCH 20/20] changes --- lean_client/Cargo.lock | 1 + lean_client/database/src/lib.rs | 341 +++++++++++++++++- lean_client/fork_choice/src/handlers.rs | 11 + lean_client/fork_choice/src/store.rs | 139 ++++++- .../tests/fork_choice_test_vectors.rs | 3 +- .../fork_choice/tests/unit_tests/common.rs | 2 +- .../fork_choice/tests/unit_tests/validator.rs | 19 +- lean_client/http_api/Cargo.toml | 1 + lean_client/http_api/src/test_driver.rs | 6 +- lean_client/http_api/tests/api_endpoint.rs | 18 + lean_client/src/main.rs | 2 +- 11 files changed, 513 insertions(+), 30 deletions(-) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 38f6ba4b..9c8ea193 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -2415,6 +2415,7 @@ dependencies = [ "axum", "clap", "containers", + "database", "fork_choice", "futures", "hex", diff --git a/lean_client/database/src/lib.rs b/lean_client/database/src/lib.rs index 23721dec..9b33d034 100644 --- a/lean_client/database/src/lib.rs +++ b/lean_client/database/src/lib.rs @@ -1,9 +1,10 @@ use std::{ borrow::Cow, + collections::HashMap, marker::PhantomData, ops::{Bound, RangeBounds}, path::PathBuf, - sync::Arc, + sync::{Arc, Mutex, OnceLock, Weak}, }; use anyhow::{Ok, Result}; @@ -19,11 +20,11 @@ use libmdbx::{ const GIB: usize = 1 << 30; const MIB: usize = 1 << 20; -const BLOCKS_TABLE_NAME: &str = "blocks"; -const GENESIS_STATE_TABLE_NAME: &str = "genesis_state"; +pub const BLOCKS_TABLE_NAME: &str = "blocks"; +pub const GENESIS_STATE_TABLE_NAME: &str = "genesis_state"; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -enum Compression { +pub enum Compression { None, #[default] Lz4, @@ -54,20 +55,27 @@ pub struct EnvironmentBuilder { } impl EnvironmentBuilder { - pub fn new(path: impl Into) -> Self { + pub fn new(path: impl Into, gib: usize) -> Self { Self { path: path.into(), - max_size: 128 * GIB, + max_size: gib * GIB, } } - pub fn max_size(mut self, bytes: usize) -> Self { - self.max_size = bytes; - self - } - pub fn build(&self) -> Result> { + static ENVIRONMENTS: OnceLock>>> = OnceLock::new(); + std::fs::create_dir_all(&self.path)?; + let canonical = self.path.canonicalize()?; + + let mut environments = ENVIRONMENTS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + + if let Some(env) = environments.get(&canonical).and_then(Weak::upgrade) { + return Ok(env); + } let env = Environment::builder() .set_max_dbs(2) @@ -87,13 +95,16 @@ impl EnvironmentBuilder { shrink_threshold: None, page_size: None, }) - .open(&self.path)?; + .open(&canonical)?; - Ok(Arc::new(env)) + let env = Arc::new(env); + environments.insert(canonical, Arc::downgrade(&env)); + Ok(env) } } -struct Database { +#[derive(Debug, Clone)] +pub struct Database { env: Arc, name: &'static str, compression: Compression, @@ -101,7 +112,11 @@ struct Database { } impl Database { - pub fn new(env: Arc, name: &'static str, compression: Compression) -> Result { + pub fn new( + env: Arc, + name: &'static str, + compression: Compression, + ) -> Result { let txn = env.begin_rw_txn()?; txn.create_db(Some(name), DatabaseFlags::default())?; txn.commit()?; @@ -271,11 +286,299 @@ impl KeyPrefix for BySlot { } } +const STATE_KEY_PREFIX: &[u8] = b"state"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StateKey(pub Slot); + +impl Key for StateKey { + fn encode(&self) -> Vec { + let mut b = Vec::with_capacity(STATE_KEY_PREFIX.len() + 8); + b.extend_from_slice(STATE_KEY_PREFIX); + b.extend_from_slice(&self.0.0.to_be_bytes()); + b + } +} + +impl KeyPrefix for StateKey { + fn encode(&self) -> Vec { + Key::encode(self) + } +} + #[derive(Clone, Copy, Debug)] -pub struct GenesisKey; +pub struct StateKeyPrefix; -impl Key for GenesisKey { +impl KeyPrefix for StateKeyPrefix { fn encode(&self) -> Vec { - vec![0] + STATE_KEY_PREFIX.to_vec() + } +} + +#[cfg(test)] +mod tests { + use containers::{Block, BlockBody, State, Validator}; + + use super::*; + + fn test_db_path(name: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "lean_database_test_{}_{}", + std::process::id(), + name + )); + std::fs::remove_dir_all(&path).ok(); + path + } + + fn test_env(name: &str) -> Arc { + EnvironmentBuilder::new(test_db_path(name), 1) + .build() + .expect("failed to open test environment") + } + + fn blocks_db(env: Arc) -> Database { + Database::new(env, BLOCKS_TABLE_NAME, Compression::Lz4) + .expect("failed to open blocks table") + } + + fn states_db(env: Arc) -> Database { + Database::new(env, GENESIS_STATE_TABLE_NAME, Compression::Zstd) + .expect("failed to open states table") + } + + fn h256(byte: u8) -> H256 { + H256::from_slice(&[byte; 32]) + } + + fn sample_block(slot: u64, root_byte: u8) -> Block { + Block { + slot: Slot(slot), + proposer_index: 0, + parent_root: H256::default(), + state_root: h256(root_byte), + body: BlockBody::default(), + } + } + + fn sample_state(genesis_time: u64) -> State { + State::generate_genesis_with_validators(genesis_time, vec![Validator::default(); 2]) + } + + fn block_key(slot: u64, root_byte: u8) -> BlockKey { + BlockKey { + slot: Slot(slot), + root: h256(root_byte), + } } -} \ No newline at end of file + + fn assert_ssz_eq(left: &T, right: &T) { + assert_eq!( + left.to_ssz().expect("left should serialize"), + right.to_ssz().expect("right should serialize"), + ); + } + + #[test] + fn compression_roundtrips() { + let data = vec![42u8; 1024]; + for compression in [Compression::None, Compression::Lz4, Compression::Zstd] { + let compressed = compression.compress(&data).expect("compress should succeed"); + let decompressed = compression + .decompress(&compressed) + .expect("decompress should succeed"); + assert_eq!(decompressed, data); + } + } + + #[test] + fn get_missing_key_returns_none() { + let db = blocks_db(test_env("get_missing")); + assert!( + db.get(&block_key(1, 1)) + .expect("get should succeed") + .is_none() + ); + } + + #[test] + fn put_then_get_roundtrips() { + let db = blocks_db(test_env("put_get")); + let block = sample_block(3, 7); + db.put(&block_key(3, 7), &block).expect("put should succeed"); + let read = db + .get(&block_key(3, 7)) + .expect("get should succeed") + .expect("value should exist"); + assert_ssz_eq(&read, &block); + } + + #[test] + fn put_overwrites_existing_value() { + let db = blocks_db(test_env("overwrite")); + let key = block_key(1, 1); + db.put(&key, &sample_block(1, 1)).expect("put should succeed"); + let updated = sample_block(1, 9); + db.put(&key, &updated).expect("second put should succeed"); + let read = db + .get(&key) + .expect("get should succeed") + .expect("value should exist"); + assert_ssz_eq(&read, &updated); + } + + #[test] + fn range_returns_blocks_in_slot_order() { + let db = blocks_db(test_env("range_order")); + for slot in [300u64, 0, 256, 255, 2] { + db.put(&block_key(slot, slot as u8), &sample_block(slot, slot as u8)) + .expect("put should succeed"); + } + let slots: Vec = db + .range::(..) + .expect("range should succeed") + .iter() + .map(|b| b.slot.0) + .collect(); + assert_eq!(slots, vec![0, 2, 255, 256, 300]); + } + + #[test] + fn range_respects_bounds() { + let db = blocks_db(test_env("range_bounds")); + for slot in 0..6u64 { + db.put(&block_key(slot, slot as u8), &sample_block(slot, slot as u8)) + .expect("put should succeed"); + } + + let slots = |blocks: Vec| blocks.iter().map(|b| b.slot.0).collect::>(); + + let from = db + .range(BySlot(Slot(2))..) + .expect("range from should succeed"); + assert_eq!(slots(from), vec![2, 3, 4, 5]); + + let below = db + .range(..BySlot(Slot(3))) + .expect("range below should succeed"); + assert_eq!(slots(below), vec![0, 1, 2]); + + let inclusive = db + .range(..=BySlot(Slot(3))) + .expect("inclusive range should succeed"); + assert_eq!(slots(inclusive), vec![0, 1, 2, 3]); + } + + #[test] + fn range_returns_all_forks_at_a_slot() { + let db = blocks_db(test_env("range_forks")); + db.put(&block_key(4, 1), &sample_block(4, 1)).expect("put should succeed"); + db.put(&block_key(4, 2), &sample_block(4, 2)).expect("put should succeed"); + db.put(&block_key(5, 3), &sample_block(5, 3)).expect("put should succeed"); + + let at_slot_4 = db + .range(BySlot(Slot(4))..=BySlot(Slot(4))) + .expect("range should succeed"); + assert_eq!(at_slot_4.len(), 2); + assert!(at_slot_4.iter().all(|b| b.slot.0 == 4)); + } + + #[test] + fn delete_range_removes_and_counts() { + let db = blocks_db(test_env("delete_range")); + for slot in 0..6u64 { + db.put(&block_key(slot, slot as u8), &sample_block(slot, slot as u8)) + .expect("put should succeed"); + } + + let deleted = db + .delete_range(..BySlot(Slot(3))) + .expect("delete_range should succeed"); + assert_eq!(deleted, 3); + + assert!( + db.get(&block_key(0, 0)) + .expect("get should succeed") + .is_none() + ); + let remaining = db.range::(..).expect("range should succeed"); + assert_eq!(remaining.len(), 3); + } + + #[test] + fn state_key_prefix_returns_newest_state() { + let db = states_db(test_env("state_newest")); + db.put(&StateKey(Slot(0)), &sample_state(1000)) + .expect("put should succeed"); + db.put(&StateKey(Slot(64)), &sample_state(2000)) + .expect("put should succeed"); + + let newest = db + .range(StateKeyPrefix..) + .expect("range should succeed") + .pop() + .expect("state should exist"); + assert_eq!(newest.config.genesis_time, 2000); + } + + #[test] + fn delete_range_prunes_old_states() { + let db = states_db(test_env("state_prune")); + db.put(&StateKey(Slot(0)), &sample_state(1000)) + .expect("put should succeed"); + db.put(&StateKey(Slot(64)), &sample_state(2000)) + .expect("put should succeed"); + + let deleted = db + .delete_range(..StateKey(Slot(64))) + .expect("delete_range should succeed"); + assert_eq!(deleted, 1); + + assert!( + db.get(&StateKey(Slot(0))) + .expect("get should succeed") + .is_none() + ); + assert!( + db.get(&StateKey(Slot(64))) + .expect("get should succeed") + .is_some() + ); + } + + #[test] + fn environment_builder_reuses_live_environment() { + let path = test_db_path("registry_reuse"); + let first = EnvironmentBuilder::new(path.clone(), 1) + .build() + .expect("first build should succeed"); + let second = EnvironmentBuilder::new(path, 1) + .build() + .expect("second build should succeed"); + assert!(Arc::ptr_eq(&first, &second)); + } + + #[test] + fn values_persist_across_environment_reopen() { + let path = test_db_path("reopen"); + let block = sample_block(7, 7); + { + let db = blocks_db( + EnvironmentBuilder::new(path.clone(), 1) + .build() + .expect("first build should succeed"), + ); + db.put(&block_key(7, 7), &block).expect("put should succeed"); + } + let db = blocks_db( + EnvironmentBuilder::new(path, 1) + .build() + .expect("reopen should succeed"), + ); + let read = db + .get(&block_key(7, 7)) + .expect("get should succeed") + .expect("value should survive reopen"); + assert_ssz_eq(&read, &block); + } +} diff --git a/lean_client/fork_choice/src/handlers.rs b/lean_client/fork_choice/src/handlers.rs index e6fcf7fd..17069c96 100644 --- a/lean_client/fork_choice/src/handlers.rs +++ b/lean_client/fork_choice/src/handlers.rs @@ -6,6 +6,7 @@ use containers::{ AttestationData, Checkpoint, SignatureKey, SignedAggregatedAttestation, SignedAttestation, SignedBlock, State, }; +use database::BlockKey; use metrics::METRICS; use parking_lot::RwLock; use ssz::{H256, SszHash}; @@ -727,6 +728,16 @@ pub fn apply_verified_block( store.blocks.insert(block_root, block.clone()); store.states.insert(block_root, Arc::clone(&new_state)); + if let Err(error) = store.blocks_db.put( + &BlockKey { + slot: block.slot, + root: block_root, + }, + &block, + ) { + warn!("failed to persist block to database: {error}"); + } + METRICS.get().map(|m| { m.grandine_store_blocks_size.set(store.blocks.len() as i64); m.grandine_store_states_size.set(store.states.len() as i64); diff --git a/lean_client/fork_choice/src/store.rs b/lean_client/fork_choice/src/store.rs index 100d4a4c..559ea312 100644 --- a/lean_client/fork_choice/src/store.rs +++ b/lean_client/fork_choice/src/store.rs @@ -1,12 +1,17 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use crate::block_cache::BlockCache; +use crate::handlers::on_block; use anyhow::{Result, anyhow, ensure}; use containers::{ AggregatedSignatureProof, AttestationData, Block, BlockHeader, Checkpoint, Config, SignatureKey, SignedAggregatedAttestation, SignedAttestation, SignedBlock, Slot, State, }; -use database::Database; +use database::{ + BLOCKS_TABLE_NAME, BlockKey, BySlot, Compression, Database, EnvironmentBuilder, + GENESIS_STATE_TABLE_NAME, StateKey, StateKeyPrefix, +}; use indexmap::IndexMap; use metrics::{METRICS, set_gauge_u64}; use ssz::{H256, SszHash}; @@ -110,7 +115,9 @@ pub struct Store { pub log_inv_rate: usize, - // pub database: Arc, + pub blocks_db: Database, + + pub genesis_db: Database, } const JUSTIFICATION_LOOKBACK_SLOTS: u64 = 3; @@ -227,7 +234,7 @@ pub fn get_forkchoice_store( config: Config, is_aggregator: bool, log_inv_rate: usize, -) -> Store { +) -> Result { // Extract the plain Block from the signed block let block = anchor_block.block.clone(); let block_slot = block.slot; @@ -264,10 +271,106 @@ pub fn get_forkchoice_store( let latest_justified = anchor_checkpoint.clone(); let latest_finalized = anchor_checkpoint; + let env_builder = EnvironmentBuilder::new("./data", 1); + let env = env_builder.build()?; + + let blocks_db: Database = + Database::new(env.clone(), BLOCKS_TABLE_NAME, Compression::Lz4)?; + let genesis_db: Database = + Database::new(env, GENESIS_STATE_TABLE_NAME, Compression::Zstd)?; + + if let Some(anchor_state_db) = genesis_db.range(StateKeyPrefix..)?.pop() { + let anchor_slot = anchor_state_db.latest_block_header.slot; + let anchor_header = BlockHeader { + slot: anchor_state_db.latest_block_header.slot, + proposer_index: anchor_state_db.latest_block_header.proposer_index, + parent_root: anchor_state_db.latest_block_header.parent_root, + state_root: anchor_state_db.hash_tree_root(), + body_root: anchor_state_db.latest_block_header.body_root, + }; + let anchor_root = anchor_header.hash_tree_root(); + if let Some(anchor_block_db) = blocks_db.get(&BlockKey { + slot: anchor_slot, + root: anchor_root, + })? { + let anchor_checkpoint = Checkpoint { + root: anchor_root, + slot: anchor_block_db.slot, + }; + + let mut store = Store { + time: anchor_block_db.slot.0 * INTERVALS_PER_SLOT, + config, + is_aggregator, + head: anchor_root, + safe_target: anchor_root, + latest_justified: anchor_checkpoint.clone(), + latest_finalized: anchor_checkpoint, + justified_ever_updated: anchor_block_db.slot.0 == 0, + finalized_ever_updated: false, + blocks: { + let mut m = HashMap::new(); + m.insert(anchor_root, anchor_block_db); + m + }, + states: { + let mut m = HashMap::new(); + m.insert(anchor_root, Arc::new(anchor_state_db)); + m + }, + latest_known_attestations: HashMap::new(), + latest_new_attestations: HashMap::new(), + gossip_signatures: HashMap::new(), + latest_known_aggregated_payloads: IndexMap::new(), + latest_new_aggregated_payloads: IndexMap::new(), + attestation_data_by_root: HashMap::new(), + pending_attestations: HashMap::new(), + pending_aggregated_attestations: HashMap::new(), + pending_fetch_roots: HashSet::new(), + log_inv_rate, + blocks_db, + genesis_db, + }; + + let mut cache = BlockCache::new(); + for block in store.blocks_db.range(BySlot(anchor_slot)..)? { + let signed_block = SignedBlock { + block, + proof: Default::default(), + }; + if let Err(error) = on_block(&mut store, &mut cache, signed_block, false) { + warn!("failed to replay block from database: {error}"); + } + } + + let prune_cutoff = store + .latest_finalized + .slot + .0 + .saturating_sub(STATE_PRUNE_BUFFER); + if prune_cutoff > 0 { + store.blocks_db.delete_range(..BySlot(Slot(prune_cutoff)))?; + } + + return Ok(store); + } + } + + if block_slot.0 == 0 { + genesis_db.put(&StateKey(block_slot), &anchor_state)?; + } + blocks_db.put( + &BlockKey { + slot: block_slot, + root: block_root, + }, + &block, + )?; + // Store the original anchor_state - do NOT modify it // Modifying checkpoints would change its hash_tree_root(), breaking the // consistency with block.state_root - Store { + Ok(Store { time: block_slot.0 * INTERVALS_PER_SLOT, config, is_aggregator, @@ -297,7 +400,9 @@ pub fn get_forkchoice_store( pending_aggregated_attestations: HashMap::new(), pending_fetch_roots: HashSet::new(), log_inv_rate, - } + blocks_db, + genesis_db, + }) } pub fn get_fork_choice_head( @@ -422,6 +527,7 @@ pub fn update_head(store: &mut Store) { } if let Some(block) = store.blocks.get(&finalized_root) { if block.slot == finalized_slot { + let finalized_advanced = finalized_slot > store.latest_finalized.slot; store.latest_finalized = Checkpoint { root: finalized_root, slot: finalized_slot, @@ -432,6 +538,29 @@ pub fn update_head(store: &mut Store) { m.lean_latest_finalized_slot.set(s); } }); + if finalized_advanced { + if let Some(finalized_state) = store.states.get(&finalized_root) { + if let Err(error) = store + .genesis_db + .put(&StateKey(finalized_slot), finalized_state.as_ref()) + { + warn!("failed to persist finalized state to database: {error}"); + } + if let Err(error) = + store.genesis_db.delete_range(..StateKey(finalized_slot)) + { + warn!("failed to prune old finalized states from database: {error}"); + } + } + let prune_cutoff = finalized_slot.0.saturating_sub(STATE_PRUNE_BUFFER); + if prune_cutoff > 0 { + if let Err(error) = + store.blocks_db.delete_range(..BySlot(Slot(prune_cutoff))) + { + warn!("failed to prune blocks from database: {error}"); + } + } + } } } } diff --git a/lean_client/fork_choice/tests/fork_choice_test_vectors.rs b/lean_client/fork_choice/tests/fork_choice_test_vectors.rs index 514e3e2b..d0c7820c 100644 --- a/lean_client/fork_choice/tests/fork_choice_test_vectors.rs +++ b/lean_client/fork_choice/tests/fork_choice_test_vectors.rs @@ -469,7 +469,8 @@ fn forkchoice(spec_file: &str) { body_root, }; - let mut store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1); + let mut store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1) + .expect("failed to create test store"); let mut cache = BlockCache::new(); let mut block_labels: HashMap = HashMap::new(); diff --git a/lean_client/fork_choice/tests/unit_tests/common.rs b/lean_client/fork_choice/tests/unit_tests/common.rs index c9c71da2..8ff3da3a 100644 --- a/lean_client/fork_choice/tests/unit_tests/common.rs +++ b/lean_client/fork_choice/tests/unit_tests/common.rs @@ -22,5 +22,5 @@ pub fn create_test_store() -> Store { proof: Default::default(), }; - get_forkchoice_store(state, signed_block, config, true, 1) + get_forkchoice_store(state, signed_block, config, true, 1).expect("failed to create test store") } diff --git a/lean_client/fork_choice/tests/unit_tests/validator.rs b/lean_client/fork_choice/tests/unit_tests/validator.rs index 948a2977..3dec435a 100644 --- a/lean_client/fork_choice/tests/unit_tests/validator.rs +++ b/lean_client/fork_choice/tests/unit_tests/validator.rs @@ -9,6 +9,9 @@ use containers::{ AggregatedSignatureProof, AggregationBits, Attestation, AttestationData, Block, BlockBody, Checkpoint, Config, MultiMessageAggregate, SignedBlock, Slot, State, Validator, }; +use database::{ + BLOCKS_TABLE_NAME, Compression, Database, EnvironmentBuilder, GENESIS_STATE_TABLE_NAME, +}; use fork_choice::block_cache::BlockCache; use fork_choice::handlers::on_block; use fork_choice::store::{Store, get_forkchoice_store, produce_block_with_signatures, update_head}; @@ -122,7 +125,8 @@ fn create_test_store_with_signers() -> (Store, HashMap) { }; ( - get_forkchoice_store(state, signed_block, config, true, 1), + get_forkchoice_store(state, signed_block, config, true, 1) + .expect("failed to create test store"), keys, ) } @@ -514,7 +518,8 @@ fn test_validator_operations_empty_store() { proof: Default::default(), }; - let mut store = get_forkchoice_store(state, signed_block, config, true, 1); + let mut store = get_forkchoice_store(state, signed_block, config, true, 1) + .expect("failed to create test store"); // Should be able to produce block and attestation let (_root, block, _sig) = @@ -554,6 +559,14 @@ fn test_produce_block_missing_parent_state() { }; // Create store with missing parent state + let env = EnvironmentBuilder::new(std::env::temp_dir().join("lean_test_missing_parent_db"), 1) + .build() + .expect("failed to open test database environment"); + let blocks_db = Database::new(env.clone(), BLOCKS_TABLE_NAME, Compression::Lz4) + .expect("failed to open blocks table"); + let genesis_db = Database::new(env, GENESIS_STATE_TABLE_NAME, Compression::Zstd) + .expect("failed to open genesis table"); + let store = Store { time: 100, config: Config { genesis_time: 1000 }, @@ -576,6 +589,8 @@ fn test_produce_block_missing_parent_state() { pending_aggregated_attestations: Default::default(), pending_fetch_roots: Default::default(), log_inv_rate: 1, + blocks_db, + genesis_db, }; let mut s = store; diff --git a/lean_client/http_api/Cargo.toml b/lean_client/http_api/Cargo.toml index 42627d89..97817181 100644 --- a/lean_client/http_api/Cargo.toml +++ b/lean_client/http_api/Cargo.toml @@ -24,6 +24,7 @@ tracing = { workspace = true } xmss = { workspace = true } [dev-dependencies] +database = { workspace = true } fork_choice = { workspace = true } http-body-util = { workspace = true } parking_lot = { workspace = true } diff --git a/lean_client/http_api/src/test_driver.rs b/lean_client/http_api/src/test_driver.rs index afa6a4ab..baeb71a9 100644 --- a/lean_client/http_api/src/test_driver.rs +++ b/lean_client/http_api/src/test_driver.rs @@ -179,7 +179,11 @@ async fn init_fork_choice( genesis_time: anchor_state.config.genesis_time, }; - let new_store = get_forkchoice_store(anchor_state, anchor_block, config, false, 1); + let std::result::Result::Ok(new_store) = + get_forkchoice_store(anchor_state, anchor_block, config, false, 1) + else { + return StatusCode::INTERNAL_SERVER_ERROR; + }; *state.store.write() = new_store; *state.cache.write() = BlockCache::new(); diff --git a/lean_client/http_api/tests/api_endpoint.rs b/lean_client/http_api/tests/api_endpoint.rs index bc52959b..09865ad9 100644 --- a/lean_client/http_api/tests/api_endpoint.rs +++ b/lean_client/http_api/tests/api_endpoint.rs @@ -12,6 +12,9 @@ use axum::{ body::Body, http::{Request, header::CONTENT_TYPE}, }; +use database::{ + BLOCKS_TABLE_NAME, Compression, Database, EnvironmentBuilder, GENESIS_STATE_TABLE_NAME, +}; use fork_choice::store::Store; use http_api::{AggregatorController, HttpServerConfig, SharedStore, normal_routes}; use http_body_util::BodyExt; @@ -68,6 +71,19 @@ fn api_endpoint(spec_file: &str) { // Build a minimal store — aggregator handlers only read/write // `store.is_aggregator`; no chain state is needed. let initial_is_aggregator = case.initial_is_aggregator.unwrap_or(false); + let env = EnvironmentBuilder::new( + std::env::temp_dir().join(format!( + "lean_api_test_db_{}", + spec_file.replace(['/', '\\'], "_") + )), + 1, + ) + .build() + .expect("failed to open test database environment"); + let blocks_db = Database::new(env.clone(), BLOCKS_TABLE_NAME, Compression::Lz4) + .expect("failed to open blocks table"); + let genesis_db = Database::new(env, GENESIS_STATE_TABLE_NAME, Compression::Zstd) + .expect("failed to open genesis table"); let store: SharedStore = Arc::new(RwLock::new(Store { time: Default::default(), config: Default::default(), @@ -90,6 +106,8 @@ fn api_endpoint(spec_file: &str) { pending_aggregated_attestations: Default::default(), pending_fetch_roots: Default::default(), log_inv_rate: Default::default(), + blocks_db, + genesis_db, })); let controller = Some(Arc::new(AggregatorController::new(store.clone(), None))); diff --git a/lean_client/src/main.rs b/lean_client/src/main.rs index de675c45..e25448ce 100644 --- a/lean_client/src/main.rs +++ b/lean_client/src/main.rs @@ -1069,7 +1069,7 @@ async fn main() -> Result<()> { config.clone(), args.is_aggregator, genesis_log_inv_rate as usize, - ))); + )?)); // Seed the block provider so we can serve the anchor block to peers via BlocksByRoot. {