From 302f7effdf7c3d9220d4fd1d391c543a42ec04b0 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 17:08:13 +0200 Subject: [PATCH 1/2] perf(drive-abci): don't create GroveDB checkpoints while replaying history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Protocol version 11 turns checkpoints on, with a policy of one every 600 seconds of chain time keeping the last 3. Mainnet blocks are about 2.5 minutes apart, so that is a checkpoint every four blocks — which at replay speed is roughly 37 a second, each one a RocksDB checkpoint over the whole database plus a copy of the platform state, and all but the last three deleted again immediately. Measured at 15.1 ms a block from height 318,704, against about 7 ms for everything else in a block put together. A finished mainnet sync had four checkpoint directories left on disk out of some 26,000 created. Checkpoints are restore points for a running node, so skip them for blocks more than ten minutes old and write the first real one on reaching the tip. They live outside the tree, so no app hash changes. --- .../block_end/should_checkpoint/v0/mod.rs | 54 +++++++++++++++++- packages/rs-drive-abci/src/utils/mod.rs | 2 + packages/rs-drive-abci/src/utils/replay.rs | 55 +++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 packages/rs-drive-abci/src/utils/replay.rs diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs index f16198bc266..79d2034f99c 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs @@ -53,6 +53,16 @@ where let block_time = block_info.block_time_ms(); let block_height = block_info.height(); + // Checkpoints are restore points for a running node. Replaying history, + // ten minutes of chain time is a handful of blocks, so this fires dozens + // of times a second and all but the last `keep_n` are deleted again + // immediately — each one a RocksDB checkpoint over the whole database + // plus a copy of the platform state. The node writes its first real + // checkpoint once it reaches the tip. + if crate::utils::is_historical_block(block_time) { + return Ok(None); + } + let most_recent_checkpoint_interval_time = block_time - block_time % checkpoint_interval_milliseconds; @@ -95,6 +105,13 @@ mod tests { use dpp::version::PlatformVersion; use std::collections::BTreeMap; + fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock is before the unix epoch") + .as_millis() as u64 + } + fn make_block_execution_context(height: u64, block_time_ms: u64) -> BlockExecutionContext { let platform_version = PlatformVersion::latest(); let platform_state = @@ -158,7 +175,10 @@ mod tests { return; } - let block_execution_context = make_block_execution_context(1, 1_000_000); + // A block the network has just produced: checkpoints are restore points + // for a running node, so the age of the block decides whether one is worth + // taking, and a fixed fixture timestamp would read as ancient history. + let block_execution_context = make_block_execution_context(1, now_ms()); let result = platform .should_checkpoint_v0(&block_execution_context, platform_version) .expect("expected Ok"); @@ -167,6 +187,38 @@ mod tests { assert!(result.is_some(), "first block should trigger checkpoint"); } + /// Replaying history, ten minutes of chain time is a handful of blocks, so a + /// checkpoint would be taken dozens of times a second and all but the last + /// few deleted again immediately. A node catching up takes none. + #[test] + fn test_historical_block_does_not_checkpoint() { + let platform_version = PlatformVersion::latest(); + if platform_version + .drive_abci + .methods + .block_end + .should_checkpoint + .is_none() + { + return; + } + + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let block_execution_context = + make_block_execution_context(1, now_ms() - 24 * 60 * 60 * 1000); + let result = platform + .should_checkpoint_v0(&block_execution_context, platform_version) + .expect("expected Ok"); + + assert!( + result.is_none(), + "a day-old block is being replayed, not followed" + ); + } + #[test] fn test_checkpoint_interval_zero_returns_none() { let platform_version = PlatformVersion::latest(); diff --git a/packages/rs-drive-abci/src/utils/mod.rs b/packages/rs-drive-abci/src/utils/mod.rs index b7292f50cff..3ca96761bdb 100644 --- a/packages/rs-drive-abci/src/utils/mod.rs +++ b/packages/rs-drive-abci/src/utils/mod.rs @@ -1,6 +1,8 @@ +mod replay; mod serialization; mod spawn; +pub use replay::is_historical_block; pub use serialization::from_opt_str_or_number; pub use serialization::from_str_or_number; pub use spawn::spawn_blocking_task_with_name_if_supported; diff --git a/packages/rs-drive-abci/src/utils/replay.rs b/packages/rs-drive-abci/src/utils/replay.rs new file mode 100644 index 00000000000..47b89a3d96d --- /dev/null +++ b/packages/rs-drive-abci/src/utils/replay.rs @@ -0,0 +1,55 @@ +//! Telling a node that is replaying history from one that is following the tip. +//! +//! Some per-block work only earns its cost at the tip. Creating a GroveDB +//! checkpoint every ten minutes of chain time is useful on a running node and +//! pure waste while catching up, where ten minutes of chain time is a handful of +//! blocks and every checkpoint but the last few is deleted within the second. + +/// A block older than this is not one the network just produced. Mainnet aims at +/// about 2.5 minutes a block, so this leaves several blocks of slack for a node +/// that is merely a little behind. +const HISTORICAL_BLOCK_AGE_MS: u64 = 10 * 60 * 1000; + +/// True when a block with this timestamp is old enough that the node producing +/// it is clearly replaying history rather than following the tip. +pub fn is_historical_block(block_time_ms: u64) -> bool { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|since_epoch| since_epoch.as_millis() as u64) + .unwrap_or(0); + now_ms.saturating_sub(block_time_ms) > HISTORICAL_BLOCK_AGE_MS +} + +#[cfg(test)] +mod tests { + use super::*; + + fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock is before the unix epoch") + .as_millis() as u64 + } + + #[test] + fn a_block_from_a_year_ago_is_historical() { + assert!(is_historical_block( + now_ms() - 365 * 24 * 60 * 60 * 1000 + )); + } + + #[test] + fn a_block_from_a_minute_ago_is_not_historical() { + assert!(!is_historical_block(now_ms() - 60 * 1000)); + } + + #[test] + fn a_block_at_the_threshold_is_not_yet_historical() { + assert!(!is_historical_block(now_ms() - HISTORICAL_BLOCK_AGE_MS)); + } + + #[test] + fn a_block_timestamped_in_the_future_is_not_historical() { + assert!(!is_historical_block(now_ms() + 60 * 1000)); + } +} From 9dfffa611a9554cb14c9464374c8de1356c1d92f Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 7 Sep 2026 16:42:45 -0500 Subject: [PATCH 2/2] test(drive-abci): keep the checkpoint simulations running under the replay skip The chain simulations start at a fixed 2023 timestamp, so with checkpoints skipped for historical blocks none of the seven checkpoint-enabled tests ever took one, and CI only runs them after merge. Give NetworkStrategy a start_time_ms and start those chains near the wall clock, at the same phase within the ten-minute interval so the asserted checkpoint heights are unchanged. Also: format replay.rs, test the age threshold against an injected clock instead of racing SystemTime, make is_historical_block crate-private, and write down the chain-halt limitation of the age heuristic. --- packages/rs-drive-abci/src/utils/mod.rs | 2 +- packages/rs-drive-abci/src/utils/replay.rs | 42 ++++++++++----- .../tests/strategy_tests/execution.rs | 4 +- .../tests/strategy_tests/strategy.rs | 6 +++ .../test_cases/address_tests.rs | 54 +++++++++++++++++-- 5 files changed, 89 insertions(+), 19 deletions(-) diff --git a/packages/rs-drive-abci/src/utils/mod.rs b/packages/rs-drive-abci/src/utils/mod.rs index 3ca96761bdb..bfebd6dcd62 100644 --- a/packages/rs-drive-abci/src/utils/mod.rs +++ b/packages/rs-drive-abci/src/utils/mod.rs @@ -2,7 +2,7 @@ mod replay; mod serialization; mod spawn; -pub use replay::is_historical_block; +pub(crate) use replay::is_historical_block; pub use serialization::from_opt_str_or_number; pub use serialization::from_str_or_number; pub use spawn::spawn_blocking_task_with_name_if_supported; diff --git a/packages/rs-drive-abci/src/utils/replay.rs b/packages/rs-drive-abci/src/utils/replay.rs index 47b89a3d96d..5dd47180132 100644 --- a/packages/rs-drive-abci/src/utils/replay.rs +++ b/packages/rs-drive-abci/src/utils/replay.rs @@ -4,6 +4,11 @@ //! checkpoint every ten minutes of chain time is useful on a running node and //! pure waste while catching up, where ten minutes of chain time is a handful of //! blocks and every checkpoint but the last few is deleted within the second. +//! +//! Block age is a proxy for "catching up", not a measurement of it. If the +//! network itself has not produced a block for longer than the threshold, a node +//! that finishes syncing during the halt sees the tip as historical too, and the +//! work gated on this predicate waits for the next block. /// A block older than this is not one the network just produced. Mainnet aims at /// about 2.5 minutes a block, so this leaves several blocks of slack for a node @@ -12,11 +17,17 @@ const HISTORICAL_BLOCK_AGE_MS: u64 = 10 * 60 * 1000; /// True when a block with this timestamp is old enough that the node producing /// it is clearly replaying history rather than following the tip. -pub fn is_historical_block(block_time_ms: u64) -> bool { +pub(crate) fn is_historical_block(block_time_ms: u64) -> bool { let now_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|since_epoch| since_epoch.as_millis() as u64) .unwrap_or(0); + is_historical_block_at(block_time_ms, now_ms) +} + +/// The comparison behind [`is_historical_block`], taking the current time as an +/// argument so the boundary can be tested without racing the real clock. +fn is_historical_block_at(block_time_ms: u64, now_ms: u64) -> bool { now_ms.saturating_sub(block_time_ms) > HISTORICAL_BLOCK_AGE_MS } @@ -24,32 +35,39 @@ pub fn is_historical_block(block_time_ms: u64) -> bool { mod tests { use super::*; - fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system clock is before the unix epoch") - .as_millis() as u64 - } + const NOW_MS: u64 = 2_000_000_000_000; #[test] fn a_block_from_a_year_ago_is_historical() { - assert!(is_historical_block( - now_ms() - 365 * 24 * 60 * 60 * 1000 + assert!(is_historical_block_at( + NOW_MS - 365 * 24 * 60 * 60 * 1000, + NOW_MS )); } #[test] fn a_block_from_a_minute_ago_is_not_historical() { - assert!(!is_historical_block(now_ms() - 60 * 1000)); + assert!(!is_historical_block_at(NOW_MS - 60 * 1000, NOW_MS)); } #[test] fn a_block_at_the_threshold_is_not_yet_historical() { - assert!(!is_historical_block(now_ms() - HISTORICAL_BLOCK_AGE_MS)); + assert!(!is_historical_block_at( + NOW_MS - HISTORICAL_BLOCK_AGE_MS, + NOW_MS + )); + } + + #[test] + fn a_block_one_millisecond_past_the_threshold_is_historical() { + assert!(is_historical_block_at( + NOW_MS - HISTORICAL_BLOCK_AGE_MS - 1, + NOW_MS + )); } #[test] fn a_block_timestamped_in_the_future_is_not_historical() { - assert!(!is_historical_block(now_ms() + 60 * 1000)); + assert!(!is_historical_block_at(NOW_MS + 60 * 1000, NOW_MS)); } } diff --git a/packages/rs-drive-abci/tests/strategy_tests/execution.rs b/packages/rs-drive-abci/tests/strategy_tests/execution.rs index 3860d10c567..b7857af5e71 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/execution.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/execution.rs @@ -908,8 +908,8 @@ pub(crate) async fn start_chain_for_strategy<'a>( current_identity_nonce_counter: Default::default(), current_identity_contract_nonce_counter: Default::default(), current_votes: Default::default(), - start_time_ms: GENESIS_TIME_MS, - current_time_ms: GENESIS_TIME_MS, + start_time_ms: strategy.start_time_ms, + current_time_ms: strategy.start_time_ms, current_identities: Vec::new(), current_addresses_with_balance: Default::default(), }, diff --git a/packages/rs-drive-abci/tests/strategy_tests/strategy.rs b/packages/rs-drive-abci/tests/strategy_tests/strategy.rs index d7900326ce2..9783e79f1f8 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/strategy.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/strategy.rs @@ -1,3 +1,4 @@ +use crate::execution::GENESIS_TIME_MS; use crate::masternodes::MasternodeListItemWithUpdates; use crate::query::QueryStrategy; use dpp::block::block_info::BlockInfo; @@ -354,6 +355,10 @@ pub struct NetworkStrategy { pub independent_process_proposal_verification: bool, pub sign_chain_locks: bool, pub sign_instant_locks: bool, + /// Timestamp of the first block. Defaults to a fixed 2023 instant so runs + /// are reproducible; tests exercising behaviour that keys on how old a + /// block is relative to the wall clock (checkpoints) set it near now. + pub start_time_ms: u64, } impl Default for NetworkStrategy { @@ -378,6 +383,7 @@ impl Default for NetworkStrategy { independent_process_proposal_verification: false, sign_chain_locks: false, sign_instant_locks: false, + start_time_ms: GENESIS_TIME_MS, } } } diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/address_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/address_tests.rs index 5918016b0a6..beef350085c 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/address_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/address_tests.rs @@ -1,7 +1,7 @@ #[cfg(test)] mod tests { - use crate::execution::run_chain_for_strategy; + use crate::execution::{run_chain_for_strategy, GENESIS_TIME_MS}; use crate::strategy::NetworkStrategy; use dapi_grpc::platform::v0::get_addresses_trunk_state_request::{ GetAddressesTrunkStateRequestV0, Version as RequestVersion, @@ -771,6 +771,11 @@ mod tests { query_testing: None, verify_state_transition_results: true, sign_instant_locks: true, + // Checkpoints are only taken for blocks the network produced recently, + // so the chain has to start near the wall clock. Keep the same phase + // within the ten-minute checkpoint interval as the fixed genesis time, + // so the heights that checkpoint are the ones asserted below. + start_time_ms: recent_start_time_with_phase_of(GENESIS_TIME_MS, 600_000), ..Default::default() }; let config = PlatformConfig { @@ -899,6 +904,16 @@ mod tests { ); } + /// The most recent instant no more than `interval_ms` ago that falls on the + /// same offset within the interval as `reference_ms`. + fn recent_start_time_with_phase_of(reference_ms: u64, interval_ms: u64) -> u64 { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock is before the unix epoch") + .as_millis() as u64; + now_ms - (now_ms - reference_ms) % interval_ms + } + #[tokio::test] async fn run_chain_address_transitions_with_checkpoints_stop_and_restart() { drive_abci::logging::init_for_tests(LogLevel::Debug); @@ -953,6 +968,11 @@ mod tests { query_testing: None, verify_state_transition_results: true, sign_instant_locks: true, + // Checkpoints are only taken for blocks the network produced recently, + // so the chain has to start near the wall clock. Keep the fixed genesis + // time's phase within the ten-minute checkpoint interval, so the same + // heights checkpoint as before. + start_time_ms: recent_start_time_with_phase_of(GENESIS_TIME_MS, 600_000), ..Default::default() }; let config = PlatformConfig { @@ -2123,6 +2143,11 @@ mod tests { query_testing: None, verify_state_transition_results: true, sign_instant_locks: true, + // Checkpoints are only taken for blocks the network produced recently, + // so the chain has to start near the wall clock. Keep the fixed genesis + // time's phase within the ten-minute checkpoint interval, so the same + // heights checkpoint as before. + start_time_ms: recent_start_time_with_phase_of(GENESIS_TIME_MS, 600_000), ..Default::default() }; let config = PlatformConfig { @@ -2341,6 +2366,11 @@ mod tests { query_testing: None, verify_state_transition_results: true, sign_instant_locks: true, + // Checkpoints are only taken for blocks the network produced recently, + // so the chain has to start near the wall clock. Keep the fixed genesis + // time's phase within the ten-minute checkpoint interval, so the same + // heights checkpoint as before. + start_time_ms: recent_start_time_with_phase_of(GENESIS_TIME_MS, 600_000), ..Default::default() }; @@ -2691,6 +2721,11 @@ mod tests { query_testing: None, verify_state_transition_results: true, sign_instant_locks: true, + // Checkpoints are only taken for blocks the network produced recently, + // so the chain has to start near the wall clock. Keep the fixed genesis + // time's phase within the ten-minute checkpoint interval, so the same + // heights checkpoint as before. + start_time_ms: recent_start_time_with_phase_of(GENESIS_TIME_MS, 600_000), ..Default::default() }; @@ -3282,6 +3317,11 @@ mod tests { query_testing: None, verify_state_transition_results: true, sign_instant_locks: true, + // Checkpoints are only taken for blocks the network produced recently, + // so the chain has to start near the wall clock. Keep the fixed genesis + // time's phase within the ten-minute checkpoint interval, so the same + // heights checkpoint as before. + start_time_ms: recent_start_time_with_phase_of(GENESIS_TIME_MS, 600_000), ..Default::default() }; @@ -3743,6 +3783,7 @@ mod tests { #[stack_size(8000000)] #[test] async fn run_chain_blast_sync_full_flow() { + let chain_start_time_ms = recent_start_time_with_phase_of(GENESIS_TIME_MS, 600_000); use crate::execution::{continue_chain_for_strategy, GENESIS_TIME_MS}; use crate::strategy::{ ChainExecutionOutcome, ChainExecutionParameters, StrategyRandomness, @@ -3805,6 +3846,11 @@ mod tests { query_testing: None, verify_state_transition_results: true, sign_instant_locks: true, + // Checkpoints are only taken for blocks the network produced recently, + // so the chain has to start near the wall clock. Keep the fixed genesis + // time's phase within the ten-minute checkpoint interval, so the same + // heights checkpoint as before. + start_time_ms: chain_start_time_ms, ..Default::default() }; @@ -4008,7 +4054,7 @@ mod tests { current_identity_nonce_counter: identity_nonce_counter, current_identity_contract_nonce_counter: identity_contract_nonce_counter, current_votes: BTreeMap::default(), - start_time_ms: GENESIS_TIME_MS, + start_time_ms: chain_start_time_ms, current_time_ms: end_time_ms, instant_lock_quorums, current_identities: identities, @@ -4168,7 +4214,7 @@ mod tests { current_identity_nonce_counter: identity_nonce_counter, current_identity_contract_nonce_counter: identity_contract_nonce_counter, current_votes: BTreeMap::default(), - start_time_ms: GENESIS_TIME_MS, + start_time_ms: chain_start_time_ms, current_time_ms: end_time_ms, instant_lock_quorums, current_identities: identities, @@ -4413,7 +4459,7 @@ mod tests { current_identity_nonce_counter: identity_nonce_counter, current_identity_contract_nonce_counter: identity_contract_nonce_counter, current_votes: BTreeMap::default(), - start_time_ms: GENESIS_TIME_MS, + start_time_ms: chain_start_time_ms, current_time_ms: end_time_ms, instant_lock_quorums, current_identities: identities,