diff --git a/crates/cli/commands/src/download/config_gen.rs b/crates/cli/commands/src/download/config_gen.rs index 2eda1df907b..f81922f4f7d 100644 --- a/crates/cli/commands/src/download/config_gen.rs +++ b/crates/cli/commands/src/download/config_gen.rs @@ -43,6 +43,7 @@ pub fn write_config(config: &Config, data_dir: &Path) -> eyre::Result { pub(crate) fn write_prune_checkpoints_tx( tx: &Tx, config: &Config, + selections: &BTreeMap, snapshot_block: u64, ) -> eyre::Result<()> where @@ -72,8 +73,36 @@ where tx.get::(snapshot_block)?.map(|indices| indices.last_tx_num()); for (segment, prune_mode) in &checkpoints { + let (block_number, tx_number) = match segment { + // Record the history actually selected, not the future pruning policy, which may + // retain a minimum distance even when no changesets were downloaded. + PruneSegment::AccountHistory | PruneSegment::StorageHistory => { + let component = if *segment == PruneSegment::AccountHistory { + SnapshotComponentType::AccountChangesets + } else { + SnapshotComponentType::StorageChangesets + }; + let target = match selections.get(&component) { + Some(ComponentSelection::All) => None, + Some(ComponentSelection::Distance(distance)) => { + snapshot_block.checked_sub(*distance) + } + Some(ComponentSelection::Since(block)) => { + block.checked_sub(1).map(|target| target.min(snapshot_block)) + } + Some(ComponentSelection::None) | None => Some(snapshot_block), + }; + let Some(target) = target else { continue }; + // Downloading a wider range cannot restore history already pruned by the source. + let existing = tx + .get::(*segment)? + .and_then(|checkpoint| checkpoint.block_number); + (existing.map_or(target, |existing| existing.max(target)), None) + } + _ => (snapshot_block, tx_number), + }; let checkpoint = PruneCheckpoint { - block_number: Some(snapshot_block), + block_number: Some(block_number), tx_number, prune_mode: *prune_mode, }; @@ -82,7 +111,7 @@ where info!(target: "reth::cli", segment = %segment, - block = snapshot_block, + block = block_number, tx = ?tx_number, mode = ?prune_mode, "Set prune checkpoint" @@ -328,7 +357,7 @@ mod tests { { let tx = db.tx_mut().unwrap(); - write_prune_checkpoints_tx(&tx, &config, snapshot_block).unwrap(); + write_prune_checkpoints_tx(&tx, &config, &selections, snapshot_block).unwrap(); tx.commit().unwrap(); } @@ -352,6 +381,40 @@ mod tests { } } + /// Writes prune checkpoints for state history retained over `distance` blocks. + fn history_checkpoint(snapshot_block: u64, distance: u64) -> Option { + let dir = tempfile::tempdir().unwrap(); + let db = reth_db::init_db(dir.path(), reth_db::mdbx::DatabaseArguments::default()).unwrap(); + let selections = BTreeMap::from([( + SnapshotComponentType::AccountChangesets, + ComponentSelection::Distance(distance), + )]); + let config = config_for_selections( + &selections, + &empty_manifest(), + None, + None::<&reth_chainspec::ChainSpec>, + ); + + let tx = db.tx_mut().unwrap(); + write_prune_checkpoints_tx(&tx, &config, &selections, snapshot_block).unwrap(); + tx.commit().unwrap(); + + db.tx().unwrap().get::(PruneSegment::AccountHistory).unwrap() + } + + #[test] + fn write_prune_checkpoints_sets_history_to_retained_window_start() { + let checkpoint = history_checkpoint(21_000_000, 500_000).unwrap(); + assert_eq!(checkpoint.block_number, Some(20_500_000)); + assert_eq!(checkpoint.prune_mode, PruneMode::Distance(500_000)); + } + + #[test] + fn write_prune_checkpoints_skips_history_shorter_than_distance() { + assert_eq!(history_checkpoint(100_000, 500_000), None); + } + #[test] fn write_prune_checkpoints_archive_no_checkpoints() { let dir = tempfile::tempdir().unwrap(); @@ -371,7 +434,7 @@ mod tests { { let tx = db.tx_mut().unwrap(); - write_prune_checkpoints_tx(&tx, &config, 21_000_000).unwrap(); + write_prune_checkpoints_tx(&tx, &config, &selections, 21_000_000).unwrap(); tx.commit().unwrap(); } @@ -658,4 +721,115 @@ mod tests { .expect("sender prune checkpoint should exist"); assert_eq!(sender_prune_checkpoint.block_number, Some(tip_checkpoint.block_number)); } + + #[test] + fn history_checkpoints_do_not_advertise_undownloaded_changesets() { + let snapshot_block = 21_000_000; + for (selection, expected) in [ + (None, Some(snapshot_block)), + (Some(ComponentSelection::None), Some(snapshot_block)), + (Some(ComponentSelection::Distance(1)), Some(snapshot_block - 1)), + (Some(ComponentSelection::Since(snapshot_block)), Some(snapshot_block - 1)), + (Some(ComponentSelection::Since(0)), None), + (Some(ComponentSelection::All), None), + ] { + let dir = tempfile::tempdir().unwrap(); + let db = + reth_db::init_db(dir.path(), reth_db::mdbx::DatabaseArguments::default()).unwrap(); + let mut selections = BTreeMap::from([ + (SnapshotComponentType::State, ComponentSelection::All), + (SnapshotComponentType::Headers, ComponentSelection::All), + (SnapshotComponentType::RocksdbIndices, ComponentSelection::All), + ]); + if let Some(selection) = selection { + selections.insert(SnapshotComponentType::AccountChangesets, selection); + selections.insert(SnapshotComponentType::StorageChangesets, selection); + } + let config = config_for_selections( + &selections, + &empty_manifest(), + None, + None::<&reth_chainspec::ChainSpec>, + ); + assert!(!super::super::should_reset_index_stage_checkpoints(&selections)); + let tx = db.tx_mut().unwrap(); + write_prune_checkpoints_tx(&tx, &config, &selections, snapshot_block).unwrap(); + let mut checkpoints = Vec::new(); + for segment in [PruneSegment::AccountHistory, PruneSegment::StorageHistory] { + let checkpoint = tx.get::(segment).unwrap(); + assert_eq!(checkpoint.and_then(|checkpoint| checkpoint.block_number), expected); + if let Some(checkpoint) = checkpoint { + checkpoints.push((segment, checkpoint)); + } + } + if let Some(boundary) = expected { + assert!(config + .prune + .segments + .ensure_unwind_target_unpruned(snapshot_block, boundary - 1, &checkpoints,) + .is_err()); + assert!(config + .prune + .segments + .ensure_unwind_target_unpruned(snapshot_block, boundary, &checkpoints,) + .is_ok()); + } + } + } + + #[test] + fn history_checkpoints_preserve_source_pruning_boundary() { + let snapshot_block = 21_000_000; + let selected_boundary = 20_500_000; + for selection in [ + ComponentSelection::Distance(snapshot_block - selected_boundary), + ComponentSelection::Since(selected_boundary + 1), + ] { + for existing_boundary in [None, Some(20_000_000), Some(20_750_000)] { + let dir = tempfile::tempdir().unwrap(); + let db = reth_db::init_db(dir.path(), reth_db::mdbx::DatabaseArguments::default()) + .unwrap(); + let selections = BTreeMap::from([ + (SnapshotComponentType::AccountChangesets, selection), + (SnapshotComponentType::StorageChangesets, selection), + (SnapshotComponentType::RocksdbIndices, ComponentSelection::All), + ]); + let config = config_for_selections( + &selections, + &empty_manifest(), + None, + None::<&reth_chainspec::ChainSpec>, + ); + let tx = db.tx_mut().unwrap(); + for segment in [PruneSegment::AccountHistory, PruneSegment::StorageHistory] { + tx.put::( + segment, + PruneCheckpoint { + block_number: existing_boundary, + tx_number: Some(100), + prune_mode: PruneMode::Full, + }, + ) + .unwrap(); + } + + write_prune_checkpoints_tx(&tx, &config, &selections, snapshot_block).unwrap(); + + let expected = existing_boundary.unwrap_or_default().max(selected_boundary); + for (segment, mode) in [ + (PruneSegment::AccountHistory, config.prune.segments.account_history), + (PruneSegment::StorageHistory, config.prune.segments.storage_history), + ] { + assert_eq!( + tx.get::(segment).unwrap(), + Some(PruneCheckpoint { + block_number: Some(expected), + tx_number: None, + prune_mode: mode.unwrap(), + }), + ); + } + } + } + } } diff --git a/crates/cli/commands/src/download/mod.rs b/crates/cli/commands/src/download/mod.rs index 22bcb5ce5f6..588269d5597 100644 --- a/crates/cli/commands/src/download/mod.rs +++ b/crates/cli/commands/src/download/mod.rs @@ -179,10 +179,14 @@ pub struct DownloadDefaults { pub snapshot_api_url: Cow<'static, str>, /// Optional custom long help text that overrides the generated help pub long_help: Option, + /// Whether the full preset also downloads the snapshot's RocksDB indices instead of + /// rebuilding them. + pub full_preset_includes_rocksdb_indices: bool, } impl DownloadDefaults { /// Initialize the global download defaults with this configuration + #[expect(clippy::result_large_err, reason = "returns the rejected defaults to the caller")] pub fn try_init(self) -> Result<(), Self> { DOWNLOAD_DEFAULTS.set(self) } @@ -203,6 +207,7 @@ impl DownloadDefaults { default_chain_aware_base_url: None, snapshot_api_url: Cow::Borrowed(RETH_SNAPSHOTS_API_URL), long_help: None, + full_preset_includes_rocksdb_indices: false, } } @@ -312,6 +317,12 @@ impl DownloadDefaults { self } + /// Builder: Make the full preset also download the snapshot's RocksDB indices. + pub const fn with_full_preset_rocksdb_indices(mut self) -> Self { + self.full_preset_includes_rocksdb_indices = true; + self + } + /// Builder: Set custom long help text, overriding the generated help pub fn with_long_help(mut self, help: impl Into) -> Self { self.long_help = Some(help.into()); @@ -419,7 +430,8 @@ pub struct DownloadCommand { /// Skip optional RocksDB indices even when archive components are selected. /// - /// This affects `--archive`/`--all` and TUI archive preset (`a`). + /// This affects `--archive`/`--all`, TUI archive preset (`a`), and the full preset when it + /// includes RocksDB indices. #[arg(long, conflicts_with_all = ["url", "with_rocksdb"])] without_rocksdb: bool, @@ -612,8 +624,17 @@ impl> DownloadCo let manifest = self.load_manifest(chain_id).await?; let ResolvedComponents { mut selections, preset } = self.resolve_components(&manifest)?; - if matches!(preset, Some(SelectionPreset::Archive)) { - inject_archive_only_components(&mut selections, &manifest, !self.without_rocksdb); + match preset { + Some(SelectionPreset::Archive) => { + inject_archive_only_components(&mut selections, &manifest, !self.without_rocksdb) + } + Some(SelectionPreset::Full) + if DownloadDefaults::get_global().full_preset_includes_rocksdb_indices && + !self.without_rocksdb => + { + inject_rocksdb_indices(&mut selections, &manifest) + } + _ => {} } let planned = collect_planned_archives(&manifest, &selections)?; @@ -667,7 +688,7 @@ impl> DownloadCo let tx = db.tx_mut()?; if should_write_prune { - config_gen::write_prune_checkpoints_tx(&tx, &config, manifest.block)?; + config_gen::write_prune_checkpoints_tx(&tx, &config, selections, manifest.block)?; } if should_reset_indices { @@ -1023,6 +1044,16 @@ fn inject_archive_only_components( } } +/// Selects the manifest's RocksDB indices, if it publishes them. +fn inject_rocksdb_indices( + selections: &mut BTreeMap, + manifest: &SnapshotManifest, +) { + if manifest.component(SnapshotComponentType::RocksdbIndices).is_some() { + selections.insert(SnapshotComponentType::RocksdbIndices, ComponentSelection::All); + } +} + /// Returns `true` when RocksDB-backed index stages should be reset after download. fn should_reset_index_stage_checkpoints( selections: &BTreeMap, @@ -1539,6 +1570,35 @@ mod tests { assert_eq!(selections.get(&SnapshotComponentType::RocksdbIndices), None); } + #[test] + fn inject_rocksdb_indices_keeps_index_stages() { + let manifest = manifest_with_archive_only_components(); + let mut selections = BTreeMap::new(); + selections + .insert(SnapshotComponentType::AccountChangesets, ComponentSelection::Distance(10)); + + inject_rocksdb_indices(&mut selections, &manifest); + + assert_eq!( + selections.get(&SnapshotComponentType::RocksdbIndices), + Some(&ComponentSelection::All) + ); + assert_eq!(selections.get(&SnapshotComponentType::TransactionSenders), None); + assert!(!should_reset_index_stage_checkpoints(&selections)); + } + + #[test] + fn inject_rocksdb_indices_skips_manifest_without_indices() { + let mut manifest = manifest_with_archive_only_components(); + manifest.components.clear(); + let mut selections = BTreeMap::new(); + + inject_rocksdb_indices(&mut selections, &manifest); + + assert!(selections.is_empty()); + assert!(should_reset_index_stage_checkpoints(&selections)); + } + #[test] fn should_reset_index_stage_checkpoints_without_rocksdb_indices() { let mut selections = BTreeMap::new(); diff --git a/docs/vocs/docs/pages/cli/reth/download.mdx b/docs/vocs/docs/pages/cli/reth/download.mdx index 04a2a3220d8..523dc2c6d66 100644 --- a/docs/vocs/docs/pages/cli/reth/download.mdx +++ b/docs/vocs/docs/pages/cli/reth/download.mdx @@ -200,7 +200,7 @@ Storage: --without-rocksdb Skip optional RocksDB indices even when archive components are selected. - This affects `--archive`/`--all` and TUI archive preset (`a`). + This affects `--archive`/`--all`, TUI archive preset (`a`), and the full preset when it includes RocksDB indices. -y, --non-interactive Skip interactive component selection. Downloads the minimal set (state + headers + transactions + changesets) unless explicit --with-* flags narrow it