feat(prune): experiment with time-based retention and gradual catch-up

crates/cli/commands/src/download/mod.rs

@@ -113,7 +113,7 @@ use reth_db::{init_db, Database};
 use reth_db_api::transaction::DbTx;
 use reth_fs_util as fs;
 use reth_node_core::args::DefaultPruningValues;
-use reth_prune_types::{PruneMode, PruneModes};
+use reth_prune_types::{PruneMode, PruneModes, PruneSegment};
 use source::{
     discover_manifest_url, fetch_manifest_from_source, fetch_snapshot_api_entries,
     print_snapshot_listing, resolve_manifest_base_url,
@@ -886,11 +886,17 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCo
             SnapshotComponentType::State | SnapshotComponentType::Headers => {
                 return ComponentSelection::All
             }
-            SnapshotComponentType::Transactions => prune_modes.bodies_history,
+            SnapshotComponentType::Transactions => {
+                prune_modes.write_prune_mode(PruneSegment::Bodies)
+            }
             SnapshotComponentType::TransactionSenders => prune_modes.sender_recovery,
-            SnapshotComponentType::Receipts => prune_modes.receipts,
-            SnapshotComponentType::AccountChangesets => prune_modes.account_history,
-            SnapshotComponentType::StorageChangesets => prune_modes.storage_history,
+            SnapshotComponentType::Receipts => prune_modes.write_prune_mode(PruneSegment::Receipts),
+            SnapshotComponentType::AccountChangesets => {
+                prune_modes.write_prune_mode(PruneSegment::AccountHistory)
+            }
+            SnapshotComponentType::StorageChangesets => {
+                prune_modes.write_prune_mode(PruneSegment::StorageHistory)
+            }
             SnapshotComponentType::RocksdbIndices => return ComponentSelection::None,
         };
 

crates/cli/commands/src/stage/run.rs

@@ -33,6 +33,7 @@ use reth_provider::{
     providers::BlockchainProvider, ChainSpecProvider, DBProvider, DatabaseProviderFactory,
     StageCheckpointReader, StageCheckpointWriter,
 };
+use reth_prune_types::PruneSegment;
 use reth_stages::{
     stages::{
         AccountHashingStage, BodyStage, ExecutionStage, HeaderStage, IndexAccountHistoryStage,
@@ -314,15 +315,15 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
                     Box::new(IndexAccountHistoryStage::new(
                         config.stages.index_account_history,
                         etl_config,
-                        prune_modes.account_history,
+                        prune_modes.write_prune_mode(PruneSegment::AccountHistory),
                     )),
                     None,
                 ),
                 StageEnum::StorageHistory => (
                     Box::new(IndexStorageHistoryStage::new(
                         config.stages.index_storage_history,
                         etl_config,
-                        prune_modes.storage_history,
+                        prune_modes.write_prune_mode(PruneSegment::StorageHistory),
                     )),
                     None,
                 ),

crates/config/src/config.rs

@@ -587,6 +587,7 @@ impl PruneConfig {
             block_interval,
             segments:
                 PruneModes {
+                    time_based,
                     sender_recovery,
                     transaction_lookup,
                     receipts,
@@ -609,6 +610,7 @@ impl PruneConfig {
         }
 
         // Merge the various segment prune modes
+        self.segments.time_based = self.segments.time_based.or(time_based);
         self.segments.sender_recovery = self.segments.sender_recovery.or(sender_recovery);
         self.segments.transaction_lookup = self.segments.transaction_lookup.or(transaction_lookup);
         self.segments.receipts = self.segments.receipts.or(receipts);
@@ -1115,12 +1117,30 @@ receipts = { distance = 16384 }
         let _conf: Config = toml::from_str(s).unwrap();
     }
 
+    #[test]
+    fn time_based_pruning_config_roundtrip() {
+        let config: Config = toml::from_str(
+            r#"
+            [prune.segments]
+            account_history = { distance = 13392000 }
+            [prune.segments.time_based]
+            retention_seconds = 2678400
+            catch_up_percent = 130
+        "#,
+        )
+        .unwrap();
+        assert_eq!(config.prune.segments.time_based, Some(Default::default()));
+        let restored: Config = toml::from_str(&toml::to_string(&config).unwrap()).unwrap();
+        assert_eq!(restored.prune, config.prune);
+    }
+
     #[test]
     fn test_prune_config_merge() {
         let mut config1 = PruneConfig {
             block_interval: 5,
             minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
             segments: PruneModes {
+                time_based: None,
                 sender_recovery: Some(PruneMode::Full),
                 transaction_lookup: None,
                 receipts: Some(PruneMode::Distance(1000)),
@@ -1138,6 +1158,7 @@ receipts = { distance = 16384 }
             block_interval: 10,
             minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
             segments: PruneModes {
+                time_based: Some(Default::default()),
                 sender_recovery: Some(PruneMode::Distance(500)),
                 transaction_lookup: Some(PruneMode::Full),
                 receipts: Some(PruneMode::Full),
@@ -1157,6 +1178,7 @@ receipts = { distance = 16384 }
         // Check that the configuration has been merged. Any configuration present in config1
         // should not be overwritten by config2
         assert_eq!(config1.block_interval, 10);
+        assert_eq!(config1.segments.time_based, Some(Default::default()));
         assert_eq!(config1.segments.sender_recovery, Some(PruneMode::Full));
         assert_eq!(config1.segments.transaction_lookup, Some(PruneMode::Full));
         assert_eq!(config1.segments.receipts, Some(PruneMode::Distance(1000)));

crates/node/core/src/args/pruning.rs

@@ -68,6 +68,7 @@ impl Default for DefaultPruningValues {
     fn default() -> Self {
         Self {
             full_prune_modes: PruneModes {
+                time_based: None,
                 sender_recovery: Some(PruneMode::Full),
                 transaction_lookup: None,
                 receipts: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
@@ -79,6 +80,7 @@ impl Default for DefaultPruningValues {
             },
             full_bodies_history_use_pre_merge: true,
             minimal_prune_modes: PruneModes {
+                time_based: None,
                 sender_recovery: Some(PruneMode::Full),
                 transaction_lookup: Some(PruneMode::Full),
                 receipts: Some(PruneMode::Distance(MINIMUM_DISTANCE)),

crates/prune/prune/src/builder.rs

@@ -99,6 +99,7 @@ impl PrunerBuilder {
                 Primitives = <PF::ProviderRW as NodePrimitivesProvider>::Primitives,
             >,
     {
+        let time_based = self.segments.time_based;
         let segments =
             SegmentSet::from_components(provider_factory.static_file_provider(), self.segments);
 
@@ -109,7 +110,8 @@ impl PrunerBuilder {
             self.delete_limit,
             self.timeout,
             self.finished_exex_height,
-        );
+        )
+        .with_time_based(time_based);
         if let Some(distance) = self.minimum_pruning_distance {
             pruner = pruner.with_minimum_pruning_distance(distance);
         }
@@ -135,6 +137,7 @@ impl PrunerBuilder {
             + StorageChangeSetReader
             + RocksDBProviderFactory,
     {
+        let time_based = self.segments.time_based;
         let segments = SegmentSet::<Provider>::from_components(static_file_provider, self.segments);
 
         let mut pruner = Pruner::new(
@@ -143,7 +146,8 @@ impl PrunerBuilder {
             self.delete_limit,
             self.timeout,
             self.finished_exex_height,
-        );
+        )
+        .with_time_based(time_based);
         if let Some(distance) = self.minimum_pruning_distance {
             pruner = pruner.with_minimum_pruning_distance(distance);
         }

crates/prune/prune/src/lib.rs

@@ -16,6 +16,7 @@ mod limiter;
 mod metrics;
 mod pruner;
 pub mod segments;
+mod time_based;
 
 use crate::metrics::Metrics;
 pub use builder::PrunerBuilder;