feat(cli): add re prune to back up and delete old CM data across datasets

CHANGELOG.md

@@ -1,4 +1,6 @@
 # Unreleased
+- Add `re prune` to delete comments and emails older than a cutoff, backing them up to disk first
+  (optionally scoped to a single mailbox via `--mailbox`)
 - Add `re delete emails` to delete emails by id from a bucket
 - Add `re delete bulk-emails` to bulk-delete emails from a bucket in a given time range
 

cli/Cargo.toml

@@ -22,6 +22,7 @@ name = "tests"
 anyhow = "1.0.66"
 chrono = "0.4.22"
 colored = "2.0.0"
+crc32fast = "1.4.2"
 dirs = "4.0.0"
 env_logger = "0.10.0"
 indicatif = "0.17.1"

cli/src/args.rs

@@ -1,7 +1,7 @@
 use crate::{
     commands::{
         auth::AuthArgs, config::ConfigArgs, create::CreateArgs, delete::DeleteArgs, get::GetArgs,
-        package::PackageArgs, parse::ParseArgs, update::UpdateArgs,
+        package::PackageArgs, parse::ParseArgs, prune::PruneArgs, update::UpdateArgs,
     },
     printer::OutputFormat,
 };
@@ -98,6 +98,13 @@ pub enum Command {
         delete_args: DeleteArgs,
     },
 
+    #[structopt(name = "prune")]
+    /// Back up and delete old data across in-scope datasets
+    Prune {
+        #[structopt(flatten)]
+        prune_args: PruneArgs,
+    },
+
     #[structopt(name = "get")]
     /// Display resources and export comments to the local filesystem.
     Get {

cli/src/commands/get/comments.rs

@@ -718,6 +718,41 @@ impl CommentDownloadOptions {
     }
 }
 
+/// Download all reviewed (annotated) comments for `source` within `dataset` as
+/// JSONL, including their labels. Used by `re prune` to back up annotations.
+pub(crate) fn download_reviewed_comments(
+    client: &Client,
+    source: SourceIdentifier,
+    dataset: DatasetIdentifier,
+    show_progress: bool,
+    writer: impl Write,
+) -> Result<()> {
+    download_comments(
+        client,
+        source,
+        writer,
+        CommentDownloadOptions {
+            dataset_identifier: Some(dataset),
+            include_predictions: false,
+            model_version: None,
+            reviewed_only: true,
+            timerange: CommentsIterTimerange {
+                from: None,
+                to: None,
+            },
+            show_progress,
+            label_attribute_filter: None,
+            attachment_property_types_filter: None,
+            user_properties_filter: None,
+            messages_filter: None,
+            attachments_dir: None,
+            only_with_attachments_filter: None,
+            shuffle: false,
+            stop_after: None,
+        },
+    )
+}
+
 fn download_comments(
     client: &Client,
     source_identifier: SourceIdentifier,

cli/src/commands/get/mod.rs

@@ -1,6 +1,6 @@
 mod audit_events;
 mod buckets;
-mod comments;
+pub(crate) mod comments;
 mod custom_label_trend_report;
 mod datasets;
 mod emails;

cli/src/commands/mod.rs

@@ -17,6 +17,7 @@ pub mod delete;
 pub mod get;
 pub mod package;
 pub mod parse;
+pub mod prune;
 pub mod update;
 
 pub fn ensure_uip_user_consents_to_ai_unit_charge(base_url: &Url) -> Result<()> {