diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 97a75d6..650fd36 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -65,6 +65,10 @@ jobs:
run: |
cargo clippy --all-targets -- -D warnings
cargo build --release
+ # The UI's pure helpers (formatting, breadcrumbs, treemap layout, diff
+ # indexing) are plain JS with Node unit tests; no npm install needed.
+ - name: UI unit tests
+ run: node --test gui/ui/lib.test.js
deny:
name: cargo-deny (supply chain)
diff --git a/Cargo.lock b/Cargo.lock
index 4a3227f..d3ffcc5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -362,6 +362,7 @@ version = "0.6.2"
dependencies = [
"clap",
"diskghost-core",
+ "serde",
"serde_json",
]
@@ -377,6 +378,7 @@ dependencies = [
"rayon",
"same-file",
"serde",
+ "serde_json",
"trash",
]
diff --git a/README.md b/README.md
index bbe7f27..cfb567f 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,15 @@ Find where your space went, hunt down byte-identical duplicates, and reclaim the
target), refuses a drive root, dry-run by default, reports how long it took.
- **Scan options** — exclude globs (`node_modules`, `*.tmp`), max depth,
follow-symlinks.
-- **Live progress + cancel** — long scans report files/bytes and can be stopped.
+- **Live progress + cancel** — long scans report files/bytes and can be stopped;
+ a cancelled scan is clearly labelled as a partial result.
+- **One walk, whole tree** — the scan builds an in-memory tree, so the GUI
+ drills into any sub-folder and back up instantly, without touching the disk
+ again. Deleting from the GUI updates the tree in place.
+- **What grew since last time** — save a scan as a snapshot and compare a
+ later one against it: which folders grew or shrank, which are new, which are
+ gone. Every directory is recorded (not just the biggest), so the comparison
+ works at any depth. The GUI keeps a history per scanned folder automatically.
- **Three faces, one Rust core**:
- a **CLI** with `--json` output for scripts and agents (headless);
- a **modern desktop GUI** (Tauri) with a treemap, folder picker, drag & drop
@@ -62,15 +70,28 @@ diskghost rm "D:\old_project\target" # shows how many files/dirs/byte
diskghost rm "D:\old_project\target" --apply # permanently delete (files unlinked in parallel)
diskghost rm "D:\old_project\target" --trash --apply # send to the OS recycle bin instead
# never follows symlinks (can't escape the target); refuses a drive root
+
+# What grew since last time: save a snapshot, compare against it later
+diskghost scan "C:\Users\me" --save before.json
+diskghost scan "C:\Users\me" --since before.json # scan + what grew/shrank/appeared/vanished
+diskghost scan "C:\Users\me" --since before.json --save after.json
+diskghost diff before.json after.json # compare two saved snapshots
+diskghost diff before.json after.json --at "C:\Users\me\Downloads" # ...at a sub-folder
```
+Snapshots are plain JSON, so a scheduled task (cron, Task Scheduler) that runs
+`diskghost scan
--since last.json --save last.json --json` gives you a
+daily "what grew" report with nothing else to set up.
+
### Headless / agent mode
Add `--json` to any command for machine-readable output:
```bash
-diskghost scan . --json # total_size, children, top_files, root_files, skipped …
+diskghost scan . --json # total_size, children, top_files, root_files, skipped, cancelled …
diskghost dupes . --min-mb 10 --json # duplicate groups with reclaimable bytes
+diskghost scan . --since old.json --json # the report plus a "diff" object (grown/shrunk/added/removed)
+diskghost diff old.json new.json --json # just the diff
```
Perfect for wiring into an automation/agent that decides what to clean.
@@ -78,9 +99,13 @@ Perfect for wiring into an automation/agent that decides what to clean.
## Desktop app (GUI)
A modern Tauri UI lives in `gui/` — **Browse** or drag a folder onto the window,
-hit **Scan size** for a treemap + biggest folders/files (click a folder to drill
-in), or **Find duplicates** to sort/filter groups and reclaim their space with one
-click. Scans show live progress and can be cancelled. From `gui/src-tauri`:
+hit **Scan size** for a treemap + biggest folders/files (click a tile or a
+breadcrumb to drill in and back up — served from the in-memory tree, so it is
+instant; **Rescan** walks the disk again), or **Find duplicates** to sort/filter
+groups and reclaim their space with one click. Scans show live progress and can
+be cancelled. Every complete scan is kept as a snapshot; pick an earlier one
+under **Since** and **Show changes** colours the treemap and folder bars by what
+grew or shrank and lists new and removed folders. From `gui/src-tauri`:
```bash
cargo tauri dev # run the app
@@ -90,7 +115,8 @@ cargo tauri build # bundle an installer
## Why it's fast — and correct
- Parallel directory walk across all cores; a **single pass** counts files *and*
- directories.
+ directories, and builds the whole tree in memory so navigating it costs no
+ further I/O.
- Duplicate detection never hashes more than it must: size buckets first, hard
links collapsed, a cheap first-block pre-hash, then BLAKE3 — streamed, so large
files never load into RAM.
@@ -102,9 +128,9 @@ cargo tauri build # bundle an installer
```
Diskghost/
├── crates/
-│ ├── diskghost-core/ the engine: scan + duplicate detection + reclaim (a library)
+│ ├── diskghost-core/ the engine: scan tree, duplicates, reclaim, snapshots + diff (a library)
│ └── diskghost-cli/ the `diskghost` command
-└── gui/ Tauri desktop app (modern dark UI)
+└── gui/ Tauri desktop app (modern dark UI; ui/lib.js has Node unit tests)
```
## Roadmap
@@ -115,9 +141,10 @@ Diskghost/
- [x] Modern GUI: treemap, folder picker, drag & drop, one-click reclaim
- [x] Delete files/folders (CLI `rm` + GUI) — permanent or to the OS trash, parallel unlink
- [x] Drive free/total space + scan & delete timings (CLI + GUI)
-- [x] CI (fmt/clippy/test/bench on 3 OS) + signed release binaries + GUI installers
-- [ ] Interactive treemap drill-up / breadcrumbs
-- [ ] Scheduled scans & "what grew since last time"
+- [x] CI (fmt/clippy/test/bench on 3 OS) + checksummed, provenance-attested release binaries + GUI installers
+- [x] Interactive treemap drill-up / breadcrumbs, served from an in-memory tree
+- [x] Snapshots & "what grew since last time" (CLI `--save`/`--since`/`diff`, GUI history)
+- [ ] Code-signed GUI installers
## Development
@@ -125,6 +152,7 @@ Diskghost/
cargo test --workspace # unit + integration tests
cargo clippy --workspace --all-targets -- -D warnings # lints (CI gate)
cargo bench -p diskghost-core # criterion benchmarks
+node --test gui/ui/lib.test.js # GUI helper unit tests
```
## License
diff --git a/crates/diskghost-cli/Cargo.toml b/crates/diskghost-cli/Cargo.toml
index bb2da37..767d763 100644
--- a/crates/diskghost-cli/Cargo.toml
+++ b/crates/diskghost-cli/Cargo.toml
@@ -15,4 +15,5 @@ path = "src/main.rs"
[dependencies]
diskghost-core = { path = "../diskghost-core" }
clap = { version = "4", features = ["derive"] }
+serde = { version = "1", features = ["derive"] }
serde_json = "1"
diff --git a/crates/diskghost-cli/src/main.rs b/crates/diskghost-cli/src/main.rs
index 64e96b1..49146b0 100644
--- a/crates/diskghost-cli/src/main.rs
+++ b/crates/diskghost-cli/src/main.rs
@@ -11,10 +11,11 @@ use std::sync::Arc;
use clap::{Args, Parser, Subcommand, ValueEnum};
use diskghost_core::{
- find_duplicates_with_progress, human_size, reclaim, remove_path, scan_with_progress,
- validate_globs, DupGroup, Options, Progress, ReclaimAction, RemoveMode, RemoveReport,
- ScanReport,
+ diff, find_duplicates_with_progress, format_timestamp, human_delta, human_size, reclaim,
+ remove_path, validate_globs, DiffReport, DupGroup, Options, Progress, ReclaimAction,
+ RemoveMode, RemoveReport, ScanReport, ScanTree, Snapshot,
};
+use serde::Serialize;
/// Run `job` with a `Progress`, printing a live line to stderr while it works
/// (only when stderr is a terminal, so JSON/pipe output stays clean).
@@ -80,6 +81,26 @@ enum Command {
top: usize,
#[command(flatten)]
walk: WalkArgs,
+ /// Save the scan as a snapshot (JSON) for a later `--since` or `diff`.
+ #[arg(long, value_name = "FILE")]
+ save: Option,
+ /// Compare against a snapshot saved earlier: what grew, shrank,
+ /// appeared or vanished since then.
+ #[arg(long, value_name = "FILE")]
+ since: Option,
+ },
+ /// Compare two snapshots saved with `scan --save`.
+ Diff {
+ /// The older snapshot.
+ old: PathBuf,
+ /// The newer snapshot.
+ new: PathBuf,
+ /// Compare at this sub-folder instead of the scanned root.
+ #[arg(long, value_name = "DIR")]
+ at: Option,
+ /// How many entries to show per list.
+ #[arg(long, default_value_t = 20)]
+ top: usize,
},
/// Find duplicate (byte-identical) files; optionally reclaim their space.
Dupes {
@@ -160,22 +181,91 @@ fn main() -> ExitCode {
let cli = Cli::parse();
match cli.command {
- Command::Scan { path, top, walk } => {
+ Command::Scan {
+ path,
+ top,
+ walk,
+ save,
+ since,
+ } => {
if !path.is_dir() {
eprintln!("error: not a directory: {}", path.display());
return ExitCode::FAILURE;
}
warn_invalid_globs(&walk.exclude);
let opts = walk.to_options();
+ // Load the old snapshot before scanning, so a bad file fails fast.
+ let old = match since.as_deref().map(Snapshot::load) {
+ Some(Ok(s)) => Some(s),
+ Some(Err(e)) => {
+ eprintln!("error: cannot read snapshot: {e}");
+ return ExitCode::FAILURE;
+ }
+ None => None,
+ };
let started = std::time::Instant::now();
- let report = with_progress(|p| scan_with_progress(&path, top, &opts, p));
+ let tree = with_progress(|p| ScanTree::build(&path, &opts, p));
let elapsed = started.elapsed();
+ let report = tree.root_report(top);
+ let snapshot = (old.is_some() || save.is_some()).then(|| Snapshot::capture(&tree, top));
+ let changes = match (&old, &snapshot) {
+ (Some(old), Some(new)) => match diff(old, new, &report.root) {
+ Ok(mut d) => {
+ d.truncate(top);
+ Some(d)
+ }
+ Err(e) => {
+ eprintln!("error: {e}");
+ return ExitCode::FAILURE;
+ }
+ },
+ _ => None,
+ };
if cli.json {
- println!("{}", serde_json::to_string_pretty(&report).unwrap());
+ let out = ScanOutput {
+ report: &report,
+ diff: changes.as_ref(),
+ };
+ println!("{}", serde_json::to_string_pretty(&out).unwrap());
} else {
print_scan(&report);
+ if let Some(d) = &changes {
+ println!();
+ print_diff(d);
+ }
println!("\nScanned in {}", fmt_duration(elapsed));
}
+ if let (Some(file), Some(snap)) = (&save, &snapshot) {
+ if let Err(e) = snap.save(file) {
+ eprintln!("error: cannot save snapshot: {e}");
+ return ExitCode::FAILURE;
+ }
+ // Keep stdout clean for --json consumers.
+ eprintln!("snapshot saved to {}", file.display());
+ }
+ }
+ Command::Diff { old, new, at, top } => {
+ let (old, new) = match (Snapshot::load(&old), Snapshot::load(&new)) {
+ (Ok(o), Ok(n)) => (o, n),
+ (Err(e), _) | (_, Err(e)) => {
+ eprintln!("error: cannot read snapshot: {e}");
+ return ExitCode::FAILURE;
+ }
+ };
+ let at = at.unwrap_or_else(|| new.root.clone());
+ let mut report = match diff(&old, &new, &at) {
+ Ok(r) => r,
+ Err(e) => {
+ eprintln!("error: {e}");
+ return ExitCode::FAILURE;
+ }
+ };
+ report.truncate(top);
+ if cli.json {
+ println!("{}", serde_json::to_string_pretty(&report).unwrap());
+ } else {
+ print_diff(&report);
+ }
}
Command::Dupes {
path,
@@ -241,6 +331,72 @@ fn main() -> ExitCode {
ExitCode::SUCCESS
}
+/// `scan --json` output: the report's fields at the top level, plus `diff`
+/// when `--since` was given.
+#[derive(Serialize)]
+struct ScanOutput<'a> {
+ #[serde(flatten)]
+ report: &'a ScanReport,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ diff: Option<&'a DiffReport>,
+}
+
+fn print_diff(d: &DiffReport) {
+ println!(
+ "Changes in {} since {} (now {})",
+ d.root.display(),
+ format_timestamp(d.old_taken_at),
+ format_timestamp(d.new_taken_at)
+ );
+ println!(
+ " total: {} ({} -> {}), {:+} file(s)",
+ human_delta(d.delta),
+ human_size(d.old_size),
+ human_size(d.new_size),
+ d.files_delta
+ );
+ if d.is_unchanged() {
+ println!(" nothing changed");
+ return;
+ }
+ if !d.grown.is_empty() {
+ println!("\nGrew:");
+ for g in &d.grown {
+ println!(
+ " {:>10} {} ({} -> {})",
+ human_delta(g.delta),
+ g.path.display(),
+ human_size(g.old_size),
+ human_size(g.new_size)
+ );
+ }
+ }
+ if !d.shrunk.is_empty() {
+ println!("\nShrank:");
+ for s in &d.shrunk {
+ println!(
+ " {:>10} {} ({} -> {})",
+ human_delta(s.delta),
+ s.path.display(),
+ human_size(s.old_size),
+ human_size(s.new_size)
+ );
+ }
+ }
+ if !d.added.is_empty() {
+ println!("\nNew folders:");
+ for a in &d.added {
+ println!(" {:>10} {}", human_size(a.size), a.path.display());
+ }
+ }
+ if !d.removed.is_empty() {
+ println!("\nRemoved folders:");
+ for r in &d.removed {
+ println!(" {:>10} {}", human_size(r.size), r.path.display());
+ }
+ }
+}
+
fn print_remove(r: &RemoveReport, trash: bool) {
let dest = if trash { "trash" } else { "delete" };
let what = format!(
@@ -299,6 +455,9 @@ fn print_scan(r: &ScanReport) {
if r.skipped > 0 {
println!(" skipped: {} unreadable entries (permissions?)", r.skipped);
}
+ if r.cancelled {
+ println!(" CANCELLED: partial result, the numbers cover only what was scanned");
+ }
println!();
println!("Biggest sub-folders:");
diff --git a/crates/diskghost-cli/tests/cli.rs b/crates/diskghost-cli/tests/cli.rs
index d9850f1..7f2261a 100644
--- a/crates/diskghost-cli/tests/cli.rs
+++ b/crates/diskghost-cli/tests/cli.rs
@@ -86,3 +86,182 @@ fn reclaim_defaults_to_dry_run() {
let s = String::from_utf8_lossy(&out.stdout);
assert!(s.contains("DRY-RUN"), "expected dry-run notice:\n{s}");
}
+
+/// A throwaway directory unique to this test process.
+fn scratch(label: &str) -> std::path::PathBuf {
+ let mut d = std::env::temp_dir();
+ d.push(format!("diskghost-cli-{}-{label}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&d);
+ std::fs::create_dir_all(&d).unwrap();
+ d
+}
+
+#[test]
+fn scan_save_then_since_and_diff() {
+ let d = scratch("snap");
+ let data = d.join("data");
+ std::fs::create_dir_all(data.join("keep")).unwrap();
+ std::fs::write(data.join("keep/a.bin"), [0u8; 100]).unwrap();
+ let old = d.join("old.json");
+ let new = d.join("new.json");
+
+ // First scan saves a snapshot; nothing to compare yet.
+ let out = Command::new(bin())
+ .args([
+ "scan",
+ data.to_str().unwrap(),
+ "--save",
+ old.to_str().unwrap(),
+ ])
+ .output()
+ .expect("run diskghost");
+ assert!(out.status.success());
+ assert!(old.is_file(), "snapshot file should exist");
+ let err = String::from_utf8_lossy(&out.stderr);
+ assert!(err.contains("snapshot saved"), "stderr:\n{err}");
+
+ // Grow one folder, add another, then compare while saving again.
+ std::fs::write(data.join("keep/b.bin"), [0u8; 100_000]).unwrap();
+ std::fs::create_dir_all(data.join("fresh")).unwrap();
+ std::fs::write(data.join("fresh/c.bin"), [0u8; 100]).unwrap();
+ let out = Command::new(bin())
+ .args([
+ "scan",
+ data.to_str().unwrap(),
+ "--since",
+ old.to_str().unwrap(),
+ "--save",
+ new.to_str().unwrap(),
+ ])
+ .output()
+ .expect("run diskghost");
+ assert!(out.status.success());
+ let s = String::from_utf8_lossy(&out.stdout);
+ assert!(s.contains("Changes in"), "missing diff section:\n{s}");
+ assert!(s.contains("Grew:"), "keep should have grown:\n{s}");
+ assert!(s.contains("New folders:"), "fresh should be new:\n{s}");
+
+ // JSON mode nests the diff next to the report fields.
+ let out = Command::new(bin())
+ .args([
+ "scan",
+ data.to_str().unwrap(),
+ "--since",
+ old.to_str().unwrap(),
+ "--json",
+ ])
+ .output()
+ .expect("run diskghost");
+ assert!(out.status.success());
+ let v: serde_json::Value =
+ serde_json::from_str(&String::from_utf8_lossy(&out.stdout)).expect("valid JSON");
+ assert!(v.get("total_size").is_some());
+ let diff = v.get("diff").expect("diff key");
+ assert_eq!(diff["grown"].as_array().unwrap().len(), 1);
+ assert_eq!(diff["added"].as_array().unwrap().len(), 1);
+ assert!(diff["delta"].as_i64().unwrap() > 0);
+
+ // Standalone diff of the two saved snapshots, human and JSON.
+ let out = Command::new(bin())
+ .args(["diff", old.to_str().unwrap(), new.to_str().unwrap()])
+ .output()
+ .expect("run diskghost");
+ assert!(out.status.success());
+ let s = String::from_utf8_lossy(&out.stdout);
+ assert!(s.contains("Grew:") && s.contains("New folders:"), "{s}");
+
+ let out = Command::new(bin())
+ .args([
+ "diff",
+ old.to_str().unwrap(),
+ new.to_str().unwrap(),
+ "--at",
+ data.join("keep").to_str().unwrap(),
+ "--json",
+ ])
+ .output()
+ .expect("run diskghost");
+ assert!(out.status.success());
+ let v: serde_json::Value =
+ serde_json::from_str(&String::from_utf8_lossy(&out.stdout)).expect("valid JSON");
+ assert_eq!(v["files_delta"].as_i64().unwrap(), 1);
+ assert!(v["grown"].as_array().unwrap().is_empty()); // keep has no sub-folders
+
+ // Comparing identical snapshots says so.
+ let out = Command::new(bin())
+ .args(["diff", new.to_str().unwrap(), new.to_str().unwrap()])
+ .output()
+ .expect("run diskghost");
+ assert!(out.status.success());
+ assert!(String::from_utf8_lossy(&out.stdout).contains("nothing changed"));
+
+ std::fs::remove_dir_all(&d).ok();
+}
+
+#[test]
+fn snapshot_errors_are_reported() {
+ let d = scratch("snaperr");
+ let bad = d.join("bad.json");
+ std::fs::write(&bad, "not json").unwrap();
+
+ // A broken --since fails before scanning.
+ let out = Command::new(bin())
+ .args(["scan", sample_dir(), "--since", bad.to_str().unwrap()])
+ .output()
+ .expect("run diskghost");
+ assert!(!out.status.success());
+ assert!(String::from_utf8_lossy(&out.stderr).contains("cannot read snapshot"));
+
+ // A missing file for diff fails too.
+ let out = Command::new(bin())
+ .args([
+ "diff",
+ bad.to_str().unwrap(),
+ d.join("none.json").to_str().unwrap(),
+ ])
+ .output()
+ .expect("run diskghost");
+ assert!(!out.status.success());
+
+ // Snapshots of different roots cannot be compared.
+ let a = d.join("a");
+ let b = d.join("b");
+ std::fs::create_dir_all(&a).unwrap();
+ std::fs::create_dir_all(&b).unwrap();
+ let sa = d.join("a.json");
+ let sb = d.join("b.json");
+ for (dir, file) in [(&a, &sa), (&b, &sb)] {
+ let out = Command::new(bin())
+ .args([
+ "scan",
+ dir.to_str().unwrap(),
+ "--save",
+ file.to_str().unwrap(),
+ ])
+ .output()
+ .expect("run diskghost");
+ assert!(out.status.success());
+ }
+ let out = Command::new(bin())
+ .args(["diff", sa.to_str().unwrap(), sb.to_str().unwrap()])
+ .output()
+ .expect("run diskghost");
+ assert!(!out.status.success());
+ assert!(String::from_utf8_lossy(&out.stderr).contains("different directories"));
+
+ // --at must name a folder inside the newer snapshot.
+ let out = Command::new(bin())
+ .args([
+ "diff",
+ sa.to_str().unwrap(),
+ sa.to_str().unwrap(),
+ "--at",
+ a.join("nope").to_str().unwrap(),
+ ])
+ .output()
+ .expect("run diskghost");
+ assert!(!out.status.success());
+ assert!(String::from_utf8_lossy(&out.stderr).contains("not a directory in the snapshot"));
+
+ std::fs::remove_dir_all(&d).ok();
+}
diff --git a/crates/diskghost-core/Cargo.toml b/crates/diskghost-core/Cargo.toml
index fbcb960..5d8080e 100644
--- a/crates/diskghost-core/Cargo.toml
+++ b/crates/diskghost-core/Cargo.toml
@@ -17,6 +17,7 @@ globset = "0.4"
trash = "5"
fs4 = "1"
serde = { version = "1", features = ["derive"] }
+serde_json = "1"
[dev-dependencies]
criterion = "0.5"
diff --git a/crates/diskghost-core/src/lib.rs b/crates/diskghost-core/src/lib.rs
index a15171c..c3463c4 100644
--- a/crates/diskghost-core/src/lib.rs
+++ b/crates/diskghost-core/src/lib.rs
@@ -8,16 +8,29 @@
//! * [`reclaim`] — act on duplicates: delete, send to trash, or replace with a
//! hard link (dry-run by default at the call site).
//!
+//! * [`ScanTree`] — the whole tree from one walk, so a UI can drill into any
+//! sub-folder and back up without touching the disk again.
+//! * [`Snapshot`] / [`diff`] — persist a scan and compare a later one against
+//! it: what grew, what shrank, what is new, what is gone.
+//!
//! [`Options`] controls the walk: exclude globs, max depth, follow-symlinks.
//! Everything is `serde`-serialisable so a CLI, a GUI or an agent can consume it.
+mod snapshot;
+mod tree;
+
+pub use snapshot::{
+ diff, format_timestamp, DiffReport, DirDelta, Snapshot, SnapshotError, SNAPSHOT_FORMAT,
+};
+pub use tree::ScanTree;
+
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use rayon::prelude::*;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
/// Options controlling how the filesystem is walked (shared by scan + dupes).
#[derive(Debug, Clone, Default)]
@@ -61,7 +74,7 @@ impl Progress {
}
/// A single file with its size in bytes.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FileEntry {
pub path: PathBuf,
/// Allocated size on disk — what the file actually occupies. This is what
@@ -76,7 +89,7 @@ pub struct FileEntry {
}
/// Aggregated size of a directory.
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DirSize {
pub path: PathBuf,
pub size: u64,
@@ -84,7 +97,7 @@ pub struct DirSize {
}
/// Summary of a scan.
-#[derive(Debug, Serialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScanReport {
pub root: PathBuf,
pub total_size: u64,
@@ -103,6 +116,10 @@ pub struct ScanReport {
pub disk_total: u64,
/// Available bytes on that filesystem for non-privileged users (0 if unknown).
pub disk_free: u64,
+ /// True if the walk was cancelled via [`Progress::cancel`] — every number
+ /// above then covers only the part of the tree seen before the stop.
+ #[serde(default)]
+ pub cancelled: bool,
}
/// A group of byte-identical files (hard links to the same physical file are
@@ -162,10 +179,11 @@ pub struct RemoveReport {
/// First N bytes hashed as a cheap pre-filter before a full-file hash.
const PREFIX_LEN: usize = 4096;
-struct Walk {
- files: Vec,
- dirs: u64,
- skipped: u64,
+pub(crate) struct Walk {
+ pub(crate) files: Vec,
+ /// Every directory below the root (the root itself is not listed).
+ pub(crate) dirs: Vec,
+ pub(crate) skipped: u64,
}
fn build_globset(patterns: &[String]) -> globset::GlobSet {
@@ -241,6 +259,7 @@ fn compressed_size(path: &Path) -> Option {
extern "system" {
fn GetCompressedFileSizeW(lp_file_name: *const u16, lp_file_size_high: *mut u32) -> u32;
fn GetLastError() -> u32;
+ fn SetLastError(dw_err_code: u32);
}
let wide: Vec = path
.as_os_str()
@@ -248,6 +267,11 @@ fn compressed_size(path: &Path) -> Option {
.chain(std::iter::once(0))
.collect();
let mut high: u32 = 0;
+ // Clear any stale error first: a legitimate low dword of 0xFFFFFFFF is only
+ // distinguishable from failure by GetLastError, which the API does not
+ // reset on success.
+ // SAFETY: no preconditions.
+ unsafe { SetLastError(0) };
// SAFETY: `wide` is a valid NUL-terminated UTF-16 string kept alive for the
// call, and `high` is a valid out-pointer to a u32.
let low = unsafe { GetCompressedFileSizeW(wide.as_ptr(), &mut high) };
@@ -276,9 +300,9 @@ fn on_disk_size(meta: &std::fs::Metadata, _path: &Path) -> u64 {
meta.len()
}
-/// Walk `root` recursively in a single pass: collect files (with size), count
+/// Walk `root` recursively in a single pass: collect files (with size), collect
/// directories, and count unreadable entries, honouring [`Options`].
-fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk {
+pub(crate) fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk {
let glob = Arc::new(build_globset(&opts.exclude));
// Each file's size is cached in the entry's client_state (Some(size), or
@@ -315,7 +339,7 @@ fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk {
});
let mut files = Vec::new();
- let mut dirs = 0u64;
+ let mut dirs = Vec::new();
let mut skipped = 0u64;
for entry in wd {
@@ -327,7 +351,7 @@ fn walk(root: &Path, opts: &Options, progress: &Progress) -> Walk {
let ft = e.file_type();
if ft.is_dir() {
if e.depth() > 0 {
- dirs += 1; // don't count the root itself
+ dirs.push(e.path()); // the root itself is not listed
}
} else if ft.is_file() {
match e.client_state {
@@ -368,74 +392,15 @@ pub fn scan(root: &Path, top_n: usize) -> ScanReport {
/// Scan `root`: total size, the `top_n` biggest immediate sub-folders, the bytes of
/// files directly in the root, and the `top_n` biggest files. Reports live progress
-/// and can be cancelled via [`Progress`].
+/// and can be cancelled via [`Progress`]. Builds a [`ScanTree`] under the hood;
+/// keep the tree instead if you want to drill into sub-folders afterwards.
pub fn scan_with_progress(
root: &Path,
top_n: usize,
opts: &Options,
progress: &Progress,
) -> ScanReport {
- let Walk {
- files,
- dirs,
- skipped,
- } = walk(root, opts, progress);
-
- let total_size: u64 = files.iter().map(|f| f.size).sum();
- let total_files = files.len() as u64;
-
- let mut by_child: HashMap = HashMap::new();
- let mut root_files_size = 0u64;
- let mut root_files_count = 0u64;
-
- for f in &files {
- if let Ok(rel) = f.path.strip_prefix(root) {
- let mut comps = rel.components();
- match (comps.next(), comps.next()) {
- // Two or more components: `first` is a real sub-directory.
- (Some(first), Some(_)) => {
- let key = root.join(first.as_os_str());
- let entry = by_child.entry(key.clone()).or_insert_with(|| DirSize {
- path: key,
- size: 0,
- file_count: 0,
- });
- entry.size += f.size;
- entry.file_count += 1;
- }
- // Exactly one component: the file sits directly in the root.
- (Some(_), None) => {
- root_files_size += f.size;
- root_files_count += 1;
- }
- _ => {}
- }
- }
- }
-
- let mut children: Vec = by_child.into_values().collect();
- children.sort_by_key(|d| std::cmp::Reverse(d.size));
- children.truncate(top_n);
-
- let mut top_files = files;
- top_files.sort_by_key(|f| std::cmp::Reverse(f.size));
- top_files.truncate(top_n);
-
- let (disk_total, disk_free) = disk_space(root);
-
- ScanReport {
- root: root.to_path_buf(),
- total_size,
- total_files,
- total_dirs: dirs,
- skipped,
- children,
- root_files_size,
- root_files_count,
- top_files,
- disk_total,
- disk_free,
- }
+ ScanTree::build(root, opts, progress).root_report(top_n)
}
/// Total and available bytes of the filesystem that holds `path`. Returns
@@ -791,6 +756,15 @@ pub fn human_size(bytes: u64) -> String {
}
}
+/// Format a signed byte difference, e.g. `+1.5 GB`, `-300.0 MB`, or `0 B`.
+pub fn human_delta(delta: i64) -> String {
+ match delta {
+ 0 => "0 B".to_string(),
+ d if d > 0 => format!("+{}", human_size(d.unsigned_abs())),
+ d => format!("-{}", human_size(d.unsigned_abs())),
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1040,6 +1014,45 @@ mod tests {
assert_eq!(human_size(1536), "1.5 KB");
}
+ #[test]
+ fn human_delta_carries_the_sign() {
+ assert_eq!(human_delta(0), "0 B");
+ assert_eq!(human_delta(1536), "+1.5 KB");
+ assert_eq!(human_delta(-1536), "-1.5 KB");
+ assert_eq!(human_delta(-1), "-1 B");
+ }
+
+ #[test]
+ fn cancelled_scan_is_flagged_in_the_report() {
+ let d = tmpdir("scancancel");
+ write(&d.join("a/1.bin"), &[0u8; 10]);
+ let ok = scan(&d, 10);
+ assert!(!ok.cancelled);
+ let c = Progress::default();
+ c.cancel();
+ let r = scan_with_progress(&d, 10, &Options::default(), &c);
+ assert!(r.cancelled);
+ assert_eq!(r.total_files, 0);
+ std::fs::remove_dir_all(&d).ok();
+ }
+
+ #[test]
+ fn cancelled_duplicate_search_stops_early() {
+ let d = tmpdir("dupecancel");
+ let body = b"duplicate content that is long enough to matter here";
+ write(&d.join("x/1.bin"), body);
+ write(&d.join("y/2.bin"), body);
+ assert_eq!(find_duplicates(&d, 1).len(), 1);
+
+ // Pre-cancelled: the walk sees nothing, so no groups can form.
+ let c = Progress::default();
+ c.cancel();
+ let groups = find_duplicates_with_progress(&d, 1, &Options::default(), &c);
+ assert!(groups.is_empty());
+ assert_eq!(c.files(), 0);
+ std::fs::remove_dir_all(&d).ok();
+ }
+
#[test]
fn remove_dry_run_counts_but_keeps_everything() {
let d = tmpdir("rmdry");
diff --git a/crates/diskghost-core/src/snapshot.rs b/crates/diskghost-core/src/snapshot.rs
new file mode 100644
index 0000000..8212d5e
--- /dev/null
+++ b/crates/diskghost-core/src/snapshot.rs
@@ -0,0 +1,515 @@
+//! Snapshots persist a scan so a later one can be compared against it:
+//! "what grew since last time".
+//!
+//! A [`Snapshot`] records *every* directory of a [`ScanTree`] with its subtree
+//! totals (not just the biggest ones), so [`diff`] can answer for the root or
+//! for any sub-folder which children grew, shrank, appeared or vanished.
+//! Snapshots are plain JSON, versioned by [`SNAPSHOT_FORMAT`].
+
+use std::collections::HashMap;
+use std::fmt;
+use std::path::{Path, PathBuf};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use serde::{Deserialize, Serialize};
+
+use crate::{DirSize, FileEntry, ScanTree};
+
+/// On-disk format version of [`Snapshot`]; bumped on incompatible changes.
+pub const SNAPSHOT_FORMAT: u32 = 1;
+
+/// Why a snapshot could not be read, written or compared.
+#[derive(Debug)]
+pub enum SnapshotError {
+ Io(std::io::Error),
+ Json(serde_json::Error),
+ /// The file is a snapshot of another format version.
+ Format(u32),
+ /// The two snapshots were taken of different directories.
+ RootMismatch {
+ old: PathBuf,
+ new: PathBuf,
+ },
+ /// The directory to compare at is not part of the newer snapshot.
+ NotInSnapshot(PathBuf),
+}
+
+impl fmt::Display for SnapshotError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ SnapshotError::Io(e) => write!(f, "{e}"),
+ SnapshotError::Json(e) => write!(f, "invalid snapshot: {e}"),
+ SnapshotError::Format(v) => write!(
+ f,
+ "unsupported snapshot format {v} (this build reads format {SNAPSHOT_FORMAT})"
+ ),
+ SnapshotError::RootMismatch { old, new } => write!(
+ f,
+ "snapshots are of different directories: {} vs {}",
+ old.display(),
+ new.display()
+ ),
+ SnapshotError::NotInSnapshot(p) => {
+ write!(f, "not a directory in the snapshot: {}", p.display())
+ }
+ }
+ }
+}
+
+impl std::error::Error for SnapshotError {}
+
+impl From for SnapshotError {
+ fn from(e: std::io::Error) -> Self {
+ SnapshotError::Io(e)
+ }
+}
+
+impl From for SnapshotError {
+ fn from(e: serde_json::Error) -> Self {
+ SnapshotError::Json(e)
+ }
+}
+
+/// A persisted scan: totals of the root, every directory below it, and the
+/// biggest files.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Snapshot {
+ pub format: u32,
+ /// When the scan was taken, in seconds since the Unix epoch.
+ pub taken_at: u64,
+ pub root: PathBuf,
+ pub total_size: u64,
+ pub total_files: u64,
+ pub total_dirs: u64,
+ /// Every directory below the root with its subtree totals, sorted by path.
+ pub dirs: Vec,
+ /// Largest files anywhere under the root, largest first.
+ pub top_files: Vec,
+}
+
+/// Only the field needed to reject a foreign format before parsing the rest.
+#[derive(Deserialize)]
+struct Header {
+ format: u32,
+}
+
+fn now_unix() -> u64 {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map(|d| d.as_secs())
+ .unwrap_or(0)
+}
+
+impl Snapshot {
+ /// Capture `tree` as it is now, keeping the `top_files` biggest files.
+ pub fn capture(tree: &ScanTree, top_files: usize) -> Snapshot {
+ let r = tree.root_report(top_files);
+ Snapshot {
+ format: SNAPSHOT_FORMAT,
+ taken_at: now_unix(),
+ root: r.root,
+ total_size: r.total_size,
+ total_files: r.total_files,
+ total_dirs: r.total_dirs,
+ dirs: tree.dir_sizes(),
+ top_files: r.top_files,
+ }
+ }
+
+ /// Read a snapshot written by [`Snapshot::save`].
+ pub fn load(path: &Path) -> Result {
+ let text = std::fs::read_to_string(path)?;
+ let header: Header = serde_json::from_str(&text)?;
+ if header.format != SNAPSHOT_FORMAT {
+ return Err(SnapshotError::Format(header.format));
+ }
+ Ok(serde_json::from_str(&text)?)
+ }
+
+ /// Write the snapshot as JSON, creating parent directories as needed.
+ pub fn save(&self, path: &Path) -> Result<(), SnapshotError> {
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent)?;
+ }
+ std::fs::write(path, serde_json::to_vec(self)?)?;
+ Ok(())
+ }
+
+ /// A short, filesystem-safe key that identifies a scanned root, so a store
+ /// can keep one history per directory (BLAKE3 of the path, 16 hex chars).
+ pub fn root_key(root: &Path) -> String {
+ let hex = blake3::hash(root.to_string_lossy().as_bytes()).to_hex();
+ hex[..16].to_string()
+ }
+}
+
+/// One sub-folder whose size changed between two snapshots.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DirDelta {
+ pub path: PathBuf,
+ pub old_size: u64,
+ pub new_size: u64,
+ /// `new_size - old_size`; positive means it grew.
+ pub delta: i64,
+ pub old_files: u64,
+ pub new_files: u64,
+}
+
+/// What changed between two snapshots of the same root, seen from one directory.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DiffReport {
+ /// The directory the comparison is about (the root or a sub-folder).
+ pub root: PathBuf,
+ pub old_taken_at: u64,
+ pub new_taken_at: u64,
+ pub old_size: u64,
+ pub new_size: u64,
+ /// `new_size - old_size` of the whole directory.
+ pub delta: i64,
+ pub old_files: u64,
+ pub new_files: u64,
+ pub files_delta: i64,
+ /// Immediate sub-folders that grew, biggest gain first.
+ pub grown: Vec,
+ /// Immediate sub-folders that shrank, biggest loss first.
+ pub shrunk: Vec,
+ /// Immediate sub-folders present now but not before, largest first.
+ pub added: Vec,
+ /// Immediate sub-folders present before but gone now, largest first.
+ pub removed: Vec,
+}
+
+impl DiffReport {
+ /// Keep at most `top` entries in each list.
+ pub fn truncate(&mut self, top: usize) {
+ self.grown.truncate(top);
+ self.shrunk.truncate(top);
+ self.added.truncate(top);
+ self.removed.truncate(top);
+ }
+
+ /// True if nothing at all changed under this directory.
+ pub fn is_unchanged(&self) -> bool {
+ self.delta == 0
+ && self.files_delta == 0
+ && self.grown.is_empty()
+ && self.shrunk.is_empty()
+ && self.added.is_empty()
+ && self.removed.is_empty()
+ }
+}
+
+fn signed(x: u64) -> i64 {
+ i64::try_from(x).unwrap_or(i64::MAX)
+}
+
+/// The immediate sub-folders of `at` in a snapshot, keyed by path.
+fn children_of<'a>(snap: &'a Snapshot, at: &Path) -> HashMap<&'a Path, &'a DirSize> {
+ snap.dirs
+ .iter()
+ .filter(|d| d.path.parent() == Some(at))
+ .map(|d| (d.path.as_path(), d))
+ .collect()
+}
+
+/// Compare two snapshots of the same root at directory `at` (the root or any
+/// sub-folder of the newer snapshot). A folder missing from the older snapshot
+/// counts as having been empty.
+pub fn diff(old: &Snapshot, new: &Snapshot, at: &Path) -> Result {
+ if old.root != new.root {
+ return Err(SnapshotError::RootMismatch {
+ old: old.root.clone(),
+ new: new.root.clone(),
+ });
+ }
+ let totals = |snap: &Snapshot| -> Option<(u64, u64)> {
+ if at == snap.root {
+ return Some((snap.total_size, snap.total_files));
+ }
+ snap.dirs
+ .iter()
+ .find(|d| d.path == at)
+ .map(|d| (d.size, d.file_count))
+ };
+ let (new_size, new_files) =
+ totals(new).ok_or_else(|| SnapshotError::NotInSnapshot(at.to_path_buf()))?;
+ let (old_size, old_files) = totals(old).unwrap_or((0, 0));
+
+ let old_kids = children_of(old, at);
+ let new_kids = children_of(new, at);
+
+ let mut grown = Vec::new();
+ let mut shrunk = Vec::new();
+ let mut added = Vec::new();
+ let mut removed = Vec::new();
+
+ for (path, n) in &new_kids {
+ match old_kids.get(path) {
+ Some(o) => {
+ let delta = signed(n.size) - signed(o.size);
+ if delta == 0 {
+ continue;
+ }
+ let entry = DirDelta {
+ path: n.path.clone(),
+ old_size: o.size,
+ new_size: n.size,
+ delta,
+ old_files: o.file_count,
+ new_files: n.file_count,
+ };
+ if delta > 0 {
+ grown.push(entry);
+ } else {
+ shrunk.push(entry);
+ }
+ }
+ None => added.push((*n).clone()),
+ }
+ }
+ for (path, o) in &old_kids {
+ if !new_kids.contains_key(path) {
+ removed.push((*o).clone());
+ }
+ }
+
+ grown.sort_by_key(|d| (std::cmp::Reverse(d.delta), d.path.clone()));
+ shrunk.sort_by_key(|d| (d.delta, d.path.clone()));
+ added.sort_by_key(|d| (std::cmp::Reverse(d.size), d.path.clone()));
+ removed.sort_by_key(|d| (std::cmp::Reverse(d.size), d.path.clone()));
+
+ Ok(DiffReport {
+ root: at.to_path_buf(),
+ old_taken_at: old.taken_at,
+ new_taken_at: new.taken_at,
+ old_size,
+ new_size,
+ delta: signed(new_size) - signed(old_size),
+ old_files,
+ new_files,
+ files_delta: signed(new_files) - signed(old_files),
+ grown,
+ shrunk,
+ added,
+ removed,
+ })
+}
+
+/// Days since 1970-01-01 to a proleptic Gregorian (year, month, day).
+fn civil_from_days(days: i64) -> (i64, u32, u32) {
+ let z = days + 719_468;
+ let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
+ let doe = z - era * 146_097;
+ let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
+ let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
+ let mp = (5 * doy + 2) / 153;
+ let day = (doy - (153 * mp + 2) / 5 + 1) as u32;
+ let month = (if mp < 10 { mp + 3 } else { mp - 9 }) as u32;
+ let year = yoe + era * 400 + i64::from(month <= 2);
+ (year, month, day)
+}
+
+/// Render a Unix timestamp as `YYYY-MM-DD HH:MM UTC`.
+pub fn format_timestamp(secs: u64) -> String {
+ let days = signed(secs) / 86_400;
+ let rem = secs % 86_400;
+ let (y, m, d) = civil_from_days(days);
+ format!(
+ "{y:04}-{m:02}-{d:02} {:02}:{:02} UTC",
+ rem / 3600,
+ (rem % 3600) / 60
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::{Options, Progress};
+ use std::io::Write;
+
+ fn tmpdir(label: &str) -> PathBuf {
+ let mut d = std::env::temp_dir();
+ d.push(format!("diskghost-snap-{}-{label}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&d);
+ std::fs::create_dir_all(&d).unwrap();
+ d
+ }
+
+ fn write(path: &Path, content: &[u8]) {
+ if let Some(p) = path.parent() {
+ std::fs::create_dir_all(p).unwrap();
+ }
+ let mut f = std::fs::File::create(path).unwrap();
+ f.write_all(content).unwrap();
+ }
+
+ fn snap(dir: &Path) -> Snapshot {
+ Snapshot::capture(
+ &ScanTree::build(dir, &Options::default(), &Progress::default()),
+ 5,
+ )
+ }
+
+ #[test]
+ fn capture_save_load_roundtrip() {
+ let d = tmpdir("roundtrip");
+ write(&d.join("a/1.bin"), &[0u8; 100]);
+ write(&d.join("a/b/2.bin"), &[0u8; 50]);
+ let s = snap(&d);
+ assert_eq!(s.format, SNAPSHOT_FORMAT);
+ assert!(s.taken_at > 1_600_000_000);
+ assert_eq!(s.root, d);
+ assert_eq!(s.total_files, 2);
+ assert_eq!(s.total_dirs, 2);
+ assert_eq!(s.dirs.len(), 2);
+ assert_eq!(s.top_files.len(), 2);
+
+ let file = d.join("history/snap.json");
+ s.save(&file).unwrap();
+ let back = Snapshot::load(&file).unwrap();
+ assert_eq!(back, s);
+
+ // Wrong format and broken JSON are reported, not silently accepted.
+ std::fs::write(&file, br#"{"format": 99}"#).unwrap();
+ assert!(matches!(
+ Snapshot::load(&file),
+ Err(SnapshotError::Format(99))
+ ));
+ std::fs::write(&file, b"not json").unwrap();
+ assert!(matches!(Snapshot::load(&file), Err(SnapshotError::Json(_))));
+ assert!(matches!(
+ Snapshot::load(&d.join("missing.json")),
+ Err(SnapshotError::Io(_))
+ ));
+ std::fs::remove_dir_all(&d).ok();
+ }
+
+ #[test]
+ fn diff_reports_grown_shrunk_added_removed() {
+ let d = tmpdir("diff");
+ write(&d.join("grow/1.bin"), &[0u8; 100]);
+ write(&d.join("shrink/1.bin"), &[0u8; 100]);
+ write(&d.join("same/1.bin"), &[0u8; 100]);
+ write(&d.join("gone/1.bin"), &[0u8; 100]);
+ let old = snap(&d);
+
+ write(&d.join("grow/2.bin"), &[0u8; 100_000]);
+ std::fs::remove_file(d.join("shrink/1.bin")).unwrap();
+ std::fs::remove_dir_all(d.join("gone")).unwrap();
+ write(&d.join("fresh/1.bin"), &[0u8; 100]);
+ let new = snap(&d);
+
+ let r = diff(&old, &new, &d).unwrap();
+ assert_eq!(r.root, d);
+ assert_eq!(r.old_taken_at, old.taken_at);
+ assert_eq!(r.new_taken_at, new.taken_at);
+ assert_eq!(r.old_files, 4);
+ assert_eq!(r.new_files, 4);
+ assert_eq!(r.files_delta, 0);
+ assert_eq!(r.delta, signed(new.total_size) - signed(old.total_size));
+ assert!(r.delta > 0);
+ assert!(!r.is_unchanged());
+
+ assert_eq!(r.grown.len(), 1);
+ assert_eq!(r.grown[0].path, d.join("grow"));
+ assert_eq!(r.grown[0].old_files, 1);
+ assert_eq!(r.grown[0].new_files, 2);
+ assert_eq!(
+ r.grown[0].delta,
+ signed(r.grown[0].new_size) - signed(r.grown[0].old_size)
+ );
+ assert_eq!(r.shrunk.len(), 1);
+ assert_eq!(r.shrunk[0].path, d.join("shrink"));
+ assert!(r.shrunk[0].delta < 0);
+ assert_eq!(r.shrunk[0].new_files, 0);
+ assert_eq!(r.added.len(), 1);
+ assert_eq!(r.added[0].path, d.join("fresh"));
+ assert_eq!(r.removed.len(), 1);
+ assert_eq!(r.removed[0].path, d.join("gone"));
+
+ // Identical snapshots: nothing to report.
+ let none = diff(&new, &new, &d).unwrap();
+ assert!(none.is_unchanged());
+
+ // Truncation caps every list.
+ let mut capped = r.clone();
+ capped.truncate(0);
+ assert!(capped.grown.is_empty() && capped.removed.is_empty());
+ std::fs::remove_dir_all(&d).ok();
+ }
+
+ #[test]
+ fn diff_at_a_sub_folder() {
+ let d = tmpdir("diffsub");
+ write(&d.join("a/x/1.bin"), &[0u8; 100]);
+ let old = snap(&d);
+ write(&d.join("a/x/2.bin"), &[0u8; 100]);
+ write(&d.join("a/y/1.bin"), &[0u8; 100]);
+ let new = snap(&d);
+
+ let r = diff(&old, &new, &d.join("a")).unwrap();
+ assert_eq!(r.root, d.join("a"));
+ assert_eq!(r.old_files, 1);
+ assert_eq!(r.new_files, 3);
+ assert_eq!(r.files_delta, 2);
+ assert_eq!(r.grown.len(), 1);
+ assert_eq!(r.grown[0].path, d.join("a/x"));
+ assert_eq!(r.added.len(), 1);
+ assert_eq!(r.added[0].path, d.join("a/y"));
+ assert!(r.shrunk.is_empty() && r.removed.is_empty());
+
+ // A folder the old snapshot never saw counts as previously empty.
+ let y = diff(&old, &new, &d.join("a/y")).unwrap();
+ assert_eq!(y.old_size, 0);
+ assert_eq!(y.old_files, 0);
+ assert_eq!(y.new_files, 1);
+
+ // But it must exist in the new one.
+ assert!(matches!(
+ diff(&old, &new, &d.join("nope")),
+ Err(SnapshotError::NotInSnapshot(_))
+ ));
+ std::fs::remove_dir_all(&d).ok();
+ }
+
+ #[test]
+ fn diff_refuses_different_roots() {
+ let d1 = tmpdir("root1");
+ let d2 = tmpdir("root2");
+ let err = diff(&snap(&d1), &snap(&d2), &d1).unwrap_err();
+ assert!(matches!(err, SnapshotError::RootMismatch { .. }));
+ assert!(err.to_string().contains("different directories"));
+ std::fs::remove_dir_all(&d1).ok();
+ std::fs::remove_dir_all(&d2).ok();
+ }
+
+ #[test]
+ fn error_messages_are_descriptive() {
+ assert!(SnapshotError::Format(7).to_string().contains("format 7"));
+ assert!(SnapshotError::NotInSnapshot(PathBuf::from("x"))
+ .to_string()
+ .contains("not a directory"));
+ let io: SnapshotError = std::io::Error::other("boom").into();
+ assert_eq!(io.to_string(), "boom");
+ let json: SnapshotError = serde_json::from_str::("{").unwrap_err().into();
+ assert!(json.to_string().starts_with("invalid snapshot:"));
+ assert!(std::error::Error::source(&json).is_none());
+ }
+
+ #[test]
+ fn root_key_is_stable_and_short() {
+ let a = Snapshot::root_key(Path::new("/some/dir"));
+ assert_eq!(a.len(), 16);
+ assert_eq!(a, Snapshot::root_key(Path::new("/some/dir")));
+ assert_ne!(a, Snapshot::root_key(Path::new("/other/dir")));
+ assert!(a.chars().all(|c| c.is_ascii_hexdigit()));
+ }
+
+ #[test]
+ fn timestamps_render_as_utc() {
+ assert_eq!(format_timestamp(0), "1970-01-01 00:00 UTC");
+ assert_eq!(format_timestamp(1_600_000_000), "2020-09-13 12:26 UTC");
+ assert_eq!(format_timestamp(951_782_400), "2000-02-29 00:00 UTC");
+ assert_eq!(format_timestamp(1_704_067_199), "2023-12-31 23:59 UTC");
+ }
+}
diff --git a/crates/diskghost-core/src/tree.rs b/crates/diskghost-core/src/tree.rs
new file mode 100644
index 0000000..fdd8f31
--- /dev/null
+++ b/crates/diskghost-core/src/tree.rs
@@ -0,0 +1,474 @@
+//! The whole directory tree from a single walk, held in memory so a UI can
+//! drill into any sub-folder — and back up — without touching the disk again.
+//!
+//! Nodes live in an arena (`Vec`) and are looked up by path through an
+//! index. A node's totals (`size`, `file_count`, `dir_count`) always cover its
+//! entire subtree; the files sitting directly in a directory are kept on that
+//! node so any sub-folder can produce the same [`ScanReport`] a fresh scan of
+//! it would.
+
+use std::cmp::Reverse;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+use crate::{disk_space, walk, DirSize, FileEntry, Options, Progress, ScanReport};
+
+struct Node {
+ path: PathBuf,
+ parent: Option,
+ /// Bytes in this directory and everything below it.
+ size: u64,
+ /// Files in this directory and everything below it.
+ file_count: u64,
+ /// Directories below this one (not counting itself).
+ dir_count: u64,
+ children: Vec,
+ /// Files sitting directly in this directory.
+ files: Vec,
+}
+
+impl Node {
+ fn new(path: PathBuf, parent: Option) -> Node {
+ Node {
+ path,
+ parent,
+ size: 0,
+ file_count: 0,
+ dir_count: 0,
+ children: Vec::new(),
+ files: Vec::new(),
+ }
+ }
+}
+
+/// An in-memory scan of a directory tree. Build it once with
+/// [`ScanTree::build`], then ask for a [`ScanReport`] of the root or of any
+/// sub-folder with [`ScanTree::report`] — no further I/O.
+pub struct ScanTree {
+ /// Arena; index 0 is always the root.
+ nodes: Vec,
+ index: HashMap,
+ skipped: u64,
+ cancelled: bool,
+ disk_total: u64,
+ disk_free: u64,
+}
+
+impl ScanTree {
+ /// Walk `root` (honouring `opts`, reporting to `progress`) and build the tree.
+ pub fn build(root: &Path, opts: &Options, progress: &Progress) -> ScanTree {
+ let walked = walk(root, opts, progress);
+ let mut tree = ScanTree {
+ nodes: vec![Node::new(root.to_path_buf(), None)],
+ index: HashMap::new(),
+ skipped: walked.skipped,
+ cancelled: progress.cancelled(),
+ disk_total: 0,
+ disk_free: 0,
+ };
+ tree.index.insert(root.to_path_buf(), 0);
+
+ for dir in walked.dirs {
+ tree.ensure_dir(&dir);
+ }
+
+ // Files of one directory arrive together, so remembering the last
+ // parent saves a hash lookup per file on the hot path.
+ let mut last: Option<(PathBuf, usize)> = None;
+ for file in walked.files {
+ let parent = file.path.parent().unwrap_or(root);
+ let id = match &last {
+ Some((p, id)) if p == parent => *id,
+ _ => {
+ let id = tree.ensure_dir(parent);
+ last = Some((parent.to_path_buf(), id));
+ id
+ }
+ };
+ let node = &mut tree.nodes[id];
+ node.size += file.size;
+ node.file_count += 1;
+ node.files.push(file);
+ }
+
+ // Roll the totals up. A child is always created after its parent, so
+ // walking the arena backwards visits every node before its parent.
+ for id in (1..tree.nodes.len()).rev() {
+ let node = &tree.nodes[id];
+ let (size, files, dirs, parent) =
+ (node.size, node.file_count, node.dir_count + 1, node.parent);
+ if let Some(p) = parent {
+ let pn = &mut tree.nodes[p];
+ pn.size += size;
+ pn.file_count += files;
+ pn.dir_count += dirs;
+ }
+ }
+
+ let (disk_total, disk_free) = disk_space(root);
+ tree.disk_total = disk_total;
+ tree.disk_free = disk_free;
+ tree
+ }
+
+ /// Node id of `path`, creating it (and any missing ancestors up to the
+ /// nearest known one) if needed. Every node is inserted after its parent.
+ fn ensure_dir(&mut self, path: &Path) -> usize {
+ if let Some(&id) = self.index.get(path) {
+ return id;
+ }
+ let mut missing: Vec = Vec::new();
+ let mut cur = path.to_path_buf();
+ let anchor = loop {
+ if let Some(&id) = self.index.get(&cur) {
+ break id;
+ }
+ missing.push(cur.clone());
+ match cur.parent() {
+ Some(p) => cur = p.to_path_buf(),
+ // Ran past every known ancestor: hang the chain off the root.
+ None => break 0,
+ }
+ };
+ let mut parent = anchor;
+ for p in missing.into_iter().rev() {
+ let id = self.nodes.len();
+ self.nodes.push(Node::new(p.clone(), Some(parent)));
+ self.nodes[parent].children.push(id);
+ self.index.insert(p, id);
+ parent = id;
+ }
+ parent
+ }
+
+ /// The directory this tree was built from.
+ pub fn root(&self) -> &Path {
+ &self.nodes[0].path
+ }
+
+ /// Whether `path` is a directory in this tree (the root included).
+ pub fn contains(&self, path: &Path) -> bool {
+ self.index.contains_key(path)
+ }
+
+ /// True if the walk was cancelled; the tree then covers only what was seen.
+ pub fn cancelled(&self) -> bool {
+ self.cancelled
+ }
+
+ /// Re-query the filesystem's total/free bytes (e.g. after deleting files).
+ pub fn refresh_disk_space(&mut self) {
+ let (total, free) = disk_space(self.root());
+ self.disk_total = total;
+ self.disk_free = free;
+ }
+
+ /// Report for the root: the same shape a fresh [`crate::scan`] returns.
+ pub fn root_report(&self, top_n: usize) -> ScanReport {
+ self.report_of(0, top_n)
+ }
+
+ /// Report for `path` — the root or any directory below it — as if that
+ /// directory had been scanned on its own. `None` if it is not in the tree.
+ pub fn report(&self, path: &Path, top_n: usize) -> Option {
+ let &id = self.index.get(path)?;
+ Some(self.report_of(id, top_n))
+ }
+
+ fn report_of(&self, id: usize, top_n: usize) -> ScanReport {
+ let node = &self.nodes[id];
+
+ let mut kids: Vec<&Node> = node.children.iter().map(|&c| &self.nodes[c]).collect();
+ kids.sort_by_key(|k| Reverse(k.size));
+ let children: Vec = kids
+ .iter()
+ .take(top_n)
+ .map(|k| DirSize {
+ path: k.path.clone(),
+ size: k.size,
+ file_count: k.file_count,
+ })
+ .collect();
+
+ let root_files_size = node.files.iter().map(|f| f.size).sum();
+ let root_files_count = node.files.len() as u64;
+
+ // Biggest files anywhere below `id`: gather the subtree, then select
+ // the top slice without sorting everything.
+ let mut all: Vec<&FileEntry> = Vec::new();
+ let mut stack = vec![id];
+ while let Some(i) = stack.pop() {
+ let n = &self.nodes[i];
+ all.extend(n.files.iter());
+ stack.extend(n.children.iter().copied());
+ }
+ if top_n == 0 {
+ all.clear();
+ } else if all.len() > top_n {
+ all.select_nth_unstable_by_key(top_n - 1, |f| Reverse(f.size));
+ all.truncate(top_n);
+ }
+ all.sort_by_key(|f| Reverse(f.size));
+ let top_files = all.into_iter().cloned().collect();
+
+ ScanReport {
+ root: node.path.clone(),
+ total_size: node.size,
+ total_files: node.file_count,
+ total_dirs: node.dir_count,
+ skipped: self.skipped,
+ children,
+ root_files_size,
+ root_files_count,
+ top_files,
+ disk_total: self.disk_total,
+ disk_free: self.disk_free,
+ cancelled: self.cancelled,
+ }
+ }
+
+ /// Every directory below the root with its subtree totals, sorted by path.
+ /// The root itself is not included.
+ pub fn dir_sizes(&self) -> Vec {
+ let mut dirs: Vec = self
+ .index
+ .values()
+ .filter(|&&id| id != 0)
+ .map(|&id| {
+ let n = &self.nodes[id];
+ DirSize {
+ path: n.path.clone(),
+ size: n.size,
+ file_count: n.file_count,
+ }
+ })
+ .collect();
+ dirs.sort_by(|a, b| a.path.cmp(&b.path));
+ dirs
+ }
+
+ /// Drop a directory (with everything below it) or a single file from the
+ /// tree, adjusting every ancestor's totals — what a UI does after deleting
+ /// the path on disk, instead of re-walking. The root cannot be removed.
+ /// Returns `false` if the path is not in the tree.
+ pub fn remove(&mut self, path: &Path) -> bool {
+ if let Some(&id) = self.index.get(path) {
+ if id == 0 {
+ return false;
+ }
+ let node = &self.nodes[id];
+ let (size, files, dirs, parent) =
+ (node.size, node.file_count, node.dir_count + 1, node.parent);
+ // Unindex the subtree; the arena slots simply become unreachable.
+ let mut stack = vec![id];
+ while let Some(i) = stack.pop() {
+ let p = std::mem::take(&mut self.nodes[i].path);
+ self.index.remove(&p);
+ self.nodes[i].files.clear();
+ stack.extend(std::mem::take(&mut self.nodes[i].children));
+ }
+ if let Some(p) = parent {
+ self.nodes[p].children.retain(|&c| c != id);
+ self.subtract(p, size, files, dirs);
+ }
+ return true;
+ }
+ let Some(parent) = path.parent() else {
+ return false;
+ };
+ let Some(&pid) = self.index.get(parent) else {
+ return false;
+ };
+ let Some(pos) = self.nodes[pid].files.iter().position(|f| f.path == path) else {
+ return false;
+ };
+ let file = self.nodes[pid].files.swap_remove(pos);
+ self.subtract(pid, file.size, 1, 0);
+ true
+ }
+
+ /// Subtract totals from `id` and every ancestor up to the root.
+ fn subtract(&mut self, mut id: usize, size: u64, files: u64, dirs: u64) {
+ loop {
+ let n = &mut self.nodes[id];
+ n.size = n.size.saturating_sub(size);
+ n.file_count = n.file_count.saturating_sub(files);
+ n.dir_count = n.dir_count.saturating_sub(dirs);
+ match n.parent {
+ Some(p) => id = p,
+ None => break,
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::io::Write;
+
+ fn tmpdir(label: &str) -> PathBuf {
+ let mut d = std::env::temp_dir();
+ d.push(format!("diskghost-tree-{}-{label}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&d);
+ std::fs::create_dir_all(&d).unwrap();
+ d
+ }
+
+ fn write(path: &Path, content: &[u8]) {
+ if let Some(p) = path.parent() {
+ std::fs::create_dir_all(p).unwrap();
+ }
+ let mut f = std::fs::File::create(path).unwrap();
+ f.write_all(content).unwrap();
+ }
+
+ fn logical_total(dir: &Path) -> u64 {
+ crate::walk_files(dir).iter().map(|f| f.logical_size).sum()
+ }
+
+ #[test]
+ fn drill_in_matches_a_fresh_scan() {
+ // Sizes are spread well beyond one filesystem block: on-disk sizes
+ // round up to the block (4 KB on ext4), so small files would tie.
+ let d = tmpdir("drill");
+ write(&d.join("a/1.bin"), &[0u8; 100_000]);
+ write(&d.join("a/x/2.bin"), &[0u8; 40_000]);
+ write(&d.join("a/x/y/3.bin"), &[0u8; 20_000]);
+ write(&d.join("b/4.bin"), &[0u8; 10_000]);
+ write(&d.join("top.bin"), &[0u8; 100]);
+ std::fs::create_dir_all(d.join("empty")).unwrap();
+
+ let tree = ScanTree::build(&d, &Options::default(), &Progress::default());
+ assert_eq!(tree.root(), d.as_path());
+ assert!(tree.contains(&d.join("a/x/y")));
+ assert!(tree.contains(&d.join("empty")));
+ assert!(!tree.contains(&d.join("nope")));
+ assert!(!tree.cancelled());
+
+ let root = tree.root_report(10);
+ let fresh = crate::scan(&d, 10);
+ assert_eq!(root.total_size, fresh.total_size);
+ assert_eq!(root.total_files, 5);
+ assert_eq!(root.total_dirs, 5); // a, a/x, a/x/y, b, empty
+ assert_eq!(root.root_files_count, 1);
+ assert_eq!(root.children[0].path, d.join("a"));
+ assert_eq!(root.top_files[0].path, d.join("a/1.bin"));
+ assert_eq!(root.children.len(), 3);
+ assert_eq!(root.children[2].path, d.join("empty"));
+ assert_eq!(root.children[2].size, 0);
+
+ // A sub-folder from the tree equals a fresh scan of just that folder.
+ let sub = tree.report(&d.join("a/x"), 10).unwrap();
+ let fresh_sub = crate::scan(&d.join("a/x"), 10);
+ assert_eq!(sub.root, fresh_sub.root);
+ assert_eq!(sub.total_size, fresh_sub.total_size);
+ assert_eq!(sub.total_files, 2);
+ assert_eq!(sub.total_dirs, 1);
+ assert_eq!(sub.root_files_count, 1);
+ assert_eq!(sub.children.len(), 1);
+ assert_eq!(sub.children[0].path, d.join("a/x/y"));
+ assert_eq!(sub.top_files.len(), 2);
+ assert_eq!(sub.top_files[0].path, d.join("a/x/2.bin"));
+
+ assert!(tree.report(&d.join("nope"), 10).is_none());
+
+ // top_n bounds both lists; 0 yields none.
+ let one = tree.root_report(1);
+ assert_eq!(one.children.len(), 1);
+ assert_eq!(one.top_files.len(), 1);
+ let none = tree.root_report(0);
+ assert!(none.children.is_empty() && none.top_files.is_empty());
+
+ std::fs::remove_dir_all(&d).ok();
+ }
+
+ #[test]
+ fn dir_sizes_lists_every_directory() {
+ let d = tmpdir("dirsizes");
+ write(&d.join("a/1.bin"), &[0u8; 10]);
+ write(&d.join("a/b/2.bin"), &[0u8; 20]);
+ let tree = ScanTree::build(&d, &Options::default(), &Progress::default());
+ let dirs = tree.dir_sizes();
+ assert_eq!(dirs.len(), 2);
+ assert_eq!(dirs[0].path, d.join("a"));
+ assert_eq!(dirs[1].path, d.join("a/b"));
+ assert_eq!(dirs[0].file_count, 2);
+ assert_eq!(dirs[1].file_count, 1);
+ assert!(dirs[0].size >= dirs[1].size);
+ std::fs::remove_dir_all(&d).ok();
+ }
+
+ #[test]
+ fn remove_updates_every_ancestor() {
+ let d = tmpdir("remove");
+ write(&d.join("a/1.bin"), &[0u8; 1000]);
+ write(&d.join("a/x/2.bin"), &[0u8; 500]);
+ write(&d.join("b/3.bin"), &[0u8; 200]);
+ let mut tree = ScanTree::build(&d, &Options::default(), &Progress::default());
+ let before = tree.root_report(10);
+ let a_x = tree.report(&d.join("a/x"), 10).unwrap();
+
+ // A directory: its subtree leaves the tree and the totals shrink.
+ assert!(tree.remove(&d.join("a/x")));
+ assert!(!tree.contains(&d.join("a/x")));
+ let after = tree.root_report(10);
+ assert_eq!(after.total_size, before.total_size - a_x.total_size);
+ assert_eq!(after.total_files, before.total_files - 1);
+ assert_eq!(after.total_dirs, before.total_dirs - 1);
+ let a = tree.report(&d.join("a"), 10).unwrap();
+ assert_eq!(a.total_files, 1);
+ assert!(a.children.is_empty());
+
+ // A single file: only its size and count go.
+ let one = tree.report(&d.join("b"), 10).unwrap().top_files[0].clone();
+ assert!(tree.remove(&one.path));
+ let b = tree.report(&d.join("b"), 10).unwrap();
+ assert_eq!(b.total_files, 0);
+ assert_eq!(b.total_size, 0);
+ assert_eq!(tree.root_report(10).total_size, after.total_size - one.size);
+
+ // Unknown paths and the root are refused.
+ assert!(!tree.remove(&d.join("a/x")));
+ assert!(!tree.remove(&d.join("b/missing.bin")));
+ assert!(!tree.remove(&d));
+ assert!(!tree.remove(Path::new("/")));
+
+ // Disk space can be refreshed in place.
+ tree.refresh_disk_space();
+ assert_eq!(tree.root_report(1).disk_total, before.disk_total);
+
+ std::fs::remove_dir_all(&d).ok();
+ }
+
+ #[test]
+ fn cancelled_walk_is_flagged() {
+ let d = tmpdir("cancel");
+ write(&d.join("a/1.bin"), &[0u8; 10]);
+ let p = Progress::default();
+ p.cancel();
+ let tree = ScanTree::build(&d, &Options::default(), &p);
+ assert!(tree.cancelled());
+ let r = tree.root_report(10);
+ assert!(r.cancelled);
+ assert_eq!(r.total_files, 0);
+ let ok = ScanTree::build(&d, &Options::default(), &Progress::default());
+ assert!(!ok.root_report(10).cancelled);
+ assert_eq!(ok.root_report(10).total_files, 1);
+ assert_eq!(logical_total(&d), 10);
+ std::fs::remove_dir_all(&d).ok();
+ }
+
+ #[test]
+ fn missing_ancestors_are_created_on_demand() {
+ let d = tmpdir("ensure");
+ let mut tree = ScanTree::build(&d, &Options::default(), &Progress::default());
+ let deep = d.join("p/q/r");
+ let id = tree.ensure_dir(&deep);
+ assert_eq!(tree.nodes[id].path, deep);
+ assert!(tree.contains(&d.join("p")));
+ assert!(tree.contains(&d.join("p/q")));
+ assert_eq!(tree.ensure_dir(&deep), id);
+ std::fs::remove_dir_all(&d).ok();
+ }
+}
diff --git a/gui/src-tauri/Cargo.lock b/gui/src-tauri/Cargo.lock
index 33d2ba8..3b1201d 100644
--- a/gui/src-tauri/Cargo.lock
+++ b/gui/src-tauri/Cargo.lock
@@ -673,6 +673,7 @@ dependencies = [
"rayon",
"same-file",
"serde",
+ "serde_json",
"trash",
]
diff --git a/gui/src-tauri/src/main.rs b/gui/src-tauri/src/main.rs
index 9a75b1a..ec5bb8d 100644
--- a/gui/src-tauri/src/main.rs
+++ b/gui/src-tauri/src/main.rs
@@ -1,21 +1,34 @@
// Hide the console window on Windows release builds.
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use diskghost_core::{
- find_duplicates_with_progress, reclaim, remove_path, scan_with_progress, DupGroup, Options,
- Progress, ReclaimAction, ReclaimReport, RemoveMode, RemoveReport, ScanReport,
+ diff, find_duplicates_with_progress, reclaim, remove_path, DiffReport, DupGroup, Options,
+ Progress, ReclaimAction, ReclaimReport, RemoveMode, RemoveReport, ScanReport, ScanTree,
+ Snapshot,
};
use serde::{Deserialize, Serialize};
use tauri::{Emitter, Manager};
use tauri_plugin_dialog::DialogExt;
-/// Holds the `Progress` of the operation currently running, so `cancel` can flag it.
+/// How many snapshots to keep per scanned root.
+const SNAPSHOTS_PER_ROOT: usize = 20;
+
+/// The tree of the last complete walk, plus the options it was walked with.
+/// Drilling into any folder below its root is served from here, no I/O.
+struct Cached {
+ tree: ScanTree,
+ opts: WalkOpts,
+}
+
+/// Holds the `Progress` of the operation currently running (so `cancel` can
+/// flag it) and the cached scan tree.
struct AppState {
current: Mutex