From 187e0494e21511859b3a3df7b68bb054f703fbc9 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 11 Sep 2026 01:53:13 +0300 Subject: [PATCH 1/5] feat(cli): file selection, verbosity, exit codes Remaining upstream v1.5.7 command-line surface outside multi-threading: - file selection: -r, --filelist, --output-dir-flat, --output-dir-mirror, --exclude-compressed; symbolic links skipped unless -f, as upstream does - ZSTD_CLEVEL sets the default level and ZSTD_NBTHREADS is validated, both read the way upstream reads them - -q / -v display levels with upstream's result summaries, "zstd: ..." error framing, per-input failure continuation, exit status 1 when any input failed and 2 on SIGINT, which also removes the partial output file - wire-format switches wired to the encoder: --[no-]check (also skips verification when decoding), --[no-]content-size, --no-dictID; --[no-]pass-through with the zstdcat / -dcf default; zstdcat and zcat take upstream's full preset (force, stdout, pass-through, quiet) - several inputs into one -o / -c are concatenated after upstream's warning and prompt; an existing output is asked about without -f and refused under -q; valued long options take the next argument as well as =value - -l columns match upstream (DictID moves to -lv, sizes in upstream's scaled layout), .tzst and .zstd suffixes, -h / -H split, -qV bare version Part of #128 --- README.md | 35 +- zstd/src/bin/structured-zstd/display.rs | 186 ++ zstd/src/bin/structured-zstd/display/tests.rs | 93 + zstd/src/bin/structured-zstd/inputs.rs | 347 +++ zstd/src/bin/structured-zstd/inputs/tests.rs | 295 +++ zstd/src/bin/structured-zstd/interrupt.rs | 113 + .../bin/structured-zstd/interrupt/tests.rs | 43 + zstd/src/bin/structured-zstd/main.rs | 2075 +++++++++++++---- zstd/src/bin/structured-zstd/progress.rs | 156 +- .../src/bin/structured-zstd/progress/tests.rs | 99 +- zstd/src/bin/structured-zstd/tests.rs | 967 ++++++-- 11 files changed, 3692 insertions(+), 717 deletions(-) create mode 100644 zstd/src/bin/structured-zstd/display.rs create mode 100644 zstd/src/bin/structured-zstd/display/tests.rs create mode 100644 zstd/src/bin/structured-zstd/inputs.rs create mode 100644 zstd/src/bin/structured-zstd/inputs/tests.rs create mode 100644 zstd/src/bin/structured-zstd/interrupt.rs create mode 100644 zstd/src/bin/structured-zstd/interrupt/tests.rs diff --git a/README.md b/README.md index 8fcc4d500..dabd3da0b 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,27 @@ in nothing extra. The binary speaks the upstream `zstd` command line: levels (`-1`..`-19`, `--ultra` for `-20`..`-22`, `--fast[=N]`), -`-d`, `-c`, `-o`, `-t`, `-l`, `-D`, `--train`, `-b`, and the usual -`-f`/`-k`/`--rm` file handling. - -Flags that only steer how the work is done (`-T`, `-B`, `--adapt`, -`--[no-]progress`, …) are accepted and ignored — their values are still -validated, so a typo is an error rather than silence. +`-d`, `-c`, `-o`, `-t`, `-l` (`-lv` for the per-archive block), `-D`, +`--train`, `-b`, the usual `-f`/`-k`/`--rm` file handling, the `-q`/`-v` +display levels, and file selection with `-r`, `--filelist`, +`--output-dir-flat` and `--output-dir-mirror`. `ZSTD_CLEVEL` sets the default +level and `ZSTD_NBTHREADS` is validated, both read as upstream reads them. +Several inputs into one `-o` or `-c` are concatenated after upstream's +warning, an existing output is asked about unless `-f` is given (and refused +under `-q`, where nothing can be asked), one failing input does not stop the +others, and the exit status is 1 when any input failed and 2 on an interrupt, +which also removes the partial output. + +The wire-format switches take effect: `--[no-]check` (`--no-check` also skips +checksum verification when decoding), `--[no-]content-size` and `--no-dictID`. +`--[no-]pass-through` copies non-zstd input through unchanged when +decompressing, on by default for `zstdcat` and `zstd -dcf` as upstream has it, +and `--exclude-compressed` skips inputs whose extension names an +already-compressed format. + +Flags that only steer how the work is done (`-T`, `-B`, `--adapt`, ...) are +accepted and ignored; their values are still validated, so a typo is an error +rather than silence. `--target-compressed-block-size` does take effect: it bounds what goes into a block, so blocks flush sooner. `--long` means `--long=27`, as upstream documents, and is capped there: a larger window would produce frames this @@ -69,9 +84,8 @@ source can fill, so a small file compressed with `--long` does not ask its decoders to reserve 128 MiB. Flags that would change the result are refused instead: `--format=` for -anything but zstd, `--patch-from`, `--rsyncable`, `--no-check`, -`--[no-]compress-literals`, and the not-yet-implemented `--pass-through` / -`--exclude-compressed`. `-M` is treated as the safety promise it is: on the +anything but zstd, `--patch-from`, `--rsyncable` and +`--[no-]compress-literals`. `-M` is treated as the safety promise it is: on the runs that decode, a limit covering the 128 MiB window, the decoder's buffers and the `-D` dictionary is kept and a tighter one is refused rather than ignored. Compressing, listing and training allocate no decoder, so the flag is @@ -90,7 +104,8 @@ familiar names works: ```bash ln -s "$(command -v structured-zstd)" ~/.local/bin/unzstd # defaults to -d -ln -s "$(command -v structured-zstd)" ~/.local/bin/zstdcat # defaults to -d -c +ln -s "$(command -v structured-zstd)" ~/.local/bin/zstdcat # -d -c -f, pass-through, quiet +ln -s "$(command -v structured-zstd)" ~/.local/bin/zstdmt # compresses like zstd ``` Distributions should register it with their alternatives mechanism rather than diff --git a/zstd/src/bin/structured-zstd/display.rs b/zstd/src/bin/structured-zstd/display.rs new file mode 100644 index 000000000..2d9f444c0 --- /dev/null +++ b/zstd/src/bin/structured-zstd/display.rs @@ -0,0 +1,186 @@ +//! What the tool says on stderr, and how loudly. +//! +//! Mirrors the reference command's display levels: `0` says nothing, `1` +//! reports errors, `2` (the default) adds the result summary, warnings and +//! interactive prompts, `3` adds progress, `4` adds information. `-v` raises +//! the level and `-q` lowers it, so `-qq` silences errors as well. + +use std::fmt; +use std::io::{BufRead, Write}; + +/// The level a run starts at, before `-q` / `-v` move it. +pub const DEFAULT_LEVEL: i32 = 2; + +/// Whether the progress counter is drawn (`--progress` / `--no-progress`). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Progress { + /// Drawn when stderr is a terminal and the level allows it. + Auto, + /// Drawn regardless of where stderr goes. + Always, + /// Never drawn; every other message is unaffected. + Never, +} + +impl Progress { + /// Whether a run at `verbosity` draws the counter, given where stderr goes. + pub fn shown(self, verbosity: i32, stderr_is_terminal: bool) -> bool { + match self { + Self::Always => true, + Self::Never => false, + Self::Auto => verbosity >= DEFAULT_LEVEL && stderr_is_terminal, + } + } +} + +/// A byte count scaled to the unit the reference command prints it in. +/// +/// Scaled in powers of two, with the precision chosen from the magnitude of +/// the scaled value: three decimals below one, two below ten, one below a +/// hundred, none above that or when the value is a whole number of units. +/// `verbose` keeps the raw byte count instead, as `-vv` does. +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct HumanSize { + value: f64, + precision: usize, + suffix: &'static str, +} + +impl HumanSize { + /// Scale `bytes` for display. + pub fn new(bytes: u64, verbose: bool) -> Self { + if verbose { + // Past the integral precision of a double the count itself is not + // representable, so it is scaled once and rounded well up. + return if bytes >= 1 << 53 { + Self { + value: bytes as f64 / (1u64 << 20) as f64, + precision: 2, + suffix: " MiB", + } + } else { + Self { + value: bytes as f64, + precision: 0, + suffix: " B", + } + }; + } + const UNITS: [(u32, &str); 6] = [ + (60, " EiB"), + (50, " PiB"), + (40, " TiB"), + (30, " GiB"), + (20, " MiB"), + (10, " KiB"), + ]; + let (value, suffix) = UNITS + .iter() + .find(|(shift, _)| bytes >= 1u64 << shift) + .map_or((bytes as f64, " B"), |(shift, suffix)| { + (bytes as f64 / (1u64 << shift) as f64, *suffix) + }); + let precision = if value >= 100.0 || value as u64 as f64 == value { + 0 + } else if value >= 10.0 { + 1 + } else if value > 1.0 { + 2 + } else { + 3 + }; + Self { + value, + precision, + suffix, + } + } + + /// The scaled value. + #[cfg(test)] + pub fn value(&self) -> f64 { + self.value + } + + /// Decimals the scaled value is shown with. + #[cfg(test)] + pub fn precision(&self) -> usize { + self.precision + } + + /// The unit, with its leading space (`" KiB"`). + #[cfg(test)] + pub fn suffix(&self) -> &'static str { + self.suffix + } + + /// The value in a column `width` wide and the unit in one four wide: the + /// reference command's `%6.*f%4s` layout for `-l` rows and multi-file + /// summaries, where `" B"` is padded out to the width of `" KiB"`. + pub fn columns(&self, width: usize) -> String { + format!( + "{:>width$.prec$}{:>4}", + self.value, + self.suffix, + width = width, + prec = self.precision + ) + } +} + +impl fmt::Display for HumanSize { + /// The value at its precision, then the unit; a field width applies to the + /// value alone, the way the reference command's `%6.*f%s` lays it out. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match f.width() { + Some(width) => write!( + f, + "{:>width$.prec$}{}", + self.value, + self.suffix, + width = width, + prec = self.precision + ), + None => write!( + f, + "{:.prec$}{}", + self.value, + self.suffix, + prec = self.precision + ), + } + } +} + +/// Ask the user a yes/no question on stderr and read the answer from `input`. +/// +/// Returns `true` only for an answer starting with `y` or `Y`. The reference +/// command refuses to ask at all when stdin is one of the inputs: the answer +/// would be read out of the data. `abort_message` is printed on a refusal so +/// the caller can say what did not happen. +pub fn confirm( + prompt: &str, + abort_message: &str, + stdin_is_input: bool, + input: &mut impl BufRead, +) -> bool { + let mut stderr = std::io::stderr().lock(); + if stdin_is_input { + let _ = writeln!(stderr, "stdin is an input - not proceeding."); + return false; + } + let _ = write!(stderr, "{prompt}"); + let _ = stderr.flush(); + let mut answer = String::new(); + // The first character decides, as the reference command's `getchar` does: + // a space before the `y` is not a yes. + let accepted = + input.read_line(&mut answer).is_ok() && matches!(answer.chars().next(), Some('y' | 'Y')); + if !accepted { + let _ = writeln!(stderr, "{abort_message}"); + } + accepted +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/bin/structured-zstd/display/tests.rs b/zstd/src/bin/structured-zstd/display/tests.rs new file mode 100644 index 000000000..bcc8f03c8 --- /dev/null +++ b/zstd/src/bin/structured-zstd/display/tests.rs @@ -0,0 +1,93 @@ +use super::{DEFAULT_LEVEL, HumanSize, Progress, confirm}; + +/// The reference command scales sizes in powers of two and picks the number +/// of decimals from the scaled value, so `-l` columns and result summaries +/// line up with what its own output shows. A whole number of units drops the +/// decimals entirely; anything else keeps three significant figures. +#[test] +fn sizes_scale_and_round_the_way_the_reference_prints_them() { + assert_eq!(HumanSize::new(0, false).to_string(), "0 B"); + assert_eq!(HumanSize::new(100, false).to_string(), "100 B"); + assert_eq!(HumanSize::new(1023, false).to_string(), "1023 B"); + assert_eq!(HumanSize::new(1024, false).to_string(), "1 KiB"); + assert_eq!(HumanSize::new(1025, false).to_string(), "1.00 KiB"); + assert_eq!(HumanSize::new(1536, false).to_string(), "1.50 KiB"); + assert_eq!( + HumanSize::new(10 * 1024 + 512, false).to_string(), + "10.5 KiB" + ); + assert_eq!(HumanSize::new(12 * 1024, false).to_string(), "12 KiB"); + assert_eq!(HumanSize::new(7 << 20, false).to_string(), "7 MiB"); + assert_eq!(HumanSize::new(123 << 30, false).to_string(), "123 GiB"); + assert_eq!(HumanSize::new(150 << 30, false).to_string(), "150 GiB"); + assert_eq!(HumanSize::new(1 << 40, false).to_string(), "1 TiB"); +} + +/// Below one unit the value keeps three decimals: a file just under a +/// kibibyte scaled to KiB would otherwise print as `1 KiB`, which it is not. +#[test] +fn a_fraction_of_a_unit_keeps_three_decimals() { + let size = HumanSize::new((1 << 20) + 1, false); + assert_eq!(size.precision(), 2); + let just_over_a_unit = HumanSize::new(1024 + 1, false); + assert_eq!(just_over_a_unit.to_string(), "1.00 KiB"); + // Only values strictly above one unit reach the two-decimal branch; a + // value scaled to exactly one unit is whole and prints without decimals. + assert_eq!(HumanSize::new(1 << 30, false).precision(), 0); +} + +/// `-vv` keeps the raw byte count: a summary being pasted into a report wants +/// the exact figure, not a rounded one. +#[test] +fn verbose_sizes_are_raw_byte_counts() { + assert_eq!(HumanSize::new(1536, true).to_string(), "1536 B"); + assert_eq!(HumanSize::new(0, true).to_string(), "0 B"); + // Past the integral precision of a double the count itself cannot be + // shown exactly, so it is scaled once. + assert_eq!(HumanSize::new(1 << 53, true).suffix(), " MiB"); +} + +/// A field width lays out the number and leaves the unit attached: that is how +/// the reference command's `%6.*f%4s` keeps its columns aligned. +#[test] +fn a_field_width_pads_the_number_not_the_unit() { + assert_eq!(format!("{:>6}", HumanSize::new(100, false)), " 100 B"); + assert_eq!(format!("{:>6}", HumanSize::new(1536, false)), " 1.50 KiB"); + assert_eq!(HumanSize::new(1536, false).value(), 1.5); +} + +/// The progress counter is drawn for a person watching a terminal, and only +/// then unless asked for outright: piped stderr must stay clean of carriage +/// returns, and `-q` says not to decorate. +#[test] +fn progress_is_drawn_for_a_terminal_at_the_default_level_unless_forced() { + assert!(Progress::Auto.shown(DEFAULT_LEVEL, true)); + assert!(!Progress::Auto.shown(DEFAULT_LEVEL, false)); + assert!(!Progress::Auto.shown(DEFAULT_LEVEL - 1, true)); + assert!(Progress::Always.shown(0, false)); + assert!(!Progress::Never.shown(4, true)); +} + +/// A prompt is answered by the first character of the line, `y` or `Y`, and +/// nothing else: an empty answer, a refusal, or the end of input all decline. +#[test] +fn only_a_yes_confirms() { + assert!(confirm("go? ", "no", false, &mut &b"y\n"[..])); + assert!(confirm("go? ", "no", false, &mut &b"Y\n"[..])); + assert!(confirm("go? ", "no", false, &mut &b"yes please\n"[..])); + assert!(!confirm("go? ", "no", false, &mut &b"n\n"[..])); + assert!(!confirm("go? ", "no", false, &mut &b"\n"[..])); + assert!(!confirm("go? ", "no", false, &mut &b""[..])); + // The first character decides, as the reference command reads it. + assert!(!confirm("go? ", "no", false, &mut &b" y\n"[..])); +} + +/// When stdin is one of the inputs, the answer would be read out of the data +/// being compressed. The reference command refuses to ask; so does this one, +/// and it must not consume a byte of the input while declining. +#[test] +fn no_prompt_is_read_from_an_input_stream() { + let mut input = &b"y\n"[..]; + assert!(!confirm("go? ", "no", true, &mut input)); + assert_eq!(input, b"y\n", "the input must be left untouched"); +} diff --git a/zstd/src/bin/structured-zstd/inputs.rs b/zstd/src/bin/structured-zstd/inputs.rs new file mode 100644 index 000000000..3e7f2a025 --- /dev/null +++ b/zstd/src/bin/structured-zstd/inputs.rs @@ -0,0 +1,347 @@ +//! Which files a run works on, and where their outputs land. +//! +//! The command line names inputs; `--filelist` adds more from a file; `-r` +//! turns the directories among them into the files underneath. The output of +//! each is then placed next to it, or under `--output-dir-flat` / +//! `--output-dir-mirror`. The reference command does this in `zstdcli.c` and +//! `util.c`, and the order of the steps is kept: symbolic links are dropped +//! from the NAMED inputs before the file lists are merged, and directories are +//! expanded after. + +use std::ffi::OsString; +use std::fs; +use std::io::{BufRead, BufReader}; +use std::path::{Component, Path, PathBuf}; + +use super::{Result, WrapErr}; + +/// Largest `--filelist` the reference command reads (50 MiB). +pub const FILELIST_MAX_BYTES: u64 = 50 << 20; + +/// The inputs a run ended up with. +#[derive(Debug)] +pub struct Selection { + /// Every file to process, in command-line order, directories expanded. + pub files: Vec, + /// How many inputs were named before directories were expanded. A run + /// that named some and ended up with none was pointed at empty + /// directories, which is not a request to read stdin. + pub named: usize, +} + +/// Resolve the command line's inputs to the files a run processes. +/// +/// `follow_links` is `-f`: without it a symbolic link is skipped with a +/// warning, both on the command line and inside a directory walked by `-r`. +/// A FIFO reached through a link is kept, since the link is how a named pipe +/// is usually handed over. +pub fn select_inputs( + named: Vec, + filelists: &[PathBuf], + recursive: bool, + follow_links: bool, + verbosity: i32, +) -> Result { + let mut files = Vec::with_capacity(named.len()); + let named_count = named.len(); + for input in named { + if !follow_links && input != Path::new("-") && is_symlink(&input) && !is_fifo(&input) { + display!( + verbosity, + 2, + "Warning : {} is a symbolic link, ignoring", + input.display() + ); + continue; + } + files.push(input); + } + if files.is_empty() && named_count > 0 { + bail!("every named input is a symbolic link; pass -f to follow them"); + } + for list in filelists { + files.extend(read_filelist(list)?); + } + let named = files.len(); + if recursive { + let mut expanded = Vec::with_capacity(files.len()); + for input in files { + if fs::metadata(&input).is_ok_and(|m| m.is_dir()) { + walk_directory(&input, follow_links, verbosity, &mut expanded); + } else { + expanded.push(input); + } + } + files = expanded; + } + Ok(Selection { files, named }) +} + +/// Whether `path` itself is a symbolic link, whatever it points at. +fn is_symlink(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok_and(|m| m.file_type().is_symlink()) +} + +/// Whether `path`, links followed, is a named pipe. +fn is_fifo(path: &Path) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::FileTypeExt; + fs::metadata(path).is_ok_and(|m| m.file_type().is_fifo()) + } + #[cfg(not(unix))] + { + let _ = path; + false + } +} + +/// Read a `--filelist`: one file name per line, blank lines skipped. +/// +/// The list has to be a regular file of bounded size, as the reference command +/// requires; a name that is not there or a directory is an error rather than +/// an empty list, since a script that mistyped the list name would otherwise +/// silently process nothing. +fn read_filelist(list: &Path) -> Result> { + let metadata = + fs::metadata(list).wrap_err_with(|| format!("error reading {}", list.display()))?; + if !metadata.is_file() { + bail!("error reading {}: not a regular file", list.display()); + } + if metadata.len() > FILELIST_MAX_BYTES { + bail!( + "error reading {}: file list is larger than {} bytes", + list.display(), + FILELIST_MAX_BYTES + ); + } + let file = + fs::File::open(list).wrap_err_with(|| format!("error reading {}", list.display()))?; + let mut names = Vec::new(); + // Lines are bytes, like the names in them: a filename need not be UTF-8, + // and reading the list as text would reject or rename such an entry. + let mut reader = BufReader::new(file); + let mut line = Vec::new(); + loop { + line.clear(); + let read = reader + .read_until(b'\n', &mut line) + .wrap_err_with(|| format!("error reading {}", list.display()))?; + if read == 0 { + break; + } + if line.last() == Some(&b'\n') { + line.pop(); + } + if line.last() == Some(&b'\r') { + line.pop(); + } + if line.is_empty() { + continue; + } + names.push(bytes_to_path(&line)); + } + Ok(names) +} + +#[cfg(unix)] +fn bytes_to_path(bytes: &[u8]) -> PathBuf { + use std::os::unix::ffi::OsStrExt; + PathBuf::from(std::ffi::OsStr::from_bytes(bytes)) +} + +#[cfg(not(unix))] +fn bytes_to_path(bytes: &[u8]) -> PathBuf { + PathBuf::from(String::from_utf8_lossy(bytes).into_owned()) +} + +/// Append every file under `dir` to `out`, depth first. +/// +/// Entries are taken in name order so two runs over one tree process it the +/// same way; the reference command takes them in directory order, which the +/// filesystem does not promise to keep. A directory that cannot be read is +/// reported and contributes nothing, as there too. +fn walk_directory(dir: &Path, follow_links: bool, verbosity: i32, out: &mut Vec) { + let entries = match fs::read_dir(dir) { + Ok(entries) => entries, + Err(err) => { + display!( + verbosity, + 1, + "Cannot open directory '{}': {err}", + dir.display() + ); + return; + } + }; + let mut names: Vec = Vec::new(); + for entry in entries { + match entry { + Ok(entry) => names.push(entry.file_name()), + Err(err) => { + display!(verbosity, 1, "readdir({}) error: {err}", dir.display()); + return; + } + } + } + names.sort(); + for name in names { + let path = dir.join(name); + if !follow_links && is_symlink(&path) { + display!( + verbosity, + 2, + "Warning : {} is a symbolic link, ignoring", + path.display() + ); + continue; + } + if fs::metadata(&path).is_ok_and(|m| m.is_dir()) { + walk_directory(&path, follow_links, verbosity, out); + } else { + out.push(path); + } + } +} + +/// Where `--output-dir-flat DIR` puts the output of `src`: under `DIR`, by +/// the source's own file name. +pub fn flat_output_path(src: &Path, dir: &Path) -> PathBuf { + dir.join(src.file_name().unwrap_or(src.as_os_str())) +} + +/// The source path with the parts `--output-dir-mirror` drops: a leading +/// `./`, the root, and a drive prefix. `None` when a `..` component is in it, +/// since mirroring that would climb out of the output tree. +fn mirrored_relative(src: &Path) -> Option { + let mut relative = PathBuf::new(); + for component in src.components() { + match component { + Component::Prefix(_) | Component::RootDir | Component::CurDir => {} + Component::ParentDir => return None, + Component::Normal(part) => relative.push(part), + } + } + Some(relative) +} + +/// The directory `--output-dir-mirror ROOT` puts the output of `src` in: the +/// source's own directory, replayed under `ROOT`. `None` for a source the +/// reference command refuses to mirror (one with `..` in its path). +pub fn mirrored_output_dir(src: &Path, root: &Path) -> Option { + let relative = mirrored_relative(src)?; + Some(match relative.parent() { + Some(parent) => root.join(parent), + None => root.to_path_buf(), + }) +} + +/// Create the directory chain under `root` that mirrors the directory of +/// `src`, each level with the permissions of the source directory it mirrors. +/// +/// `root` itself is created with the default permissions when it is missing. +/// A source whose path cannot be mirrored (see [`mirrored_output_dir`]) is +/// left for the caller to report. +pub fn create_mirrored_dirs(src: &Path, root: &Path) -> Result<()> { + let Some(relative) = mirrored_relative(src) else { + return Ok(()); + }; + create_dir_if_missing(root, None) + .wrap_err_with(|| format!("failed to create DIR {}", root.display()))?; + let Some(parent) = relative.parent() else { + return Ok(()); + }; + // The source directory that each mirrored level stands for, rebuilt from + // the source path as given so its permissions can be read: for + // `/var/tmp/abc` the levels are `/var` and `/var/tmp`. + let stripped: usize = src + .components() + .count() + .saturating_sub(relative.components().count()); + let source_root: PathBuf = src.components().take(stripped).collect(); + let mut source_level = source_root; + let mut destination = root.to_path_buf(); + for level in parent.components() { + source_level.push(level); + destination.push(level); + let mode = fs::metadata(&source_level) + .ok() + .map(|metadata| metadata.permissions()); + create_dir_if_missing(&destination, mode) + .wrap_err_with(|| format!("failed to create DIR {}", destination.display()))?; + } + Ok(()) +} + +/// Create `dir` unless it is already there, with `permissions` when given. +fn create_dir_if_missing(dir: &Path, permissions: Option) -> std::io::Result<()> { + match fs::metadata(dir) { + Ok(existing) if existing.is_dir() => return Ok(()), + Ok(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "exists and is not a directory", + )); + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err), + } + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + if let Some(permissions) = &permissions { + use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; + builder.mode(permissions.mode() & 0o7777); + } + #[cfg(not(unix))] + let _ = permissions; + builder.create(dir) +} + +/// The file names that more than one of `files` share. +/// +/// Under `--output-dir-flat` two such inputs land on one output, the second +/// replacing the first; the reference command warns about it after the run. +pub fn shared_file_names(files: &[PathBuf]) -> Vec { + let mut names: Vec<&std::ffi::OsStr> = + files.iter().filter_map(|file| file.file_name()).collect(); + names.sort_unstable(); + let mut shared = Vec::new(); + for pair in names.windows(2) { + if pair[0] == pair[1] && shared.last() != Some(&pair[0].to_os_string()) { + shared.push(pair[0].to_os_string()); + } + } + shared +} + +/// Extensions the reference command treats as already compressed under +/// `--exclude-compressed` (`fileio.c`, `compressedFileExtensions`). +const COMPRESSED_EXTENSIONS: &[&str] = &[ + ".zst", ".tzst", ".gz", ".tgz", ".lzma", ".xz", ".txz", ".lz4", ".tlz4", ".7z", ".aa3", ".aac", + ".aar", ".ace", ".alac", ".ape", ".apk", ".apng", ".arc", ".archive", ".arj", ".ark", ".asf", + ".avi", ".avif", ".ba", ".br", ".bz2", ".cab", ".cdx", ".chm", ".cr2", ".divx", ".dmg", ".dng", + ".docm", ".docx", ".dotm", ".dotx", ".dsft", ".ear", ".eftx", ".emz", ".eot", ".epub", ".f4v", + ".flac", ".flv", ".gho", ".gif", ".gifv", ".gnp", ".iso", ".jar", ".jpeg", ".jpg", ".jxl", + ".lz", ".lzh", ".m4a", ".m4v", ".mkv", ".mov", ".mp2", ".mp3", ".mp4", ".mpa", ".mpc", ".mpe", + ".mpeg", ".mpg", ".mpl", ".mpv", ".msi", ".odp", ".ods", ".odt", ".ogg", ".ogv", ".otp", + ".ots", ".ott", ".pea", ".png", ".pptx", ".qt", ".rar", ".s7z", ".sfx", ".sit", ".sitx", + ".sqx", ".svgz", ".swf", ".tbz2", ".tib", ".tlz", ".vob", ".war", ".webm", ".webp", ".wma", + ".wmv", ".woff", ".woff2", ".wvl", ".xlsx", ".xpi", ".xps", ".zip", ".zipx", ".zoo", ".zpaq", +]; + +/// Whether `path` carries an extension of an already-compressed format. +/// +/// The extension is the file name's last dot onward, compared exactly: a +/// dotfile has none, and `.GZ` is not `.gz`, as in the reference command. +pub fn has_compressed_extension(path: &Path) -> bool { + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + return false; + }; + match name.rfind('.') { + Some(0) | None => false, + Some(at) => COMPRESSED_EXTENSIONS.contains(&&name[at..]), + } +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/bin/structured-zstd/inputs/tests.rs b/zstd/src/bin/structured-zstd/inputs/tests.rs new file mode 100644 index 000000000..23c46de62 --- /dev/null +++ b/zstd/src/bin/structured-zstd/inputs/tests.rs @@ -0,0 +1,295 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use super::{ + create_mirrored_dirs, flat_output_path, has_compressed_extension, mirrored_output_dir, + select_inputs, shared_file_names, +}; + +/// A scratch directory unique to the test, removed when dropped. +struct Scratch(PathBuf); + +impl Scratch { + fn new(tag: &str) -> Self { + let dir = std::env::temp_dir().join(format!("szstd-inputs-{tag}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + Self(dir) + } + + fn path(&self) -> &Path { + &self.0 + } + + fn file(&self, relative: &str) -> PathBuf { + let path = self.0.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, relative.as_bytes()).unwrap(); + path + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// `-r` replaces a directory by the files beneath it, all the way down, in a +/// fixed order: a script that compresses a tree gets the same sequence of +/// outputs and messages every time it runs. +#[test] +fn recursion_walks_directories_depth_first_in_name_order() { + let scratch = Scratch::new("walk"); + let b = scratch.file("b.txt"); + let a = scratch.file("a.txt"); + let nested = scratch.file("sub/deeper/c.txt"); + let sibling = scratch.file("sub/d.txt"); + + let selection = select_inputs(vec![scratch.path().to_path_buf()], &[], true, false, 0) + .expect("a readable tree expands"); + // Name order at each level: `d.txt` sorts before the `deeper` directory + // (`.` before `e`), so the sibling file comes before the nested one. + assert_eq!(selection.files, vec![a, b, sibling, nested]); + assert_eq!(selection.named, 1, "one input was named before expansion"); +} + +/// Without `-r` a directory stays a directory: the caller reports it as one +/// rather than silently walking into it, which is what the reference command +/// does ("is a directory -- ignored"). +#[test] +fn without_recursion_a_directory_is_kept_for_the_caller_to_refuse() { + let scratch = Scratch::new("nowalk"); + scratch.file("inside.txt"); + let selection = select_inputs(vec![scratch.path().to_path_buf()], &[], false, false, 0) + .expect("selection itself does not fail"); + assert_eq!(selection.files, vec![scratch.path().to_path_buf()]); +} + +/// A symbolic link on the command line is skipped unless `-f` follows links: +/// compressing through a link writes the archive beside the link and, with +/// `--rm`, deletes the link rather than the file. When every input was a link +/// the run has nothing left and says so instead of falling back to stdin. +#[cfg(unix)] +#[test] +fn named_symlinks_are_skipped_unless_links_are_followed() { + let scratch = Scratch::new("links"); + let target = scratch.file("target.txt"); + let link = scratch.path().join("link.txt"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let kept = select_inputs(vec![target.clone(), link.clone()], &[], false, false, 0).unwrap(); + assert_eq!(kept.files, vec![target.clone()], "the link is dropped"); + + let followed = select_inputs(vec![target.clone(), link.clone()], &[], false, true, 0).unwrap(); + assert_eq!(followed.files, vec![target, link.clone()], "-f keeps it"); + + let err = select_inputs(vec![link], &[], false, false, 0) + .expect_err("a run whose every input was a link has nothing to do") + .to_string(); + assert!(err.contains("symbolic link"), "the refusal says why: {err}"); +} + +/// The same rule inside a walked tree: a link found by `-r` is skipped without +/// `-f`, so a tree with a link back into itself does not loop, and a link to a +/// directory is not descended into. +#[cfg(unix)] +#[test] +fn symlinks_inside_a_walked_tree_are_skipped_unless_followed() { + let scratch = Scratch::new("treelinks"); + let real = scratch.file("dir/real.txt"); + let other = scratch.file("elsewhere/other.txt"); + std::os::unix::fs::symlink(&other, scratch.path().join("dir/link.txt")).unwrap(); + + let dir = scratch.path().join("dir"); + let skipped = select_inputs(vec![dir.clone()], &[], true, false, 0).unwrap(); + assert_eq!(skipped.files, vec![real.clone()]); + + let followed = select_inputs(vec![dir.clone()], &[], true, true, 0).unwrap(); + assert_eq!(followed.files, vec![dir.join("link.txt"), real]); +} + +/// `--filelist` names inputs one per line, in the shape `ls` prints them. +/// Blank lines carry no name and are skipped; a Windows line ending is +/// stripped like a Unix one, since the list may have been written elsewhere. +#[test] +fn a_filelist_adds_one_input_per_line() { + let scratch = Scratch::new("filelist"); + let list = scratch.path().join("list.txt"); + fs::write(&list, "first.bin\n\nsecond.bin\r\nthird.bin").unwrap(); + + let selection = + select_inputs(vec![PathBuf::from("argv.bin")], &[list], false, false, 0).unwrap(); + assert_eq!( + selection.files, + vec![ + PathBuf::from("argv.bin"), + PathBuf::from("first.bin"), + PathBuf::from("second.bin"), + PathBuf::from("third.bin"), + ], + "command-line inputs come first, then the list, blank lines dropped" + ); + assert_eq!(selection.named, 4, "list entries count as named inputs"); +} + +/// A list that is not there is an error, not an empty list: a mistyped +/// `--filelist` would otherwise process nothing and report success. +#[test] +fn a_missing_or_irregular_filelist_is_an_error() { + let scratch = Scratch::new("badlist"); + let missing = scratch.path().join("nope.txt"); + let err = select_inputs(Vec::new(), &[missing], false, false, 0) + .expect_err("a missing list cannot be read") + .to_string(); + assert!(err.contains("error reading"), "{err}"); + + let err = select_inputs( + Vec::new(), + std::slice::from_ref(&scratch.path().to_path_buf()), + false, + false, + 0, + ) + .expect_err("a directory is not a list") + .to_string(); + assert!(err.contains("not a regular file"), "{err}"); +} + +/// Names from a list are expanded by `-r` like names from the command line: +/// the list is just another way of typing them. +#[test] +fn filelist_entries_are_expanded_recursively_too() { + let scratch = Scratch::new("listwalk"); + let inside = scratch.file("tree/leaf.txt"); + let list = scratch.path().join("list.txt"); + fs::write( + &list, + format!("{}\n", scratch.path().join("tree").display()), + ) + .unwrap(); + + let selection = select_inputs(Vec::new(), &[list], true, false, 0).unwrap(); + assert_eq!(selection.files, vec![inside]); + assert_eq!(selection.named, 1); +} + +/// `--output-dir-flat` drops the source's directory and keeps its name. +#[test] +fn flat_output_keeps_only_the_file_name() { + assert_eq!( + flat_output_path(Path::new("a/b/c.txt"), Path::new("out")), + PathBuf::from("out/c.txt") + ); + assert_eq!( + flat_output_path(Path::new("c.txt"), Path::new("out/")), + PathBuf::from("out/c.txt") + ); +} + +/// `--output-dir-mirror` replays the source's directory under the root, with +/// a leading `./` or `/` removed so an absolute input lands inside the root +/// rather than replacing it. A path that climbs with `..` is refused, since +/// mirroring it could write outside the root. +#[test] +fn mirrored_output_replays_the_source_directory_under_the_root() { + let root = Path::new("out"); + assert_eq!( + mirrored_output_dir(Path::new("a/b/c.txt"), root), + Some(PathBuf::from("out/a/b")) + ); + assert_eq!( + mirrored_output_dir(Path::new("./x.txt"), root), + Some(PathBuf::from("out")) + ); + assert_eq!( + mirrored_output_dir(Path::new("x.txt"), root), + Some(PathBuf::from("out")) + ); + assert_eq!( + mirrored_output_dir(Path::new("/var/tmp/abc"), root), + Some(PathBuf::from("out/var/tmp")) + ); + assert_eq!(mirrored_output_dir(Path::new("../x.txt"), root), None); + assert_eq!(mirrored_output_dir(Path::new("a/../b/x.txt"), root), None); +} + +/// The mirrored directories are created before the output is written, each +/// with the permissions of the source directory it stands for, so a private +/// source tree does not become a world-readable mirror. +#[test] +fn mirrored_directories_are_created_with_the_source_permissions() { + let scratch = Scratch::new("mirror"); + let src = scratch.file("tree/inner/leaf.txt"); + let root = scratch.path().join("mirror-root"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + scratch.path().join("tree/inner"), + fs::Permissions::from_mode(0o700), + ) + .unwrap(); + } + + create_mirrored_dirs(&src, &root).expect("the chain is created"); + let expected = mirrored_output_dir(&src, &root).unwrap(); + assert!(expected.is_dir(), "{} must exist", expected.display()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(&expected).unwrap().permissions().mode() & 0o777; + assert_eq!( + mode, 0o700, + "the mirrored directory takes the source's mode" + ); + } + // Doing it again is harmless: the directories are already there. + create_mirrored_dirs(&src, &root).expect("an existing chain is fine"); +} + +/// A source that cannot be mirrored creates nothing: the caller reports it. +#[test] +fn an_unmirrorable_source_creates_no_directories() { + let scratch = Scratch::new("nomirror"); + let root = scratch.path().join("root"); + create_mirrored_dirs(Path::new("../escape.txt"), &root).expect("nothing to do is not an error"); + assert!(!root.exists(), "no root is created for a refused source"); +} + +/// Two inputs with one file name land on one output under +/// `--output-dir-flat`; the run warns about each such name once. +#[test] +fn shared_names_are_reported_once_each() { + let files = vec![ + PathBuf::from("a/data.txt"), + PathBuf::from("b/data.txt"), + PathBuf::from("c/data.txt"), + PathBuf::from("a/other.txt"), + PathBuf::from("b/other.txt"), + PathBuf::from("unique.txt"), + ]; + assert_eq!( + shared_file_names(&files), + vec![ + std::ffi::OsString::from("data.txt"), + std::ffi::OsString::from("other.txt") + ] + ); + assert!(shared_file_names(&[PathBuf::from("one")]).is_empty()); +} + +/// `--exclude-compressed` judges by the last extension, exactly as spelled: a +/// `.tar.zst` is compressed, a dotfile has no extension, and case matters as +/// it does in the reference list. +#[test] +fn compressed_extensions_are_matched_on_the_last_dot() { + assert!(has_compressed_extension(Path::new("a.gz"))); + assert!(has_compressed_extension(Path::new("dir.d/archive.tar.zst"))); + assert!(has_compressed_extension(Path::new("movie.mp4"))); + assert!(!has_compressed_extension(Path::new("a.txt"))); + assert!(!has_compressed_extension(Path::new(".gz"))); + assert!(!has_compressed_extension(Path::new("A.GZ"))); + assert!(!has_compressed_extension(Path::new("noext"))); + assert!(!has_compressed_extension(Path::new("dir.gz/plain"))); +} diff --git a/zstd/src/bin/structured-zstd/interrupt.rs b/zstd/src/bin/structured-zstd/interrupt.rs new file mode 100644 index 000000000..c4a53c2f4 --- /dev/null +++ b/zstd/src/bin/structured-zstd/interrupt.rs @@ -0,0 +1,113 @@ +//! Removes the output being written when the run is interrupted. +//! +//! A `Ctrl-C` in the middle of a file would otherwise leave the partial +//! temporary beside the source. The reference command installs a `SIGINT` +//! handler that unlinks the artefact and exits with status 2; this does the +//! same, through the C library the standard library already links, so the +//! tool takes on no dependency for it. Platforms without POSIX signals get +//! the no-op version and keep the temporary on interruption. + +#[cfg(unix)] +mod imp { + use core::ffi::{c_char, c_int, c_void}; + use std::os::unix::ffi::OsStrExt; + use std::path::Path; + use std::ptr; + use std::sync::atomic::{AtomicPtr, Ordering}; + + type Handler = extern "C" fn(c_int); + + unsafe extern "C" { + fn signal(signum: c_int, handler: Option) -> Option; + fn unlink(path: *const c_char) -> c_int; + fn write(fd: c_int, buf: *const c_void, count: usize) -> isize; + fn _exit(status: c_int) -> !; + } + + /// `SIGINT` has this number on every POSIX system. + const SIGINT: c_int = 2; + + /// Longest path the guard covers, NUL included. A longer temporary is not + /// guarded rather than truncated to a name that is not the file's. + pub const PATH_CAPACITY: usize = 4096; + + /// The guarded path as a C string, or all zeros. Written only while + /// `ARTEFACT` is null, so the handler never reads a half-written name. + static mut PATH: [u8; PATH_CAPACITY] = [0; PATH_CAPACITY]; + + /// Points into `PATH` while a file is guarded, null otherwise. + static ARTEFACT: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + + /// Async-signal-safe by construction: `unlink`, `write` and `_exit` + /// only, no allocation, no locks, no formatting. + extern "C" fn on_interrupt(_signum: c_int) { + let path = ARTEFACT.load(Ordering::SeqCst); + // SAFETY: a non-null `path` points at `PATH`, which holds a + // NUL-terminated string from the moment the pointer was published + // and is not rewritten until the pointer has been cleared. + if !path.is_null() { + unsafe { + unlink(path); + } + } + // SAFETY: plain libc calls on a valid buffer and a constant status. + unsafe { + write(2, b"\n".as_ptr().cast(), 1); + _exit(2); + } + } + + /// Remove `path` if the process is interrupted before [`clear`] is called. + pub fn guard(path: &Path) { + let bytes = path.as_os_str().as_bytes(); + if bytes.is_empty() || bytes.len() >= PATH_CAPACITY || bytes.contains(&0) { + return; + } + ARTEFACT.store(ptr::null_mut(), Ordering::SeqCst); + // SAFETY: the handler reads `PATH` only through `ARTEFACT`, which is + // null for the length of this write; raw pointer access keeps no + // reference to the static alive. + unsafe { + let buffer = (&raw mut PATH).cast::(); + ptr::copy_nonoverlapping(bytes.as_ptr(), buffer, bytes.len()); + *buffer.add(bytes.len()) = 0; + ARTEFACT.store(buffer.cast::(), Ordering::SeqCst); + signal(SIGINT, Some(on_interrupt)); + } + } + + /// Stop guarding: an interruption from here on keeps the file and takes + /// the default action. + pub fn clear() { + ARTEFACT.store(ptr::null_mut(), Ordering::SeqCst); + // SAFETY: restoring the default disposition is always valid. + unsafe { + signal(SIGINT, None); + } + } + + /// Whether a file is currently guarded (for tests). + #[cfg(test)] + pub fn is_guarded() -> bool { + !ARTEFACT.load(Ordering::SeqCst).is_null() + } +} + +#[cfg(not(unix))] +mod imp { + use std::path::Path; + + pub fn guard(_path: &Path) {} + + pub fn clear() {} + + #[cfg(test)] + pub fn is_guarded() -> bool { + false + } +} + +pub use imp::{clear, guard}; + +#[cfg(test)] +mod tests; diff --git a/zstd/src/bin/structured-zstd/interrupt/tests.rs b/zstd/src/bin/structured-zstd/interrupt/tests.rs new file mode 100644 index 000000000..6ce358e5e --- /dev/null +++ b/zstd/src/bin/structured-zstd/interrupt/tests.rs @@ -0,0 +1,43 @@ +use std::path::Path; + +use super::imp::is_guarded; +use super::{clear, guard}; + +/// The guard is a window: it opens when a temporary is being written and +/// closes when the file is in place. A guard left open past `clear` would +/// delete a finished output on the next interruption. +#[cfg(unix)] +#[test] +fn a_guard_is_set_by_guard_and_removed_by_clear() { + clear(); + assert!(!is_guarded()); + guard(Path::new("/tmp/szstd-guard-test")); + assert!(is_guarded(), "a path in range is guarded"); + clear(); + assert!(!is_guarded()); +} + +/// A path the buffer cannot hold is not guarded rather than guarded under a +/// truncated name, which would be some other file's. +#[cfg(unix)] +#[test] +fn a_path_that_does_not_fit_is_left_unguarded() { + clear(); + let long = "x".repeat(super::imp::PATH_CAPACITY); + guard(Path::new(&long)); + assert!( + !is_guarded(), + "a name that would not fit must not be guarded" + ); + guard(Path::new("")); + assert!(!is_guarded(), "nor an empty one"); + clear(); +} + +#[cfg(not(unix))] +#[test] +fn the_no_op_guard_never_reports_a_guard() { + guard(Path::new("anything")); + assert!(!is_guarded()); + clear(); +} diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index ace2bc034..84dfc036b 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -2,15 +2,16 @@ //! //! The argument model mirrors upstream zstd v1.5.7: mode + level FLAGS (not //! subcommands), `argv[0]` dispatch (`unzstd` / `zstdcat` change the default -//! mode), stdin/stdout streaming, and the conventional `-o`/`-f`/`-k`/`-D` -//! file flags. Compression/decompression run through the streaming codec, so -//! peak memory stays O(window), not O(file). - -mod progress; -use progress::{ProgressMonitor, fmt_size}; - +//! mode), stdin/stdout streaming, `-r` / `--filelist` / `--output-dir-*` file +//! selection, the `ZSTD_CLEVEL` / `ZSTD_NBTHREADS` environment, the `-q` / +//! `-v` display levels, and the conventional `-o`/`-f`/`-k`/`-D` file flags. +//! Compression/decompression run through the streaming codec, so peak memory +//! stays O(window), not O(file). Exit status is 0, 1 when any input failed, +//! and 2 when interrupted, as the reference command's is. + +use std::ffi::{OsStr, OsString}; use std::fs::{self, File, OpenOptions}; -use std::io::{self, BufRead, BufReader, ErrorKind, IsTerminal, Read, Write}; +use std::io::{self, BufReader, ErrorKind, IsTerminal, Read, Write}; use std::path::{Path, PathBuf}; use structured_zstd::encoding::CompressionLevel; @@ -56,16 +57,40 @@ impl WrapErr for core::result::Result { self.map_err(|source| eyre!("{}: {source}", msg())) } } -/// Status line to stderr. A tool this size does not need a tracing subscriber -/// to say "file -> file.zst"; keeping it a macro preserves every call site. -macro_rules! info { - ($($arg:tt)*) => { - eprintln!($($arg)*) +/// Say something on stderr when `$verbosity` reaches `$level`: `1` carries +/// errors, `2` results and warnings, `3` progress, `4` detail. A tool this +/// size does not need a tracing subscriber; a macro keeps every call site a +/// line. +macro_rules! display { + ($verbosity:expr, $level:expr, $($arg:tt)*) => { + if $verbosity >= $level { + eprintln!($($arg)*); + } }; } +mod display; +mod inputs; +mod interrupt; +mod progress; + +use display::{DEFAULT_LEVEL, HumanSize, Progress, confirm}; +use inputs::Selection; +use progress::ProgressMonitor; + const ZSTD_SUFFIX: &str = ".zst"; +/// Suffixes a decompressed name is derived from, each with what replaces it: +/// `.zst` and `.zstd` are dropped, `.tzst` becomes `.tar`, as the reference +/// command's suffix list has it. +const DECOMPRESS_SUFFIXES: [(&str, &str); 3] = [("zst", ""), ("zstd", ""), ("tzst", "tar")]; + +/// The reference command version whose command line this tool follows. +const UPSTREAM_VERSION: &str = "1.5.7"; + +/// How the reference command names stdout in a summary line. +const STDOUT_MARK: &str = "/*stdout*\\"; + /// Highest level the CLI compresses at when `--ultra` was not given (upstream /// `ZSTDCLI_CLEVEL_MAX`). Asking for more without naming `--ultra` reduces to /// this with a warning rather than failing. @@ -137,6 +162,42 @@ struct Options { /// and table sizing only; being wrong costs ratio, never correctness, so /// it must NOT reach the header. size_hint: Option, + /// Display level: `-v` raises it, `-q` lowers it (see [`display`]). + verbosity: i32, + /// Whether frames carry a content checksum (`-C` / `--[no-]check`), and + /// whether decoding verifies one. On by default, as the reference + /// command's is. + checksum: bool, + /// Whether a known input length is written into the frame header + /// (`--[no-]content-size`). + content_size_flag: bool, + /// Whether a dictionary frame records the dictionary's ID + /// (`--no-dictID`). + dict_id_flag: bool, + /// Copy input that is not a zstd stream through unchanged when + /// decompressing (`--[no-]pass-through`). `None` is the reference + /// command's default: on only when forced and writing to stdout, which + /// is what `zstdcat` does. + pass_through: Option, + /// Skip inputs whose extension says they are already compressed + /// (`--exclude-compressed`). + exclude_compressed: bool, + /// Replace directories among the inputs by the files beneath them (`-r`). + recursive: bool, + /// Process symbolic links rather than skipping them; part of `-f`. + follow_links: bool, + /// Read stdin even when it is a terminal; part of `-f`. + force_stdin: bool, + /// Files naming further inputs, one per line (`--filelist`). + filelists: Vec, + /// Directory every output is written into, by file name + /// (`--output-dir-flat`). + output_dir: Option, + /// Root under which each input's directory is replayed for its output + /// (`--output-dir-mirror`). Wins over `output_dir` when both are given. + output_dir_mirror: Option, + /// Whether the progress counter is drawn (`--[no-]progress`). + progress: Progress, } /// Upstream `zstd --maxdict` default (110 KiB). @@ -369,72 +430,277 @@ fn guard_binary_stdout(stdout_is_terminal: bool, force: bool) -> Result<()> { /// Outcome of argument parsing: either run with `Options`, or a terminal /// message already handled (help / version). enum Parsed { - Run(Options), + /// Boxed: the options are a few hundred bytes, and the other variant is + /// nothing at all. + Run(Box), Handled, } -fn main() -> Result<()> { +/// A command line that could not be parsed, with the display level the +/// flags before the mistake had reached: `-q --bogus` reports the mistake +/// alone, where the default level adds the short usage under it. +struct ParseFailure { + error: Error, + verbosity: i32, +} + +fn main() { // `args_os`, not `args`: the latter panics on an argument that is not // UTF-8, which on Unix is a legitimate filename rather than a mistake. - let raw: Vec = std::env::args_os().collect(); + let raw: Vec = std::env::args_os().collect(); let prog = raw .first() .map(|arg| arg.to_string_lossy().into_owned()) .unwrap_or_else(|| "zstd".to_string()); - let (default_mode, argv0_stdout) = program_mode(&prog); - - let parsed = parse_args(&raw[1..], default_mode, argv0_stdout)?; - let options = match parsed { - Parsed::Run(options) => options, - Parsed::Handled => return Ok(()), + let preset = program_preset(&prog); + // The environment is read before the command line, at the default + // level: `-q` further along cannot silence a warning about a variable + // that was already applied when it was met. + let default_level = level_from_env(std::env::var_os("ZSTD_CLEVEL").as_deref(), DEFAULT_LEVEL); + check_threads_env(std::env::var_os("ZSTD_NBTHREADS").as_deref(), DEFAULT_LEVEL); + + let options = match parse_args(&raw[1..], &preset, default_level) { + Ok(Parsed::Run(options)) => *options, + Ok(Parsed::Handled) => return, + Err(failure) => { + display!(failure.verbosity, 1, "zstd: {}", failure.error); + if failure.verbosity >= DEFAULT_LEVEL { + let mut stderr = io::stderr().lock(); + let _ = write_short_usage(&mut stderr, &preset.name); + } + std::process::exit(1); + } }; + let verbosity = options.verbosity; + // Status goes to stderr through `display!`, so it never contaminates a + // `-c` stdout data stream. + let status = match run(options) { + Ok(0) => 0, + Ok(_failed_inputs) => 1, + Err(err) => { + display!(verbosity, 1, "zstd: {err}"); + 1 + } + }; + std::process::exit(status); +} - // Status goes to stderr through `info!` below, so it never contaminates a - // `-c` stdout data stream. No subscriber to install: the macro writes - // there directly. - run(options) +/// What `argv[0]` presets before any flag is read: the conventional symlink +/// dispatch. `unzstd` decompresses; `zstdcat` and `zcat` decompress to +/// stdout, overwriting, passing non-zstd input through, and quietly, as the +/// reference command sets them up; `zstdmt` compresses like `zstd` (its +/// worker count has no effect here). +struct ProgramPreset { + /// The name the tool was invoked by, as the usage text shows it. + name: String, + mode: Mode, + to_stdout: bool, + force: bool, + pass_through: Option, + verbosity: i32, } -/// Default mode + forced-stdout from `argv[0]` (the conventional symlink -/// dispatch). `unzstd` decompresses; `zstdcat` decompresses to stdout. -fn program_mode(prog: &str) -> (Mode, bool) { +fn program_preset(prog: &str) -> ProgramPreset { let name = Path::new(prog) .file_name() .and_then(|n| n.to_str()) .unwrap_or(prog); // Strip a trailing `.exe` for Windows symlink names. let stem = name.strip_suffix(".exe").unwrap_or(name); + let plain = ProgramPreset { + name: name.to_string(), + mode: Mode::Compress, + to_stdout: false, + force: false, + pass_through: None, + verbosity: DEFAULT_LEVEL, + }; match stem { - "unzstd" => (Mode::Decompress, false), - "zstdcat" | "zcat" => (Mode::Decompress, true), - // "zstd", "zstdmt", anything else → compress by default. - _ => (Mode::Compress, false), + "unzstd" => ProgramPreset { + mode: Mode::Decompress, + ..plain + }, + "zstdcat" | "zcat" => ProgramPreset { + mode: Mode::Decompress, + to_stdout: true, + force: true, + pass_through: Some(true), + verbosity: 1, + ..plain + }, + // "zstd", "zstdmt", anything else: compress by default. + _ => plain, } } +/// The default compression level, from `ZSTD_CLEVEL` when it is set. +/// +/// Read the way the reference command reads it: an optional sign, then the +/// leading unsigned number with its `K`/`M` multiplier. A value that is not +/// that is ignored with a warning rather than failing the run, since an +/// environment variable is set far from the command that trips over it. The +/// variable replaces the DEFAULT only; `-#` on the command line still wins. +fn level_from_env(value: Option<&OsStr>, verbosity: i32) -> i32 { + let Some(value) = value else { + return CompressionLevel::DEFAULT_LEVEL; + }; + let text = value.to_string_lossy(); + let (sign, digits) = match text.strip_prefix('-') { + Some(rest) => (-1, rest), + None => (1, text.strip_prefix('+').unwrap_or(&text)), + }; + if digits.starts_with(|c: char| c.is_ascii_digit()) { + match read_leading_u32(digits) { + Ok((magnitude, "")) => { + // The scale runs from `MIN_LEVEL` up; the library clamps a + // value below it, and the command line's own ceiling reduces + // one above 19 with the usual warning later. + return i32::try_from(magnitude) + .map(|magnitude| sign * magnitude) + .unwrap_or(if sign < 0 { i32::MIN } else { i32::MAX }) + .max(CompressionLevel::MIN_LEVEL); + } + Err(_) => { + display!( + verbosity, + 2, + "Ignore environment variable setting ZSTD_CLEVEL={text}: numeric value too large" + ); + return CompressionLevel::DEFAULT_LEVEL; + } + Ok(_) => {} + } + } + display!( + verbosity, + 2, + "Ignore environment variable setting ZSTD_CLEVEL={text}: not a valid integer value" + ); + CompressionLevel::DEFAULT_LEVEL +} + +/// Validate `ZSTD_NBTHREADS` the way the reference command does, warning +/// about a value that is not an unsigned number. The count itself has no +/// effect here: compression runs single-threaded, which is also what the +/// reference command does when built without threads. +fn check_threads_env(value: Option<&OsStr>, verbosity: i32) { + let Some(value) = value else { + return; + }; + let text = value.to_string_lossy(); + if text.starts_with(|c: char| c.is_ascii_digit()) { + match read_leading_u32(&text) { + Ok((_, "")) => return, + Err(_) => { + display!( + verbosity, + 2, + "Ignore environment variable setting ZSTD_NBTHREADS={text}: numeric value too large" + ); + return; + } + Ok(_) => {} + } + } + display!( + verbosity, + 2, + "Ignore environment variable setting ZSTD_NBTHREADS={text}: not a valid unsigned value" + ); +} + +/// The value of a long option that takes one, given attached +/// (`--name=value`) or as the next argument (`--name value`), the way the +/// reference command's `NEXT_FIELD` reads it. `None` when `long` is not this +/// option at all. The next argument may not start with `-`: an option there +/// means the value was left out, and reading the option as the value would +/// hide the mistake. +fn option_value<'a>( + long: &str, + name: &str, + arg_os: &OsStr, + rest: &mut impl Iterator, +) -> Result> { + if long == name { + let Some((_, value)) = rest.next() else { + bail!("error: missing command argument for --{name}"); + }; + if value.to_string_lossy().starts_with('-') { + bail!("error: command cannot be separated from its argument by another command"); + } + return Ok(Some(PathBuf::from(value))); + } + if long.len() > name.len() && long.starts_with(name) && long.as_bytes()[name.len()] == b'=' { + // Past `--`, the name and the `=`, all ASCII. + return Ok(Some(attached_path(arg_os, 2 + name.len() + 1))); + } + Ok(None) +} + +/// [`option_value`] for an option whose value is a number or a word rather +/// than a path. +fn option_text<'a>( + long: &str, + name: &str, + arg_os: &OsStr, + rest: &mut impl Iterator, +) -> Result> { + Ok(option_value(long, name, arg_os, rest)? + .map(|value| value.as_os_str().to_string_lossy().into_owned())) +} + +/// [`option_text`] for an option spelled several ways (`--memory`, +/// `--memlimit`): the first spelling that matches supplies the value. +fn first_option_text<'a>( + long: &str, + names: &[&str], + arg_os: &OsStr, + rest: &mut impl Iterator, +) -> Result> { + for name in names { + if let Some(value) = option_text(long, name, arg_os, rest)? { + return Ok(Some(value)); + } + } + Ok(None) +} + /// Manual upstream-style parse: bare `-N` is a level, short flags combine /// (`-dc`), `-o`/`-D` take a value, `--long-opts` are matched whole. `clap`'s /// derive cannot model bare numeric levels, so we parse argv directly. +/// `default_level` is what `-#` overrides: the built-in default, or +/// `ZSTD_CLEVEL`. fn parse_args( - args: &[std::ffi::OsString], - default_mode: Mode, - argv0_stdout: bool, + args: &[OsString], + preset: &ProgramPreset, + default_level: i32, +) -> Result { + let mut verbosity = preset.verbosity; + parse_args_into(args, preset, default_level, &mut verbosity) + .map_err(|error| ParseFailure { error, verbosity }) +} + +fn parse_args_into( + args: &[OsString], + preset: &ProgramPreset, + default_level: i32, + verbosity: &mut i32, ) -> Result { let mut opts = Options { - mode: default_mode, - level: CompressionLevel::DEFAULT_LEVEL, + mode: preset.mode, + level: default_level, store: false, dict: None, - to_stdout: argv0_stdout, + to_stdout: preset.to_stdout, output: None, - force: false, + force: preset.force, keep: false, remove_source: false, inputs: Vec::new(), max_dict: DEFAULT_MAX_DICT, dict_id: None, bench: false, - bench_start: CompressionLevel::DEFAULT_LEVEL, + bench_start: default_level, bench_end: 0, bench_secs: 1.0, bench_separately: false, @@ -444,6 +710,19 @@ fn parse_args( target_block_size: None, pledged_size: None, size_hint: None, + verbosity: preset.verbosity, + checksum: true, + content_size_flag: true, + dict_id_flag: true, + pass_through: preset.pass_through, + exclude_compressed: false, + recursive: false, + follow_links: preset.force, + force_stdin: false, + filelists: Vec::new(), + output_dir: None, + output_dir_mirror: None, + progress: Progress::Auto, }; let mut ultra = false; let mut iter = args.iter().enumerate().peekable(); @@ -488,46 +767,55 @@ fn parse_args( opts.to_stdout = true; opts.output = None; } - "force" => opts.force = true, + // `-f` disables every input and output check at once, as the + // reference command's does: overwriting, a terminal on either + // end, and symbolic links. + "force" => { + opts.force = true; + opts.force_stdin = true; + opts.follow_links = true; + } "keep" => opts.keep = true, "rm" => opts.remove_source = true, "ultra" => ultra = true, - // Verbosity aliases are honest no-ops (our logging is fixed). - "quiet" | "verbose" => {} - // These change the wire format (suppress the checksum, the - // Frame_Content_Size field, or the Dictionary_ID). They are not - // wired through to the encoder yet, so accepting them silently - // would hand the caller the default layout instead of the - // requested one. Reject until they are honoured. - "no-check" | "no-content-size" | "no-dictID" => { - bail!("--{long} is not supported yet"); - } + "quiet" => *verbosity -= 1, + "verbose" => *verbosity += 1, + // The wire-format switches: the checksum, the + // Frame_Content_Size field, the Dictionary_ID. Each reaches + // the encoder, so the frame that comes out is the one asked + // for. + "check" => opts.checksum = true, + "no-check" => opts.checksum = false, + "content-size" => opts.content_size_flag = true, + "no-content-size" => opts.content_size_flag = false, + "no-dictID" => opts.dict_id_flag = false, + "pass-through" => opts.pass_through = Some(true), + "no-pass-through" => opts.pass_through = Some(false), + "exclude-compressed" => opts.exclude_compressed = true, + "progress" => opts.progress = Progress::Always, + "no-progress" => opts.progress = Progress::Never, "version" => { - print_version(); + print_version(*verbosity); return Ok(Parsed::Handled); } "help" => { - print_help(); + print_help(*verbosity, &preset.name); return Ok(Parsed::Handled); } // Flags that steer HOW the work is done, not what comes out: - // thread counts, memory ceilings, IO strategy, progress - // display, matcher hints. We are single-threaded and pick our - // own limits, so accepting them yields the same valid stream. - // Upstream takes them, so a script that passes them must not - // fail here — that is the whole drop-in contract. + // thread counts, IO strategy, matcher hints. We are + // single-threaded and pick our own limits, so accepting them + // yields the same valid stream. Upstream takes them, so a + // script that passes them must not fail here: that is the + // whole drop-in contract. "single-thread" | "adapt" - | "progress" - | "no-progress" - | "check" | "sparse" | "no-sparse" | "asyncio" | "no-asyncio" | "mmap-dict" | "no-mmap-dict" - | "no-pass-through" | "row-match-finder" | "no-row-match-finder" => {} // Forces literals compressed or stored, which changes the @@ -536,14 +824,6 @@ fn parse_args( "compress-literals" | "no-compress-literals" => { bail!("--{long} is not implemented"); } - // These decide WHICH files are processed, or what happens to - // input that is not compressed. Accepting them without doing - // the work would compress a file the caller asked to skip, or - // fail on one they asked to copy through — a wrong answer, not - // a slower one. - "pass-through" | "exclude-compressed" => { - bail!("--{long} is not implemented"); - } _ => { if long == "fast" { // `--fast` is the level -1 alias. @@ -567,53 +847,80 @@ fn parse_args( .expect("capped at |MIN_LEVEL|, which is an i32 magnitude"); } else if long.starts_with("use-dict=") { opts.dict = Some(attached_path(arg_os, "--use-dict=".len())); - } else if let Some(v) = long.strip_prefix("maxdict=") { + } else if let Some(v) = option_text(long, "maxdict", arg_os, &mut iter)? { opts.max_dict = v.parse::().wrap_err("invalid --maxdict size")?; - } else if let Some(v) = long.strip_prefix("dictID=") { + } else if let Some(v) = option_text(long, "dictID", arg_os, &mut iter)? { // Zero is how the dictionary API spells "choose one for // me", so it selects the default rather than being // carried through as an id the trainer would refuse. let id = v.parse::().wrap_err("invalid --dictID")?; opts.dict_id = (id != 0).then_some(id); - } else if let Some(v) = long.strip_prefix("stream-size=") { + } else if let Some(v) = option_text(long, "stream-size", arg_os, &mut iter)? { // An exact pledge: it goes into the frame header, so a // stream of a different length is an error. - opts.pledged_size = Some(parse_size(v).wrap_err("invalid --stream-size")?); - } else if let Some(v) = long.strip_prefix("size-hint=") { + opts.pledged_size = Some(parse_size(&v).wrap_err("invalid --stream-size")?); + } else if let Some(v) = option_text(long, "size-hint", arg_os, &mut iter)? { // An estimate: it sizes the encoder and nothing else, // so a wrong guess costs ratio rather than failing. - opts.size_hint = Some(parse_size(v).wrap_err("invalid --size-hint")?); - } else if let Some(v) = long - .strip_prefix("memory=") - .or_else(|| long.strip_prefix("memlimit=")) - .or_else(|| long.strip_prefix("memlimit-decompress=")) - { + opts.size_hint = Some(parse_size(&v).wrap_err("invalid --size-hint")?); + } else if let Some(v) = first_option_text( + long, + &["memory", "memlimit", "memlimit-decompress"], + arg_os, + &mut iter, + )? { // Recorded now, checked once the mode is final: the // ceiling describes decoding, and a later flag can // still decide this run does none. opts.memory_limit = - parse_memory_limit(v).wrap_err("invalid memory limit")?; + parse_memory_limit(&v).wrap_err("invalid memory limit")?; } else if let Some(params) = long.strip_prefix("adapt=") { // Parameterised form (`--adapt=min=1,max=9`). We do not - // vary the level, so the bounds change nothing — but a + // vary the level, so the bounds change nothing, but a // misspelled key or a non-numeric bound is still a // broken command line, and the contract is that ignored // options validate what they are given. parse_adapt_params(params)?; - } else if let Some(v) = long.strip_prefix("auto-threads=") { + } else if let Some(v) = option_text(long, "auto-threads", arg_os, &mut iter)? { // Single-threaded: the choice has no effect, but a bad // value is still a bad command line. if v != "physical" && v != "logical" { bail!("--auto-threads must be `physical` or `logical`, got `{v}`"); } - } else if let Some(v) = long.strip_prefix("target-compressed-block-size=") { + } else if let Some(v) = + option_text(long, "target-compressed-block-size", arg_os, &mut iter)? + { let target = - parse_size(v).wrap_err("invalid --target-compressed-block-size")?; + parse_size(&v).wrap_err("invalid --target-compressed-block-size")?; opts.target_block_size = Some(u32::try_from(target).map_err(|_| { eyre!("--target-compressed-block-size={v} is too large") })?); - } else if let Some(v) = long.strip_prefix("threads=") { + } else if let Some(v) = option_text(long, "threads", arg_os, &mut iter)? { let _ = v.parse::().wrap_err("invalid --threads")?; + } else if let Some(v) = option_text(long, "block-size", arg_os, &mut iter)? { + // The job size of a multi-threaded run: nothing here, + // but a malformed size is still a broken command line. + parse_size(&v).wrap_err("invalid --block-size")?; + } else if let Some(list) = option_value(long, "filelist", arg_os, &mut iter)? { + opts.filelists.push(list); + } else if let Some(dir) = + option_value(long, "output-dir-flat", arg_os, &mut iter)? + { + if dir.as_os_str().is_empty() { + bail!( + "error: output dir cannot be empty string (did you mean to pass '.' instead?)" + ); + } + opts.output_dir = Some(dir); + } else if let Some(dir) = + option_value(long, "output-dir-mirror", arg_os, &mut iter)? + { + if dir.as_os_str().is_empty() { + bail!( + "error: output dir cannot be empty string (did you mean to pass '.' instead?)" + ); + } + opts.output_dir_mirror = Some(dir); } else if let Some(v) = long.strip_prefix("format=") { // Anything but zstd would hand back a file the caller // did not ask for, so it fails rather than silently @@ -621,7 +928,7 @@ fn parse_args( if v != "zstd" { bail!("--format={v} is not supported; this build only writes zstd"); } - } else if long == "rsyncable" || long.starts_with("patch-from=") { + } else if long == "rsyncable" || long.starts_with("patch-from") { // Both change the emitted frame, so silence would be a // wrong answer rather than a slower one. bail!("--{long} is not implemented"); @@ -696,12 +1003,18 @@ fn parse_args( opts.to_stdout = true; opts.output = None; } - 'f' => opts.force = true, + 'f' => { + opts.force = true; + opts.force_stdin = true; + opts.follow_links = true; + } 'k' => opts.keep = true, - // `-S` measures each input on its own; `-q`/`-v` verbosity are - // accepted no-ops. + // `-S` measures each input on its own. 'S' => opts.bench_separately = true, - 'q' | 'v' => {} + 'q' => *verbosity -= 1, + 'v' => *verbosity += 1, + 'C' => opts.checksum = true, + 'r' => opts.recursive = true, 'B' | 'T' => { // `-B[N]` job / block size, `-T[N]` thread count. Both // steer how the work is done, not what comes out: we use a @@ -742,11 +1055,17 @@ fn parse_args( continue; } 'V' => { - print_version(); + print_version(*verbosity); + return Ok(Parsed::Handled); + } + 'H' => { + print_help(*verbosity, &preset.name); return Ok(Parsed::Handled); } - 'h' | 'H' => { - print_help(); + 'h' => { + let mut stdout = io::stdout().lock(); + write_short_usage(&mut stdout, &preset.name) + .wrap_err("failed to write usage")?; return Ok(Parsed::Handled); } 'D' | 'o' => { @@ -785,6 +1104,7 @@ fn parse_args( } let _ = idx; } + opts.verbosity = *verbosity; // `-M` bounds decompression, so it is weighed only on the runs that decode. // Compressing, listing or training allocates no decoder, and upstream takes @@ -797,11 +1117,6 @@ fn parse_args( check_memory_limit(limit, 0, 0)?; } validate_level(opts.level)?; - // `-o` names a single output, so it can't fan out over multiple inputs — - // except `--train`, where many sample files legitimately feed one dictionary. - if opts.mode != Mode::Train && !opts.bench && opts.output.is_some() && opts.inputs.len() > 1 { - bail!("-o cannot be combined with multiple input files"); - } if opts.bench && opts.bench_end < opts.bench_start { opts.bench_end = opts.bench_start; } @@ -826,8 +1141,10 @@ fn parse_args( // way upstream reduces it — a script that runs `zstd -22` compresses at 19 // rather than failing, and refusing here is what would break it. if !ultra && highest_level > CLI_MAX_LEVEL_WITHOUT_ULTRA { - info!( - "Warning : compression level higher than max, reduced to {CLI_MAX_LEVEL_WITHOUT_ULTRA} " + display!( + *verbosity, + 2, + "Warning : compression level higher than max, reduced to {CLI_MAX_LEVEL_WITHOUT_ULTRA}" ); opts.level = opts.level.min(CLI_MAX_LEVEL_WITHOUT_ULTRA); opts.bench_start = opts.bench_start.min(CLI_MAX_LEVEL_WITHOUT_ULTRA); @@ -850,7 +1167,7 @@ fn parse_args( matching runs; at level {long_level} it would only widen the window", ); } - Ok(Parsed::Run(opts)) + Ok(Parsed::Run(Box::new(opts))) } /// The part of `arg` from byte `at`, as the bytes it was given in. @@ -907,67 +1224,181 @@ fn validate_level(level: i32) -> Result<()> { Ok(()) } -fn print_version() { +/// `--version`: the reference command version this tool follows, and this +/// build's own. Below the default display level (`-qV`) only the bare +/// version number is printed, for a script to read; at `-vV` the supported +/// formats follow, as they do there. +fn print_version(verbosity: i32) { + if verbosity < DEFAULT_LEVEL { + println!("{UPSTREAM_VERSION}"); + return; + } println!( - "zstd (structured-zstd) {} — pure-Rust Zstandard", + "zstd version {UPSTREAM_VERSION} (structured-zstd v{})", env!("CARGO_PKG_VERSION") ); + if verbosity >= 3 { + println!("*** supports: zstd"); + } } -fn print_help() { - print_version(); - println!( - "\nUsage: zstd [OPTIONS] [FILE...]\n\ - \n\ - Modes:\n\ - \x20 -z, --compress compress (default)\n\ - \x20 -d, --decompress decompress\n\ - \x20 -t, --test test a compressed file's integrity\n\ - \x20 -l, --list list information about .zst files\n\ - \x20 --train FILEs train a dictionary from sample files\n\ - \x20 -b[N] [-e[N]] benchmark level N (through e)\n\ - \n\ - Options:\n\ - \x20 - compression level (1-19; 20-22 need --ultra)\n\ - \x20 --fast[=N] ultra-fast negative level\n\ - \x20 --ultra allow levels 20-22\n\ - \x20 --long[=N] enable long-distance matching\n\ - \x20 -D FILE use FILE as a dictionary\n\ - \x20 --maxdict=N dictionary size cap for --train\n\ - \x20 --dictID=N dictionary ID for --train\n\ - \x20 -o FILE write output to FILE\n\ - \x20 -c, --stdout write to stdout\n\ - \x20 -f, --force overwrite output / allow stdout to terminal\n\ - \x20 -k, --keep keep (do not delete) source files\n\ - \x20 --rm remove source files after success\n\ - \x20 --stream-size=N pledge the size of a streamed input\n\ - \x20 --size-hint=N same, as an estimate\n\ - \x20 -V, --version print version\n\ - \x20 -h, --help print this help\n\ - \n\ - Accepted for compatibility, with no effect here: -T/--single-thread/\n\ - --auto-threads (single-threaded), -B, --adapt, --[no-]progress,\n\ - --check, --[no-]sparse, --[no-]asyncio, --[no-]mmap-dict,\n\ - --no-pass-through, --[no-]row-match-finder.\n\ - \n\ - --target-compressed-block-size=N bounds what goes into each block, so\n\ - blocks flush sooner and stay near N. --long is --long=27, capped\n\ - there (above it the frame would declare a window this build refuses\n\ - to decode) and available from level 16 up, where long-distance\n\ - matching runs. A new output file keeps its source's permissions.\n\ - \n\ - Rejected rather than ignored, because they would change the result:\n\ - --no-check, --no-content-size, --no-dictID, --format= (other than\n\ - zstd), --patch-from, --rsyncable, --pass-through,\n\ - --exclude-compressed, --[no-]compress-literals, -M/--memory below\n\ - the enforced ceiling when decoding, --long below level 16, and\n\ - --train-cover / --train-legacy (--train and --train-fastcover train\n\ - with FastCOVER).\n\ - \n\ - With no FILE, or when FILE is `-`, read stdin / write stdout." - ); +/// The short usage: `-h`, and what a mistaken command line gets under its +/// error. Laid out as the reference command lays it out. +fn write_short_usage(out: &mut impl Write, program: &str) -> io::Result<()> { + writeln!( + out, + "Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.\n" + )?; + writeln!( + out, + "Usage: {program} [OPTIONS...] [INPUT... | -] [-o OUTPUT]\n" + )?; + writeln!(out, "Options:")?; + writeln!( + out, + " -o OUTPUT Write output to a single file, OUTPUT." + )?; + writeln!( + out, + " -k, --keep Preserve INPUT file(s). [Default]" + )?; + writeln!( + out, + " --rm Remove INPUT file(s) after successful (de)compression.\n" + )?; + writeln!( + out, + " -# Desired compression level, where `#` is a number between 1 and {CLI_MAX_LEVEL_WITHOUT_ULTRA};" + )?; + writeln!( + out, + " lower numbers provide faster compression, higher numbers yield" + )?; + writeln!( + out, + " better compression ratios. [Default: {}]\n", + CompressionLevel::DEFAULT_LEVEL + )?; + writeln!( + out, + " -d, --decompress Perform decompression." + )?; + writeln!( + out, + " -D DICT Use DICT as the dictionary for compression or decompression.\n" + )?; + writeln!( + out, + " -f, --force Disable input and output checks. Allows overwriting existing files," + )?; + writeln!( + out, + " receiving input from the console, printing output to STDOUT, and" + )?; + writeln!( + out, + " operating on links, block devices, etc. Unrecognized formats will be" + )?; + writeln!( + out, + " passed through as-is.\n" + )?; + writeln!( + out, + " -h Display short usage and exit." + )?; + writeln!( + out, + " -H, --help Display full help and exit." + )?; + writeln!( + out, + " -V, --version Display the program version and exit.\n" + ) } +/// The full help (`-H`, `--help`): the short usage, then every option this +/// build honours, then the ones it accepts without effect and the ones it +/// refuses, so a reader is not surprised by either. +fn print_help(verbosity: i32, program: &str) { + print_version(verbosity.max(DEFAULT_LEVEL)); + println!(); + let mut stdout = io::stdout().lock(); + let _ = write_short_usage(&mut stdout, program); + let _ = stdout.write_all(HELP_ADVANCED.as_bytes()); +} + +/// The part of the full help below the short usage. +const HELP_ADVANCED: &str = "\ +Advanced options: + -c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s). + + -v, --verbose Enable verbose output; pass multiple times to increase verbosity. + -q, --quiet Suppress warnings; pass twice to suppress errors. + + --[no-]progress Forcibly show/hide the progress counter. NOTE: Any (de)compressed + output to terminal will mix with progress counter text. + + -r Operate recursively on directories. + --filelist LIST Read a list of files to operate on from LIST. + --output-dir-flat DIR Store processed files in DIR. + --output-dir-mirror DIR Store processed files in DIR, respecting original directory structure. + + --[no-]check Add XXH64 integrity checksums during compression. [Default: Add, Validate] + If `-d` is present, ignore/validate checksums during decompression. + + -- Treat remaining arguments after `--` as files. + +Advanced compression options: + --ultra Enable levels beyond 19, up to 22; requires more memory. + --fast[=#] Use to very fast compression levels. [Default: 1] + --long[=#] Enable long distance matching with window log #. [Default: 27] + Available from level 16 up, where long-distance matching runs; + capped at 27, the window this build can read back. + --exclude-compressed Only compress files that are not already compressed. + + --stream-size=# Specify size of streaming input from STDIN. + --size-hint=# Optimize compression parameters for streaming input of approximately size #. + --target-compressed-block-size=# + Generate compressed blocks of approximately # size. + + --no-dictID Don't write `dictID` into the header (dictionary compression only). + --[no-]content-size Write the input size into the frame header when it is known. [Default: Write] + + --format=zstd Compress files to the `.zst` format. [Default] + +Advanced decompression options: + -l Print information about Zstandard-compressed files. + --test Test compressed file integrity. + -M# Set the memory usage limit to # megabytes. + --[no-]pass-through Pass through uncompressed files as-is. [Default: Disabled; Enabled for zstdcat] + +Dictionary builder: + --train Create a dictionary from a training set of files. + --train-fastcover Use the fast cover algorithm (the trainer --train also runs). + -o NAME Use NAME as dictionary name. [Default: dictionary] + --maxdict=# Limit dictionary to specified size #. [Default: 112640] + --dictID=# Force dictionary ID to #. [Default: Random] + +Benchmark options: + -b# Perform benchmarking with compression level #. [Default: 3] + -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1] + -i# Set the minimum evaluation to time # seconds. [Default: 1] + -S Output one benchmark result per input file. [Default: Consolidated result] + -D dictionary Benchmark using dictionary + +Environment: ZSTD_CLEVEL sets the default compression level; ZSTD_NBTHREADS is read and validated. + +Accepted for compatibility, with no effect here: -T#/--threads=#, --single-thread, +--auto-threads, -B#, --block-size=#, --adapt, --[no-]sparse, --[no-]asyncio, +--[no-]mmap-dict, --[no-]row-match-finder (compression runs single-threaded). + +Rejected rather than ignored, because they would change the result: --format= +other than zstd, --patch-from, --rsyncable, --[no-]compress-literals, +--train-cover, --train-legacy, and -M/--memory below the enforced ceiling when +decoding. A new output file keeps its source's permissions. +"; + /// Read the `-D` dictionary, if there is one, without breaking `-M` to do it. /// /// The limit was already weighed against what decoding alone needs; the @@ -1103,7 +1534,77 @@ impl Dictionaries { } } -fn run(opts: Options) -> Result<()> { +/// How the reference command names stdin in a summary line. +const STDIN_MARK: &str = "/*stdin*\\"; + +/// Whether the run reads stdin: no input named, or `-` among them. +fn reads_stdin(inputs: &[PathBuf]) -> bool { + inputs.is_empty() || inputs.iter().any(|input| input == Path::new("-")) +} + +/// Whether the run's data goes to stdout: `-c`, or stdin in and no `-o` out. +/// The reference command's `hasStdout`, which silences the result summary +/// and disables `--rm`. +fn writes_stdout(opts: &Options) -> bool { + opts.to_stdout + || (opts.output.is_none() && opts.inputs.iter().all(|input| input == Path::new("-"))) +} + +/// Run the command line. The count of inputs that failed comes back; the +/// exit status is 1 when it is not zero, as the reference command's is, +/// while an error that ends the run early is returned outright. +fn run(mut opts: Options) -> Result { + let Selection { files, named } = inputs::select_inputs( + std::mem::take(&mut opts.inputs), + &opts.filelists, + opts.recursive, + opts.follow_links, + opts.verbosity, + )?; + if files.is_empty() && named > 0 { + // Pointed at empty directories: nothing to do, and not a request to + // read stdin. The reference command says so and exits 0. + display!( + opts.verbosity, + 1, + "please provide correct input file(s) or non-empty directories -- ignored" + ); + return Ok(0); + } + opts.inputs = files; + + // Listing, training and benchmarking take named files only and refuse + // stdin with their own reasons; the streaming modes read it, and refuse to + // read it from a terminal unless forced, as the reference command does. + let streams = + matches!(opts.mode, Mode::Compress | Mode::Decompress | Mode::Test) && !opts.bench; + if streams && reads_stdin(&opts.inputs) && !opts.force_stdin && io::stdin().is_terminal() { + bail!("stdin is a console, aborting"); + } + let has_stdout_output = matches!(opts.mode, Mode::Compress | Mode::Decompress) + && !opts.bench + && writes_stdout(&opts); + // No status message by default when the data goes to stdout. + if has_stdout_output && opts.verbosity == DEFAULT_LEVEL { + opts.verbosity = 1; + } + // When stderr is not a terminal, do not pollute it with progress updates + // unless asked. + if !io::stderr().is_terminal() && opts.progress != Progress::Always { + opts.progress = Progress::Never; + } + if has_stdout_output && opts.remove_source { + display!( + opts.verbosity, + 3, + "Note: src files are not removed when output is stdout" + ); + opts.remove_source = false; + } + if opts.mode == Mode::Test { + opts.remove_source = false; + } + let dict_bytes = load_dictionary(&opts)?; // `-b` benchmarks compression/decompression across levels instead of @@ -1111,7 +1612,8 @@ fn run(opts: Options) -> Result<()> { // blob rather than the parsed forms, since its own memory ceiling has to be // weighed before anything is built from them. if opts.bench { - return run_benchmark(&opts, dict_bytes); + run_benchmark(&opts, dict_bytes)?; + return Ok(0); } let dicts = Dictionaries::prepare(dict_bytes.as_deref(), compresses(&opts), decodes(&opts))?; // Everything from here on primes from the parsed form, so the blob it was @@ -1122,39 +1624,14 @@ fn run(opts: Options) -> Result<()> { // `--train` builds a dictionary from the sample files rather than // (de)compressing them; handle it before the streaming flow. if opts.mode == Mode::Train { - return train_dictionary(&opts); + train_dictionary(&opts)?; + return Ok(0); } // `--list` walks frame headers without decoding; it needs a seekable file // (not a stream), so it is handled separately from the (de)compress flow. if opts.mode == Mode::List { - if opts.inputs.is_empty() { - bail!("--list requires regular files (cannot list stdin)"); - } - // `-` is stdin, which the walk cannot seek through any more than a - // FIFO. Answered before the stat below, or the marker would name a file - // whenever one happens to sit in the working directory under that name - // — and stdin whenever one does not. A file really called `-` is still - // reachable, spelled `./-`. - if opts.inputs.iter().any(|input| input == Path::new("-")) { - bail!("--list cannot list stdin; name a file (`./-` for one called `-`)"); - } - // The walk seeks between frame headers, so it needs a file that can - // seek. Settled for every input before any is opened: opening a FIFO - // blocks until a writer appears, and the failure would then arrive - // from the seek rather than from the thing that was wrong. - for input in &opts.inputs { - let metadata = fs::metadata(input) - .wrap_err_with(|| format!("failed to inspect {}", input.display()))?; - if !metadata.is_file() { - bail!("--list needs regular files: {} is not one", input.display()); - } - } - print_list_header(); - for input in &opts.inputs { - list_file(input)?; - } - return Ok(()); + return list_files(&opts); } // A destination named outright belongs to the whole run, whatever it reads: @@ -1165,7 +1642,7 @@ fn run(opts: Options) -> Result<()> { // // Compression only. Decompressing has already finished with the dictionary // by the time anything is written, and what it writes is plaintext that - // never needed it — the reference command permits that, and refusing would + // never needed it: the reference command permits that, and refusing would // break a working script to protect nothing. if let (Some(output), Some(dict)) = (&opts.output, &opts.dict) && !opts.to_stdout @@ -1178,29 +1655,203 @@ fn run(opts: Options) -> Result<()> { dict.display(), ); } - if opts.inputs.is_empty() { - return process_stdin_stdout(&opts, &dicts); + let total = opts.inputs.len().max(1); + match (opts.to_stdout, &opts.output) { + (true, _) => process_concatenated(&opts, &dicts, None, total), + // `-t` writes nothing, so a destination it was given is set aside. + (false, Some(output)) if opts.mode != Mode::Test => { + process_concatenated(&opts, &dicts, Some(output), total) + } + _ => process_separately(&opts, &dicts, total), } +} + +/// Every input into one destination: stdout, or the `-o` file. +/// +/// Several inputs concatenated lose their names and boundaries, so the +/// reference command warns, disables `--rm`, and, for a file it was not +/// forced to write, asks first. An input that cannot be opened is reported +/// and skipped; one that fails while streaming ends the run, since a partial +/// frame would already be in the shared output. +fn process_concatenated( + opts: &Options, + dicts: &Dictionaries, + output: Option<&Path>, + total: usize, +) -> Result { + let mut remove_source = opts.remove_source; + if total > 1 { + match output { + None => display!( + opts.verbosity, + 2, + "zstd: WARNING: all input files will be processed and concatenated into stdout." + ), + Some(output) => display!( + opts.verbosity, + 2, + "zstd: WARNING: all input files will be processed and concatenated into a single output file: {}", + output.display() + ), + } + display!( + opts.verbosity, + 2, + "The concatenated output CANNOT regenerate original file names nor directory structure." + ); + if remove_source { + display!( + opts.verbosity, + 2, + "Since it's a destructive operation, input files will not be removed." + ); + remove_source = false; + } + if output.is_some() && !opts.force { + if opts.verbosity <= 1 { + // Quiet mode: no prompt is possible, so the run refuses. + display!( + opts.verbosity, + 1, + "Concatenating multiple processed inputs into a single output loses file metadata." + ); + display!(opts.verbosity, 1, "Aborting."); + return Ok(total); + } + if !confirm( + "Proceed? (y/n): ", + "Aborting...", + reads_stdin(&opts.inputs), + &mut io::stdin().lock(), + ) { + return Ok(total); + } + } + } + let mut tally = Tally::default(); + let inputs: Vec<&Path> = if opts.inputs.is_empty() { + vec![Path::new("-")] + } else { + opts.inputs.iter().map(PathBuf::as_path).collect() + }; + match output { + None => { + let stdout = io::stdout(); + // Only compression produces binary; `-d` to a terminal is text the + // user asked for, which the reference command also allows. + if opts.mode == Mode::Compress { + guard_binary_stdout(stdout.is_terminal(), opts.force)?; + } + let mut sink = stdout.lock(); + for input in inputs { + let outcome = stream_input_to(opts, dicts, input, &mut sink, STDOUT_MARK, total)?; + tally.record(outcome); + } + } + Some(output) => { + // One input keeps the reference command's single-file path, where + // the output takes the source's permissions; several inputs share + // an output that takes none of theirs. + if let [input] = inputs[..] { + let (source, metadata) = if input == Path::new("-") { + (None, None) + } else { + match open_input(opts, input) { + Ok(Some((source, metadata))) => (Some(source), Some(metadata)), + Ok(None) => return Ok(0), + Err(err) => { + display!(opts.verbosity, 1, "zstd: {err}"); + return Ok(1); + } + } + }; + let name = if source.is_some() { + input.display().to_string() + } else { + STDIN_MARK.to_string() + }; + let written = + write_output_file(opts, output, metadata.as_ref(), |sink| match source { + Some(source) => stream_opened( + opts, + dicts, + source, + metadata.as_ref().expect("an opened input has metadata"), + sink, + ), + None => stream_stdin(opts, dicts, sink), + })?; + match written { + Some(processed) => { + file_summary( + opts, + total, + &name, + &output.display().to_string(), + &processed, + ); + if remove_source && input != Path::new("-") { + remove_source_if_requested(opts, input)?; + } + tally.record(Outcome::Done(processed)); + } + None => tally.record(Outcome::Refused), + } + } else { + let written = write_output_file(opts, output, None, |sink| { + let mut tally = Tally::default(); + for input in &inputs { + let outcome = stream_input_to( + opts, + dicts, + input, + &mut *sink, + &output.display().to_string(), + total, + )?; + tally.record(outcome); + } + Ok(tally) + })?; + match written { + Some(inner) => tally = inner, + None => tally.failed = total, + } + } + } + } + multi_summary(opts, total, &tally); + Ok(tally.failed) +} + +/// Every input into its own output, placed beside it or under the output +/// directory; stdin, when it is among them, goes to stdout. An input that +/// fails is reported and the rest are still processed, as the reference +/// command does; the count that failed decides the exit status. +fn process_separately(opts: &Options, dicts: &Dictionaries, total: usize) -> Result { // The inputs are processed one after another, so an output derived from an // early one can land on a file still waiting its turn: `-f foo foo.zst` // would replace `foo.zst` before it is ever read. The `-D` dictionary is a - // file this run needs too — and the one a frame will need to be read back, + // file this run needs too, and the one a frame will need to be read back, // so writing over it destroys the key to what was just produced. `-f` // permits overwriting the output, not destroying either, so everything the // run reads is checked before the first byte is written. // // Only for the modes that write one. Testing decodes into a sink and names - // no destination, so asking what it would produce has no answer. - if !opts.to_stdout && matches!(opts.mode, Mode::Compress | Mode::Decompress) { + // no destination, so asking what it would produce has no answer. An input + // whose output cannot be derived is left for the loop below to report. + if matches!(opts.mode, Mode::Compress | Mode::Decompress) { for input in &opts.inputs { if input == Path::new("-") { continue; } - let output = derive_output_path(&opts, input)?; + let Ok(output) = derive_output_path(opts, input) else { + continue; + }; // Compared as files rather than as spellings: `foo.zst`, // `./foo.zst` and `dir/../dir/foo.zst` name one file, and a match // on the string alone would miss two of the three. Compression - // only, for the reason given at the `-o` check above. + // only, for the reason given at the `-o` check in `run`. if let Some(dict) = &opts.dict && opts.mode == Mode::Compress && names_the_same_file(&output, dict)? @@ -1242,14 +1893,170 @@ fn run(opts: Options) -> Result<()> { } } } + let mut tally = Tally::default(); + if opts.inputs.is_empty() { + let outcome = stream_input_to( + opts, + dicts, + Path::new("-"), + io::stdout().lock(), + STDOUT_MARK, + total, + )?; + tally.record(outcome); + } for input in &opts.inputs { - if input == Path::new("-") { - process_stdin_stdout(&opts, &dicts)?; + let outcome = if input == Path::new("-") { + stream_input_to(opts, dicts, input, io::stdout().lock(), STDOUT_MARK, total)? } else { - process_file(&opts, input, &dicts)?; + match process_file(opts, input, dicts, total) { + Ok(outcome) => outcome, + Err(err) => { + display!(opts.verbosity, 1, "zstd: {err}"); + Outcome::Refused + } + } + }; + tally.record(outcome); + } + // Under `--output-dir-flat` two inputs with one name land on one output, + // the later replacing the earlier; the reference command warns after the + // run, once per shared name. + if opts.output_dir.is_some() && opts.output_dir_mirror.is_none() { + for name in inputs::shared_file_names(&opts.inputs) { + display!( + opts.verbosity, + 2, + "WARNING: Two files have same filename: {}", + Path::new(&name).display() + ); } } - Ok(()) + multi_summary(opts, total, &tally); + Ok(tally.failed) +} + +/// What became of one input. +enum Outcome { + /// Processed, with what went in and came out. + Done(Processed), + /// Deliberately left alone (`--exclude-compressed`); not a failure. + Skipped, + /// Not processed, and already reported. + Refused, +} + +/// Bytes an input contributed: read from it, and written for it. +struct Processed { + read: u64, + written: u64, +} + +/// The run's running totals, for the multi-file summary and the exit status. +#[derive(Default)] +struct Tally { + processed: usize, + failed: usize, + read: u64, + written: u64, +} + +impl Tally { + fn record(&mut self, outcome: Outcome) { + match outcome { + Outcome::Done(processed) => { + self.processed += 1; + self.read += processed.read; + self.written += processed.written; + } + Outcome::Skipped => {} + Outcome::Refused => self.failed += 1, + } + } +} + +/// The per-file result line, in the reference command's layout, shown for a +/// single input or under `-v` for each of several. +fn file_summary( + opts: &Options, + total: usize, + name: &str, + destination: &str, + processed: &Processed, +) { + if total > 1 && opts.verbosity < 3 { + return; + } + let verbose = opts.verbosity > 3; + match opts.mode { + Mode::Compress => { + let read = HumanSize::new(processed.read, verbose); + let written = HumanSize::new(processed.written, verbose); + if processed.read == 0 { + display!( + opts.verbosity, + 2, + "{name:<20} : ({read:>6} => {written:>6}, {destination})" + ); + } else { + display!( + opts.verbosity, + 2, + "{name:<20} :{:>6.2}% ({read:>6} => {written:>6}, {destination})", + processed.written as f64 / processed.read as f64 * 100.0 + ); + } + } + Mode::Decompress | Mode::Test => { + display!(opts.verbosity, 2, "{name:<20}: {} bytes", processed.written); + } + Mode::List | Mode::Train => {} + } +} + +/// The closing line of a run over several inputs, when at least one went +/// through. +fn multi_summary(opts: &Options, total: usize, tally: &Tally) { + if tally.processed < 1 || total <= 1 { + return; + } + let verbose = opts.verbosity > 3; + match opts.mode { + Mode::Compress => { + let read = HumanSize::new(tally.read, verbose); + let written = HumanSize::new(tally.written, verbose); + if tally.read == 0 { + display!( + opts.verbosity, + 2, + "{:>3} files compressed : ({} => {})", + tally.processed, + read.columns(6), + written.columns(6) + ); + } else { + display!( + opts.verbosity, + 2, + "{:>3} files compressed : {:.2}% ({} => {})", + tally.processed, + tally.written as f64 / tally.read as f64 * 100.0, + read.columns(6), + written.columns(6) + ); + } + } + Mode::Decompress | Mode::Test => { + display!( + opts.verbosity, + 2, + "{} files decompressed : {:>6} bytes total", + tally.processed, + tally.written + ); + } + Mode::List | Mode::Train => {} + } } /// `-b`: benchmark compression + decompression of the input across the @@ -1454,7 +2261,7 @@ fn benchmark_one(opts: &Options, dicts: &Dictionaries, label: &str, data: &[u8]) let mb = data.len() as f64 / 1e6; println!( "benchmarking {label} ({}) levels {}..={}", - fmt_size(data.len() as f64), + HumanSize::new(data.len() as u64, false), opts.bench_start, opts.bench_end, ); @@ -1498,7 +2305,12 @@ fn benchmark_one(opts: &Options, dicts: &Dictionaries, label: &str, data: &[u8]) loop { decoded.clear(); let t = Instant::now(); - decompress_stream(compressed.as_slice(), &mut decoded, dicts)?; + decompress_stream( + compressed.as_slice(), + &mut decoded, + dicts, + &DecodeSettings::from_options(opts), + )?; best_decompress = best_decompress.min(t.elapsed().as_secs_f64()); if start.elapsed().as_secs_f64() >= opts.bench_secs { break; @@ -1518,7 +2330,7 @@ fn benchmark_one(opts: &Options, dicts: &Dictionaries, label: &str, data: &[u8]) }; println!( "{level:>3} {:>10} {ratio:>7.3} {c_speed:>7.1} MB/s comp {d_speed:>8.1} MB/s decomp", - fmt_size(compressed.len() as f64), + HumanSize::new(compressed.len() as u64, false), ); } Ok(()) @@ -1679,10 +2491,12 @@ fn train_dictionary(opts: &Options) -> Result<()> { } drop(temp_file); replace_output_file(&temp_path, &output, sample_permissions)?; - info!( + display!( + opts.verbosity, + 2, "trained {} ({}) from {} sample file(s)", output.display(), - fmt_size(dict.len() as f64), + HumanSize::new(dict.len() as u64, false), opts.inputs.len() ); Ok(()) @@ -1816,9 +2630,101 @@ fn read_filling(reader: &mut R, buf: &mut [u8]) -> Result { Ok(filled) } -/// Column header for `--list`, matching upstream's `zstd -l` layout. -fn print_list_header() { - println!("Frames Skips Compressed Uncompressed Ratio Check DictID Filename"); +/// `--list`: one row per archive in the reference command's layout, or one +/// block per archive under `-v`, then a total when there are several. An +/// archive that cannot be listed is reported and the rest are still listed; +/// the count that failed comes back. +fn list_files(opts: &Options) -> Result { + if opts.inputs.is_empty() { + bail!("No files given"); + } + // `-` is stdin, which the walk cannot seek through any more than a FIFO. + // Answered before any file is opened, or the marker would name a file + // whenever one happens to sit in the working directory under that name, + // and stdin whenever one does not. A file really called `-` is still + // reachable, spelled `./-`. + if opts.inputs.iter().any(|input| input == Path::new("-")) { + bail!("--list does not support reading from standard input"); + } + let verbose = opts.verbosity > DEFAULT_LEVEL; + if !verbose { + println!("Frames Skips Compressed Uncompressed Ratio Check Filename"); + } + let mut total = ListTotal::default(); + let mut failed = 0; + for input in &opts.inputs { + match list_file(input, verbose, opts.verbosity) { + Ok(summary) => total.add(&summary), + Err(err) => { + display!(opts.verbosity, 1, "zstd: {err}"); + failed += 1; + } + } + } + if opts.inputs.len() > 1 && !verbose { + total.print(); + } + Ok(failed) +} + +/// What the archives listed so far add up to, for the closing row. +#[derive(Default)] +struct ListTotal { + frames: u64, + skips: u64, + compressed: u64, + decompressed: u64, + /// Some archive omitted a Frame_Content_Size, so the total is unknowable. + decompressed_unknown: bool, + /// Some archive carries no checksum, so the total's `Check` says nothing. + without_check: bool, + files: usize, +} + +impl ListTotal { + fn add(&mut self, summary: &ArchiveSummary) { + self.frames += summary.frames; + self.skips += summary.skips; + self.compressed += summary.compressed; + match summary.decompressed { + Some(decompressed) => self.decompressed += decompressed, + None => self.decompressed_unknown = true, + } + self.without_check |= !summary.check; + self.files += 1; + } + + fn print(&self) { + println!("----------------------------------------------------------------- "); + let compressed = HumanSize::new(self.compressed, false); + let check = if self.without_check { "" } else { "XXH64" }; + if self.decompressed_unknown { + println!( + "{:>6} {:>5} {} {:>5} {} files", + self.frames, + self.skips, + compressed.columns(6), + check, + self.files + ); + } else { + let ratio = if self.compressed == 0 { + 0.0 + } else { + self.decompressed as f64 / self.compressed as f64 + }; + println!( + "{:>6} {:>5} {} {} {:>5.3} {:>5} {} files", + self.frames, + self.skips, + compressed.columns(6), + HumanSize::new(self.decompressed, false).columns(8), + ratio, + check, + self.files + ); + } + } } /// Largest possible zstd frame header: 4-byte magic + 1-byte descriptor + up to @@ -1827,6 +2733,7 @@ fn print_list_header() { const MAX_FRAME_HEADER_LEN: usize = 18; /// What one `--list` row says about an archive. +#[derive(Debug)] struct ArchiveSummary { frames: u64, skips: u64, @@ -1853,6 +2760,13 @@ struct ArchiveSummary { /// Whether the data frames named one dictionary between them. False makes /// the id above absent rather than wrong. dict_ids_agree: bool, + /// The window the last data frame declares, which is what a decoder will + /// need in memory to read it. + window_size: u64, + /// The stored checksum of the last data frame that carries one; shown + /// under `-lv` for a single-frame archive, as the reference command + /// shows it. + checksum: Option<[u8; 4]>, } /// Walk every frame in the file (no body decode), summing compressed and @@ -1890,6 +2804,8 @@ fn summarize_archive(path: &Path) -> Result { let mut check = false; let mut dict_id = None; let mut dict_ids_agree = true; + let mut window_size = 0u64; + let mut checksum = None; while offset < compressed { // Read just enough for the frame header (a short read near EOF is fine — @@ -1913,7 +2829,10 @@ fn summarize_archive(path: &Path) -> Result { frames += 1; continue; } - Err(err) => bail!("{}: not a zstd frame: {err:?}", path.display()), + Err(err) => bail!( + "File \"{}\" not compressed by zstd ({err:?})", + path.display() + ), }; // The frame's Block_Maximum_Size bounds every block (RFC 8878 §3.1.1.2). @@ -1951,13 +2870,20 @@ fn summarize_archive(path: &Path) -> Result { } // A trailing 4-byte content checksum follows the last block when present. let frame_end = if info.content_checksum { - block_offset + let end = block_offset .checked_add(4) .filter(|end| *end <= compressed) - .ok_or_else(|| eyre!("{}: truncated content checksum", path.display()))? + .ok_or_else(|| eyre!("{}: truncated content checksum", path.display()))?; + file.seek(SeekFrom::Start(block_offset))?; + let mut stored = [0u8; 4]; + file.read_exact(&mut stored) + .map_err(|_| eyre!("{}: truncated content checksum", path.display()))?; + checksum = Some(stored); + end } else { block_offset }; + window_size = info.window_size; match info.content_size { // Declared, not measured: a handful of header bytes can claim any @@ -2002,90 +2928,89 @@ fn summarize_archive(path: &Path) -> Result { check, dict_id: dict_ids_agree.then_some(dict_id).flatten(), dict_ids_agree, + window_size, + checksum, }) } -/// Print one `--list` row, in the reference tool's `zstd -l` layout. -/// -/// Decompressed size is `--` when any frame omits its Frame_Content_Size. -fn list_file(path: &Path) -> Result<()> { +/// Print one `--list` entry: a row in the reference command's `zstd -l` +/// layout, or, when `verbose`, the block its `-lv` prints. The decompressed +/// columns are left blank when any frame omits its Frame_Content_Size. +fn list_file(path: &Path, verbose: bool, verbosity: i32) -> Result { + // The walk seeks between frame headers, so it needs a file that can seek. + // Settled before the file is opened: opening a FIFO blocks until a writer + // appears, and the failure would then arrive from the seek rather than + // from the thing that was wrong. + let metadata = + fs::metadata(path).wrap_err_with(|| format!("Error : {} is not a file", path.display()))?; + if !metadata.is_file() { + bail!("Error : {} is not a file", path.display()); + } let summary = summarize_archive(path)?; - let ArchiveSummary { - frames, - skips, - compressed, - decompressed, - check, - dict_id, - dict_ids_agree, - } = summary; - let ratio = match decompressed { - Some(d) if d > 0 => format!("{:.3}", d as f64 / compressed as f64), - _ => "--".to_string(), - }; - let decompressed_str = match decompressed { - Some(d) => fmt_size(d as f64), - None => "--".to_string(), - }; - // The column holds one id, so an archive built from several dictionaries - // has nothing to put there — said out loud rather than left to the `0`, - // which on its own reads as "no dictionary needed". - if !dict_ids_agree { - info!( - "{}: frames use different dictionaries; no single dictionary ID applies", - path.display() + // The id column holds one id, so an archive built from several + // dictionaries has nothing to put there. Said out loud rather than left to + // the `0`, which on its own reads as "no dictionary needed". + if !summary.dict_ids_agree { + display!( + verbosity, + 2, + "WARNING: File contains multiple frames with different dictionary IDs. Showing dictID 0 instead" ); } + let ratio = summary + .decompressed + .map(|decompressed| decompressed as f64 / summary.compressed as f64); + let check = if summary.check { "XXH64" } else { "None" }; + let compressed = HumanSize::new(summary.compressed, false); + if !verbose { + match (summary.decompressed, ratio) { + (Some(decompressed), Some(ratio)) => println!( + "{:>6} {:>5} {} {} {ratio:>5.3} {check:>5} {}", + summary.frames, + summary.skips, + compressed.columns(6), + HumanSize::new(decompressed, false).columns(8), + path.display() + ), + _ => println!( + "{:>6} {:>5} {} {check:>5} {}", + summary.frames, + summary.skips, + compressed.columns(6), + path.display() + ), + } + return Ok(summary); + } + let data_frames = summary.frames - summary.skips; + println!("{} ", path.display()); + println!("# Zstandard Frames: {data_frames}"); + if summary.skips > 0 { + println!("# Skippable Frames: {}", summary.skips); + } + println!("DictID: {}", summary.dict_id.unwrap_or(0)); println!( - "{frames:>6} {skips:>5} {:>10} {:>12} {ratio:>5} {:>5} {:>6} {}", - fmt_size(compressed as f64), - decompressed_str, - if check { "XXH64" } else { "None" }, - dict_id - .map(|id| id.to_string()) - .unwrap_or_else(|| "0".to_string()), - path.display(), + "Window Size: {} ({} B)", + HumanSize::new(summary.window_size, false), + summary.window_size ); - Ok(()) -} - -/// stdin → stdout (or → `-o` file) for a `-` input or no inputs. -fn process_stdin_stdout(opts: &Options, dicts: &Dictionaries) -> Result<()> { - let stdin = io::stdin(); - let reader = stdin.lock(); - // `-o` redirects stdin's (de)compressed output to a file, unless `-c` - // explicitly forces stdout. stdin has no length to stat, which is exactly - // why upstream offers `--stream-size` / `--size-hint`: pass whatever the - // caller pledged. - if let Some(output) = &opts.output - && !opts.to_stdout - && matches!(opts.mode, Mode::Compress | Mode::Decompress) - { - // stdin has no file to take permissions from, so the umask decides. - return write_stream_to_file(opts, reader, output, opts.pledged_size, dicts, None); + println!("Compressed Size: {compressed} ({} B)", summary.compressed); + if let (Some(decompressed), Some(ratio)) = (summary.decompressed, ratio) { + println!( + "Decompressed Size: {} ({decompressed} B)", + HumanSize::new(decompressed, false) + ); + println!("Ratio: {ratio:.4}"); } - match opts.mode { - Mode::Compress => { - let stdout = io::stdout(); - guard_binary_stdout(stdout.is_terminal(), opts.force)?; - compress_stream( - reader, - stdout.lock(), - &FrameSettings::from_options(opts), - dicts, - ) - } - Mode::Decompress => { - let stdout = io::stdout(); - decompress_stream(reader, stdout.lock(), dicts) - } - Mode::Test => decompress_stream(reader, io::sink(), dicts).map(|_| { - info!("stdin: OK"); - }), - Mode::List | Mode::Train => { - unreachable!("--list / --train handled in run() before streaming") - } + match summary.checksum { + Some(stored) if summary.check && data_frames == 1 => println!( + "Check: {check} {:02x}{:02x}{:02x}{:02x}", + stored[3], stored[2], stored[1], stored[0] + ), + _ => println!("Check: {check}"), } + println!(); + Ok(summary) } /// Remove the source file after a successful (de)compression when `--rm` is set @@ -2096,155 +3021,394 @@ fn remove_source_if_requested(opts: &Options, input: &Path) -> Result<()> { // there is no saved copy to justify deleting the original. Upstream keeps // the file for `-c` too. if opts.remove_source && !opts.keep && !opts.to_stdout { + // Removing the source is past the point where an interruption should + // delete anything: the output is in place, and the guard would take + // it along with the source. + interrupt::clear(); fs::remove_file(input).wrap_err("failed to remove source file after success")?; } Ok(()) } -/// Run the (de)compression core into an arbitrary writer for the current mode. -fn run_stream_core( +/// Counts what passes through to the writer beneath, so the summary can say +/// how much came out of a compression whatever the sink was. +struct CountingWriter { + inner: W, + written: u64, +} + +impl Write for CountingWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.inner.write(buf)?; + self.written += written as u64; + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// Whether a file type is a named pipe. +fn is_fifo_type(kind: &fs::FileType) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::FileTypeExt; + kind.is_fifo() + } + #[cfg(not(unix))] + { + let _ = kind; + false + } +} + +/// Whether a file type is a block device. +fn is_block_device_type(kind: &fs::FileType) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::FileTypeExt; + kind.is_block_device() + } + #[cfg(not(unix))] + { + let _ = kind; + false + } +} + +/// Open one named input for streaming, or say why it cannot be. +/// +/// `Ok(None)` is an input deliberately left alone under +/// `--exclude-compressed`. A directory, a socket or a device is refused the +/// way the reference command refuses them ("-- ignored"); `-f` admits block +/// devices. The kind and length that matter are those of the OPEN file: a +/// path is only a name, and can be made to name something else between the +/// look and the open. +fn open_input(opts: &Options, input: &Path) -> Result> { + let named = fs::metadata(input) + .map_err(|err| eyre!("can't stat {} : {err} -- ignored", input.display()))?; + if named.is_dir() { + bail!("{} is a directory -- ignored", input.display()); + } + let admissible = |kind: &fs::FileType| { + kind.is_file() || is_fifo_type(kind) || (opts.force && is_block_device_type(kind)) + }; + if !admissible(&named.file_type()) { + bail!("{} is not a regular file -- ignored", input.display()); + } + if compresses(opts) && opts.exclude_compressed && inputs::has_compressed_extension(input) { + display!( + opts.verbosity, + 4, + "File is already compressed : {}", + input.display() + ); + return Ok(None); + } + let source = File::open(input).wrap_err_with(|| input.display().to_string())?; + let metadata = source + .metadata() + .wrap_err_with(|| format!("failed to inspect {}", input.display()))?; + if metadata.is_dir() || !admissible(&metadata.file_type()) { + bail!("{} is not a regular file -- ignored", input.display()); + } + Ok(Some((source, metadata))) +} + +/// Stream one opened input through the codec into `sink`. +fn stream_opened( opts: &Options, - reader: R, - writer: W, - // Exact length of THIS input when it has one to stat; `--stream-size` - // stands in when it does not, which is what that option is for. - // `--size-hint` travels separately in `opts`. - pledged_size: Option, dicts: &Dictionaries, -) -> Result<()> { - match opts.mode { - Mode::Compress => compress_stream( - reader, - writer, - &FrameSettings { - pledged_size: pledged_size.or(opts.pledged_size), - ..FrameSettings::from_options(opts) - }, + source: File, + metadata: &fs::Metadata, + sink: W, +) -> Result { + // Kept as the `u64` the filesystem reports: the work streams, so a file + // only has to fit the window, and narrowing to a pointer would refuse + // 4 GiB archives on 32-bit targets for no reason the work has. + let source_size = metadata.len(); + // Only a regular file's length says how many bytes will be read. A FIFO, + // a device or a socket reports something unrelated (commonly zero), and + // pledging that turns a perfectly good stream into a length mismatch. + let pledged_size = metadata.is_file().then_some(source_size); + let shown = opts + .progress + .shown(opts.verbosity, io::stderr().is_terminal()); + let reader = ProgressMonitor::new(BufReader::new(source), source_size, shown); + stream(opts, dicts, reader, pledged_size, sink) +} + +/// Stream stdin through the codec into `sink`. stdin has no length to stat, +/// which is exactly why `--stream-size` / `--size-hint` exist: whatever the +/// caller pledged travels in `opts`. +fn stream_stdin(opts: &Options, dicts: &Dictionaries, sink: W) -> Result { + let stdin = io::stdin(); + let reader = ProgressMonitor::new(stdin.lock(), 0, false); + stream(opts, dicts, reader, None, sink) +} + +/// Run the mode's codec from `reader` into `sink` and count both sides. +/// `pledged_size` is the exact length of THIS input when it has one to stat; +/// `--stream-size` stands in when it does not. `-t` decodes into nothing, +/// whatever sink it was handed. +fn stream( + opts: &Options, + dicts: &Dictionaries, + mut reader: ProgressMonitor, + pledged_size: Option, + mut sink: W, +) -> Result { + let written = match opts.mode { + Mode::Compress => { + let mut counting = CountingWriter { + inner: &mut sink, + written: 0, + }; + compress_stream( + &mut reader, + &mut counting, + &FrameSettings { + pledged_size: pledged_size.or(opts.pledged_size), + ..FrameSettings::from_options(opts) + }, + dicts, + )?; + counting.written + } + Mode::Decompress => decompress_stream( + &mut reader, + &mut sink, dicts, - ), - Mode::Decompress => decompress_stream(reader, writer, dicts), - Mode::Test | Mode::List | Mode::Train => { - unreachable!("test / list / train modes never stream to a writer here") + &DecodeSettings::from_options(opts), + )?, + Mode::Test => decompress_stream( + &mut reader, + io::sink(), + dicts, + &DecodeSettings::from_options(opts), + )?, + Mode::List | Mode::Train => unreachable!("list / train never stream"), + }; + sink.flush().wrap_err("failed to flush output")?; + Ok(Processed { + read: reader.read, + written, + }) +} + +/// One input into a destination that is already open and shared: stdout, or +/// the `-o` file being filled. An input that cannot be opened is reported +/// here and refused; a failure while streaming is returned for the caller to +/// end the run on, since the shared output now holds a partial frame. +fn stream_input_to( + opts: &Options, + dicts: &Dictionaries, + input: &Path, + mut sink: W, + destination: &str, + total: usize, +) -> Result { + let (name, processed) = if input == Path::new("-") { + ( + STDIN_MARK.to_string(), + stream_stdin(opts, dicts, &mut sink)?, + ) + } else { + match open_input(opts, input) { + Ok(Some((source, metadata))) => ( + input.display().to_string(), + stream_opened(opts, dicts, source, &metadata, &mut sink)?, + ), + Ok(None) => return Ok(Outcome::Skipped), + Err(err) => { + display!(opts.verbosity, 1, "zstd: {err}"); + return Ok(Outcome::Refused); + } } - } + }; + file_summary(opts, total, &name, destination, &processed); + Ok(Outcome::Done(processed)) } -/// Stream `reader` into `output` atomically: write to a sibling temp file, then -/// rename into place on success (and clean the temp up on failure). Honours the -/// `-f` overwrite gate. -fn write_stream_to_file( +/// Fill `output` atomically: `fill` writes into a sibling temporary, which is +/// renamed into place on success and removed on failure or interruption. +/// +/// Honours the `-f` overwrite gate the way the reference command does: an +/// existing output is refused outright below the default display level, where +/// no question can be asked, and asked about at it. `Ok(None)` is such a +/// refusal, already reported. `source` is the file whose permissions the +/// output takes; `None` (stdin, a concatenation) leaves the umask to decide. +fn write_output_file( opts: &Options, - mut reader: R, output: &Path, - size_hint: Option, - dicts: &Dictionaries, source: Option<&fs::Metadata>, -) -> Result<()> { + fill: impl FnOnce(&mut File) -> Result, +) -> Result> { ensure_regular_output_destination(output)?; if output.exists() && !opts.force { - bail!("{} already exists; use -f to overwrite", output.display()); + if opts.verbosity <= 1 { + display!( + opts.verbosity, + 1, + "zstd: {} already exists; not overwritten", + output.display() + ); + return Ok(None); + } + eprint!("zstd: {} already exists; ", output.display()); + if !confirm( + "overwrite (y/n) ? ", + "Not overwritten", + reads_stdin(&opts.inputs), + &mut io::stdin().lock(), + ) { + return Ok(None); + } } let (temp_path, temp_file) = create_temporary_output_file(output)?; + // From here until the rename, an interruption removes the temporary + // rather than leaving it beside the source. + interrupt::guard(&temp_path); + let abandon = |temp_path: &Path| { + let _ = fs::remove_file(temp_path); + interrupt::clear(); + }; // The output is as private as its source, whether it is created or replaced: // an archive of a 0600 secret must not arrive at whatever the umask allows, // and must not take a world-readable mode from the name it lands on either. // The reference command applies the source's mode in both directions. Only a - // source with no mode of its own — stdin — leaves the destination's alone. + // source with no mode of its own (stdin) leaves the destination's alone. // Asked of the temporary file, since that is the one renamed into place and // so the one whose own group the mode's group bits would admit. let source_permissions = match source { - Some(metadata) => permissions_from_source(metadata, &temp_path).inspect_err(|_err| { - let _ = fs::remove_file(&temp_path); - })?, + Some(metadata) => match permissions_from_source(metadata, &temp_path) { + Ok(permissions) => permissions, + Err(err) => { + abandon(&temp_path); + return Err(err); + } + }, None => None, }; if let Some(permissions) = source_permissions.clone() && let Err(err) = fs::set_permissions(&temp_path, permissions) { - let _ = fs::remove_file(&temp_path); + abandon(&temp_path); return Err(err).wrap_err("failed to apply the source's permissions to the output"); } - let result: Result<()> = (|| { + let result: Result = (|| { let mut sink = temp_file; - run_stream_core(opts, &mut reader, &mut sink, size_hint, dicts)?; + let value = fill(&mut sink)?; sink.flush().wrap_err("failed to flush output")?; - Ok(()) + Ok(value) })(); - if let Err(err) = result { - let _ = fs::remove_file(&temp_path); - return Err(err); - } - replace_output_file(&temp_path, output, source_permissions) + let value = match result { + Ok(value) => value, + Err(err) => { + abandon(&temp_path); + return Err(err); + } + }; + // `replace_output_file` removes the temporary itself when it fails. + let replaced = replace_output_file(&temp_path, output, source_permissions); + interrupt::clear(); + replaced.map(|()| Some(value)) } -/// Resolve the output path for a file input under the current mode. +/// Resolve the output path for a file input under the current mode: the +/// input's name with the suffix added or removed, placed beside it, under +/// `--output-dir-flat`, or under the mirrored tree of `--output-dir-mirror`. fn derive_output_path(opts: &Options, input: &Path) -> Result { if let Some(out) = &opts.output { return Ok(out.clone()); } - match opts.mode { - Mode::Compress => Ok(add_extension(input, ZSTD_SUFFIX)), + let beside = match opts.mode { + Mode::Compress => add_extension(input, ZSTD_SUFFIX), Mode::Decompress => { // Drop the extension as a path component rather than as text: a // path is bytes, and rebuilding it from a lossy string renames the // file it decompresses, with different inputs colliding on one // replacement-character name. - if input.extension() != Some(ZSTD_SUFFIX.trim_start_matches('.').as_ref()) { + let replacement = input.extension().and_then(|extension| { + DECOMPRESS_SUFFIXES + .iter() + .find(|(known, _)| extension == *known) + .map(|(_, replacement)| *replacement) + }); + let Some(replacement) = replacement else { bail!( - "{}: unknown suffix (expected {ZSTD_SUFFIX}); use -o to set the output", + "{}: unknown suffix (.zst/.tzst/.zstd expected). Can't derive the output \ + file name. Specify it with -o dstFileName. Ignoring.", input.display() ); - } - Ok(input.with_extension("")) + }; + input.with_extension(replacement) } Mode::Test | Mode::List | Mode::Train => { unreachable!("test / list / train modes never write an output file") } + }; + if let Some(root) = &opts.output_dir_mirror { + let verb = if opts.mode == Mode::Compress { + "compress" + } else { + "decompress" + }; + let directory = inputs::mirrored_output_dir(input, root).ok_or_else(|| { + eyre!( + "--output-dir-mirror cannot {verb} '{}' into '{}'", + input.display(), + root.display() + ) + })?; + return Ok(directory.join(beside.file_name().unwrap_or(beside.as_os_str()))); } + if let Some(directory) = &opts.output_dir { + return Ok(inputs::flat_output_path(&beside, directory)); + } + Ok(beside) } -fn process_file(opts: &Options, input: &Path, dicts: &Dictionaries) -> Result<()> { - let source = File::open(input) - .wrap_err_with(|| format!("failed to open input file {}", input.display()))?; - let metadata = source.metadata()?; - // Kept as the `u64` the filesystem reports: the work streams, so a file - // only has to fit the window, and narrowing to a pointer would refuse - // 4 GiB archives on 32-bit targets for no reason the work has. - let source_size = metadata.len(); - // Only a regular file's length says how many bytes will be read. A FIFO, - // a device or a socket reports something unrelated (commonly zero), and - // pledging that turns a perfectly good stream into a length mismatch. - let pledged_size = metadata.is_file().then_some(source_size); - let mut reader = ProgressMonitor::new(BufReader::new(source), source_size); +/// One named input into an output of its own, derived from its name. +fn process_file( + opts: &Options, + input: &Path, + dicts: &Dictionaries, + total: usize, +) -> Result { + let Some((source, metadata)) = open_input(opts, input)? else { + return Ok(Outcome::Skipped); + }; + let name = input.display().to_string(); - // Test mode: decompress into the void, report integrity. + // Test mode: decompress into the void, report what was there. if opts.mode == Mode::Test { - decompress_stream(&mut reader, io::sink(), dicts)?; - info!("{}: OK", input.display()); - return Ok(()); - } - - // stdout sink: bypass the temp-file dance. `--rm` still applies to the - // source file once the stream completes, so run the removal before - // returning rather than short-circuiting past it. - if opts.to_stdout { - let stdout = io::stdout(); - // Only compression produces binary; `-d` to a terminal is text the - // user asked for, which upstream also allows. - if matches!(opts.mode, Mode::Compress) { - guard_binary_stdout(stdout.is_terminal(), opts.force)?; - } - let mut out = stdout.lock(); - run_stream_core(opts, &mut reader, &mut out, pledged_size, dicts)?; - return remove_source_if_requested(opts, input); + let processed = stream_opened(opts, dicts, source, &metadata, io::sink())?; + file_summary(opts, total, &name, "", &processed); + return Ok(Outcome::Done(processed)); } let output = derive_output_path(opts, input)?; ensure_distinct_paths(input, &output)?; - write_stream_to_file(opts, reader, &output, pledged_size, dicts, Some(&metadata))?; - - info!("{} -> {}", input.display(), output.display()); - remove_source_if_requested(opts, input) + if let Some(root) = &opts.output_dir_mirror { + inputs::create_mirrored_dirs(input, root)?; + } + let Some(processed) = write_output_file(opts, &output, Some(&metadata), |sink| { + stream_opened(opts, dicts, source, &metadata, sink) + })? + else { + return Ok(Outcome::Refused); + }; + file_summary( + opts, + total, + &name, + &output.display().to_string(), + &processed, + ); + remove_source_if_requested(opts, input)?; + Ok(Outcome::Done(processed)) } /// Everything the command line says about how one frame is to be built. @@ -2252,7 +3416,7 @@ fn process_file(opts: &Options, input: &Path, dicts: &Dictionaries) -> Result<() /// Grouped rather than passed one by one: these travel together through every /// compression entry point, and a positional list this long invites the caller /// to line the arguments up wrong. -#[derive(Clone, Copy, Default)] +#[derive(Clone, Copy)] struct FrameSettings { /// Numeric compression level, ignored when `store` is set. level: i32, @@ -2267,6 +3431,32 @@ struct FrameSettings { long_window_log: Option, /// Soft block-size target from `--target-compressed-block-size`. target_block_size: Option, + /// Trailing XXH64 checksum (`--[no-]check`). + checksum: bool, + /// Whether a known length is recorded in the header (`--[no-]content-size`). + content_size_flag: bool, + /// Whether a dictionary frame records the dictionary's ID (`--no-dictID`). + dict_id_flag: bool, +} + +impl Default for FrameSettings { + /// The frame the reference COMMAND writes by default: checksummed, with + /// the content size and the dictionary ID in the header. (The library's + /// own default omits the checksum, which is why this is spelled out.) + fn default() -> Self { + Self { + level: CompressionLevel::DEFAULT_LEVEL, + store: false, + pledged_size: None, + size_hint: None, + long: false, + long_window_log: None, + target_block_size: None, + checksum: true, + content_size_flag: true, + dict_id_flag: true, + } + } } impl FrameSettings { @@ -2281,6 +3471,43 @@ impl FrameSettings { long: opts.long, long_window_log: opts.long_window_log, target_block_size: opts.target_block_size, + checksum: opts.checksum, + content_size_flag: opts.content_size_flag, + dict_id_flag: opts.dict_id_flag, + } + } +} + +/// How a stream is decoded: whether a stored checksum is compared, and what +/// happens to input that is not a zstd stream. +#[derive(Clone, Copy)] +struct DecodeSettings { + /// Compare the trailing checksum against the data (`--[no-]check`). + verify_checksum: bool, + /// Copy input that is not a zstd stream through unchanged rather than + /// failing on it (`--pass-through`). + pass_through: bool, +} + +impl Default for DecodeSettings { + fn default() -> Self { + Self { + verify_checksum: true, + pass_through: false, + } + } +} + +impl DecodeSettings { + /// What the command line asked for. Pass-through defaults to the + /// reference command's rule: on when forced and writing to stdout, which + /// is how `zstdcat` and `zstd -dcf` behave. + fn from_options(opts: &Options) -> Self { + Self { + verify_checksum: opts.checksum, + pass_through: opts + .pass_through + .unwrap_or(opts.force && writes_stdout(opts)), } } } @@ -2300,6 +3527,9 @@ fn compress_stream( long, long_window_log, target_block_size, + checksum, + content_size_flag, + dict_id_flag, } = settings; let compression_level = if store { CompressionLevel::Uncompressed @@ -2308,11 +3538,17 @@ fn compress_stream( }; let mut encoder = structured_zstd::encoding::StreamingEncoder::new(writer, compression_level); // The reference `zstd` COMMAND defaults the content checksum ON (unlike - // the library API, whose default is off and which our encoder mirrors) — - // set it explicitly so CLI output matches `zstd ` byte layout. + // the library API, whose default is off and which our encoder mirrors), so + // it is set explicitly either way: on by default, off under `--no-check`. + encoder + .set_content_checksum(checksum) + .wrap_err("failed to set the content checksum flag")?; encoder - .set_content_checksum(true) - .wrap_err("failed to enable content checksum")?; + .set_content_size_flag(content_size_flag) + .wrap_err("failed to set the content size flag")?; + encoder + .set_dictionary_id_flag(dict_id_flag) + .wrap_err("failed to set the dictionary ID flag")?; // A smaller block target is what the caller asked for when they want // bounded latency; the encoder clamps it to the format's own range. Zero // is the parameter's own way of saying "no target", so it stays off rather @@ -2371,7 +3607,15 @@ fn compress_stream( Ok(()) } -/// Streaming decompression core (file, stdout, or sink), optionally dict-primed. +/// The magic number every zstd frame opens with (RFC 8878 §3.1.1). +const FRAME_MAGIC: u32 = 0xFD2F_B528; + +/// The 16 magic numbers a skippable frame may open with, less their low +/// nibble (RFC 8878 §3.1.2). +const SKIPPABLE_MAGIC_BASE: u32 = 0x184D_2A50; + +/// Streaming decompression core (file, stdout, or sink), optionally +/// dict-primed. Returns the number of bytes written. /// /// A zstd stream is a sequence of frames: `cat a.zst b.zst` is a valid archive /// that decodes to `a` then `b`, and skippable frames may sit between them. The @@ -2379,36 +3623,62 @@ fn compress_stream( /// on whatever follows until the source is exhausted. The library's /// `read_to_end` walks frames too, but only by buffering the whole stream in /// memory, which a command-line tool handed a multi-gigabyte archive cannot do. +/// +/// Each frame is recognised by its magic number before a decoder is built on +/// it, the way the reference command looks before it decodes: input that is +/// not a zstd stream is then copied through under `--pass-through`, or +/// refused as an unknown format. fn decompress_stream( reader: R, mut writer: W, dicts: &Dictionaries, -) -> Result<()> { + settings: &DecodeSettings, +) -> Result { use structured_zstd::decoding::errors::{FrameDecoderError, ReadFrameHeaderError}; // Parsed once for the whole run rather than per stream or per frame: every // frame here is primed with the same dictionary, and the handle is shared, // so priming costs a reference rather than a rebuild. let handle = dicts.decoder.as_ref(); - // Buffered so the end of the stream can be told from the start of another - // frame without consuming the bytes that answer the question. let mut source = BufReader::new(reader); let mut frames = 0u64; + let mut written = 0u64; loop { - if source - .fill_buf() - .wrap_err("failed to read the compressed stream")? - .is_empty() - { + // The magic is read ahead of the decoder and handed back to it in + // front of the rest of the stream, so the source itself is never + // rewound. + let mut magic = [0u8; 4]; + let read = read_filling(&mut source, &mut magic)?; + if read == 0 { // End of the last frame is success; end before the first one means // the input never held a frame at all, which is not an archive that // decodes to nothing. if frames == 0 { - bail!("unexpected end of input: no zstd frame"); + bail!("unexpected end of file"); + } + return Ok(written); + } + let is_frame = read == 4 && { + let magic = u32::from_le_bytes(magic); + magic == FRAME_MAGIC || magic & 0xFFFF_FFF0 == SKIPPABLE_MAGIC_BASE + }; + if !is_frame { + if settings.pass_through { + writer + .write_all(&magic[..read]) + .wrap_err("failed to write the passed-through input")?; + written += read as u64; + written += io::copy(&mut source, &mut writer) + .wrap_err("failed to pass the input through")?; + return Ok(written); + } + if read < 4 { + bail!("unknown header"); } - return Ok(()); + bail!("unsupported format"); } frames += 1; + let mut stream = io::Cursor::new(magic).chain(&mut source); // Borrowed, not moved: a frame that turns out to be skippable leaves // the reader with us to step over it and carry on. let built = match &handle { @@ -2416,10 +3686,10 @@ fn decompress_stream( // the registration path does not: a frame may legitimately omit the // optional dictionary ID, and then nothing would select it. Some(h) => structured_zstd::decoding::StreamingDecoder::new_with_dictionary_handle( - &mut source, + &mut stream, h, ), - None => structured_zstd::decoding::StreamingDecoder::new(&mut source), + None => structured_zstd::decoding::StreamingDecoder::new(&mut stream), }; let mut decoder = match built { Ok(decoder) => decoder, @@ -2430,7 +3700,7 @@ fn decompress_stream( // Metadata a decoder is required to step over. The header is // already consumed, so only the payload is left to discard. let skipped = io::copy( - &mut source.by_ref().take(u64::from(length)), + &mut stream.by_ref().take(u64::from(length)), &mut io::sink(), ) .wrap_err("failed to skip a skippable frame")?; @@ -2443,14 +3713,20 @@ fn decompress_stream( }; // The library computes the digest but does not compare it, leaving the // decision to the caller. For a command-line tool that decision is - // made: upstream validates by default, and `-t` exists to answer - // exactly this question, so a frame whose stored checksum disagrees - // with its data has to fail rather than decode quietly. Read at the end - // of the frame, so setting it after construction is in time. + // made: the reference command validates by default, and `-t` exists to + // answer exactly this question, so a frame whose stored checksum + // disagrees with its data has to fail rather than decode quietly. + // `--no-check` asks for the opposite. Read at the end of the frame, so + // setting it after construction is in time. decoder .decoder_mut() - .set_content_checksum(structured_zstd::decoding::ContentChecksum::Verify); - io::copy(&mut decoder, &mut writer).wrap_err("streaming decompression failed")?; + .set_content_checksum(if settings.verify_checksum { + structured_zstd::decoding::ContentChecksum::Verify + } else { + structured_zstd::decoding::ContentChecksum::None + }); + written += + io::copy(&mut decoder, &mut writer).wrap_err("streaming decompression failed")?; } } @@ -2564,8 +3840,11 @@ fn create_temporary_output_file(output: &Path) -> Result<(PathBuf, File)> { { Ok(file) => return Ok((candidate, file)), Err(err) if err.kind() == ErrorKind::AlreadyExists => continue, + // Named after the output rather than the temporary: the reader + // knows the output they asked for, and the usual cause is its + // directory not being there. Err(err) => { - return Err(err).wrap_err("failed to create temporary output file"); + return Err(err).wrap_err_with(|| output.display().to_string()); } } } diff --git a/zstd/src/bin/structured-zstd/progress.rs b/zstd/src/bin/structured-zstd/progress.rs index cb4bddb8f..8dfb7f864 100644 --- a/zstd/src/bin/structured-zstd/progress.rs +++ b/zstd/src/bin/structured-zstd/progress.rs @@ -1,4 +1,4 @@ -//! Progress display for the command-line tool. +//! Progress counter for the command-line tool. //! //! Written against `std` alone. A progress bar is a few dozen lines of //! formatting, and the crates that provide one are the reason a tool would @@ -7,10 +7,12 @@ use std::{ fmt::Write as _, - io::{IsTerminal, Read, Write as _}, + io::{Read, Write as _}, time::{Duration, Instant}, }; +use super::display::HumanSize; + /// Redraw at most this often. The work between reads is measured in /// microseconds, so repainting per read would cost more than the compression. const REDRAW_INTERVAL: Duration = Duration::from_millis(125); @@ -18,8 +20,11 @@ const REDRAW_INTERVAL: Duration = Duration::from_millis(125); /// Width of the drawn bar, in characters. const BAR_WIDTH: usize = 32; -/// A generic wrapper around a reader that keeps track of how many bytes have been read -/// from the total. +/// Room the bar line takes on the terminal, cleared before the summary. +const LINE_WIDTH: usize = BAR_WIDTH + 48; + +/// A reader that counts what passes through it and, when asked, draws how far +/// along the total that is. pub struct ProgressMonitor { /// The total amount that the reader will read. Counted in `u64` rather /// than `usize`: both directions stream, so a file has to fit the window, @@ -28,39 +33,37 @@ pub struct ProgressMonitor { pub total: u64, /// Amount read so far pub read: u64, - /// Whether the summary has been printed, which happens once the reader - /// says it is done. + /// Whether the reader has reported its end. pub finished: bool, /// The internal reader reader: R, - started: Instant, last_draw: Instant, - /// Only draw when stderr is a terminal: piped output must stay clean. - interactive: bool, + /// Whether the bar is drawn at all. Decided by the caller from the + /// verbosity, the `--progress` setting and where stderr goes, so piped + /// stderr and `-q` runs stay clean of carriage returns. + shown: bool, } impl ProgressMonitor { - /// Create a new progress monitor, initialized with zero bytes read - pub fn new(reader: R, size: u64) -> Self { - let now = Instant::now(); + /// Wrap `reader`, expecting `size` bytes; `shown` says whether to draw. + pub fn new(reader: R, size: u64, shown: bool) -> Self { Self { reader, total: size, read: 0, - started: now, - last_draw: now, - interactive: std::io::stderr().is_terminal(), + last_draw: Instant::now(), + shown, finished: false, } } /// Repaint the bar in place, throttled to [`REDRAW_INTERVAL`]. - fn draw(&mut self, force: bool) { - if !self.interactive { + fn draw(&mut self) { + if !self.shown { return; } let now = Instant::now(); - if !force && now.duration_since(self.last_draw) < REDRAW_INTERVAL { + if now.duration_since(self.last_draw) < REDRAW_INTERVAL { return; } self.last_draw = now; @@ -70,7 +73,7 @@ impl ProgressMonitor { (self.read as f64 / self.total as f64).clamp(0.0, 1.0) }; let filled = (fraction * BAR_WIDTH as f64).round() as usize; - let mut line = String::with_capacity(BAR_WIDTH + 48); + let mut line = String::with_capacity(LINE_WIDTH); line.push('\r'); line.push('['); for i in 0..BAR_WIDTH { @@ -79,8 +82,8 @@ impl ProgressMonitor { let _ = write!( &mut line, "] {}/{}", - fmt_size(self.read as f64), - fmt_size(self.total as f64) + HumanSize::new(self.read, false), + HumanSize::new(self.total, false) ); let mut err = std::io::stderr().lock(); let _ = err.write_all(line.as_bytes()); @@ -92,36 +95,22 @@ impl ProgressMonitor { /// The end of the work is the reader saying it has no more, and nothing /// else: the total is a number from a directory entry, which a FIFO reports /// as zero and which a file that grew after it was measured has already - /// passed. Ending on it would print the summary over a stream still being - /// read, and the bytes that arrived afterwards would go unreported. Waiting - /// for the reader costs one more read, which every caller here performs to - /// find the end anyway. + /// passed. Ending on it would clear the bar over a stream still being read. + /// Waiting for the reader costs one more read, which every caller here + /// performs to find the end anyway. fn update(&mut self, last_read: usize) { let done = last_read == 0; if done && !self.finished { self.finished = true; - // Clear the bar's line before the summary, or the leftovers of the - // longer bar line trail after it. - if self.interactive { + // Clear the bar's line, or its leftovers trail after whatever the + // caller prints next. + if self.shown { let mut err = std::io::stderr().lock(); - let _ = write!(err, "\r{:width$}\r", "", width = BAR_WIDTH + 48); + let _ = write!(err, "\r{:width$}\r", "", width = LINE_WIDTH); let _ = err.flush(); } - // Reported from what was actually read: the declared total can be - // zero for a stream, or stale for a file that changed size. - let elapsed = self.started.elapsed(); - let rate = if elapsed.as_secs_f64() > 0.0 { - fmt_size(self.read as f64 / elapsed.as_secs_f64()) - } else { - fmt_size(self.read as f64) - }; - eprintln!( - "processed {} in {} ({rate}/s avg)", - fmt_size(self.read as f64), - fmt_duration(elapsed), - ); } else { - self.draw(false); + self.draw(); } } } @@ -137,7 +126,7 @@ impl Read for ProgressMonitor { // `Ok(0)` means the end of the stream only when there was room to read // into: the contract gives the same answer for an empty buffer, which // says nothing about the reader. Taking it as the end would finish the - // monitor before the work did, and the summary would never come. + // monitor before the work did. if !buf.is_empty() { self.update(out); } @@ -145,84 +134,5 @@ impl Read for ProgressMonitor { } } -/// Converts a quantity in bytes to a human readable size, "GiB, MiB, KiB, etc" -pub fn fmt_size(size_in_bytes: f64) -> String { - let units = ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]; - let order_of_magnitude = (size_in_bytes).log10() as usize; - // Overflow to the next order of magnitude if there are more than `upper_bound` figures - // before the decimal - let upper_bound = 3; - let unit_index = (order_of_magnitude / upper_bound).clamp(0, units.len() - 1); - let decimal = size_in_bytes / 2_f64.powi((unit_index * 10) as i32); - // Only use a decimal if displaying a unit larger than a byte - if unit_index > 0 { - format!("{:.2}{}", decimal, units[unit_index]) - } else { - format!("{:.0}{}", decimal, units[unit_index]) - } -} - -/// Converts a [`std::time::Duration`] to a human readable format -fn fmt_duration(duration: Duration) -> String { - let as_secs = duration.as_secs_f64(); - let as_min = (as_secs / 60.0).floor() as usize; - // When displayed in long form, the value shown - let mut secs_portion: f64 = as_secs % 60.0; - let mut min_portion: usize = ((as_secs - secs_portion) as usize / 60) % 60; - // Counted on rather than wrapped: hours are the largest unit here, so a - // sixtieth of one is still an hour and not the start of something bigger. - // Wrapping them leaves a two-day run with no hours, no minutes and no - // seconds to print — an empty summary. - let mut hr_portion: usize = (as_min - min_portion) / 60; - - // The seconds are shown rounded, and rounding can reach a full minute: - // 1m 59.5s printed as it stands reads `1m 60s`, which is no duration. What - // counts is the value as it will be WRITTEN, at the precision the branches - // below pick from the same two thresholds — 59.5s keeps its decimal and is - // not a minute, while 59.96s shown to one place is. The carry happens - // before anything is written, so it runs up through the minutes as well: - // 59m 59.6s is an hour. - let shown_decimals = if as_secs > 60.0 { - 0 - } else if secs_portion > 4.0 { - 1 - } else { - 2 - }; - let scale = 10f64.powi(shown_decimals); - if (secs_portion * scale).round() / scale >= 60.0 { - secs_portion = 0.0; - min_portion += 1; - if min_portion == 60 { - min_portion = 0; - hr_portion += 1; - } - } - - let mut output = String::with_capacity(8); - if hr_portion > 0 { - write!(&mut output, "{hr_portion}h ").unwrap(); - } - if min_portion > 0 { - write!(&mut output, "{min_portion}m ").unwrap(); - } - // Formatting for seconds is fairly manual - // to provide a "useful" level of precision - if as_secs > 60.0 && secs_portion != 0.0 { - // Zero points of precision - write!(&mut output, "{:.0}s", secs_portion.round()).unwrap(); - } else if secs_portion > 4.0 { - // One point of precision - write!(&mut output, "{secs_portion:.1}s").unwrap(); - } else if secs_portion > 1.0 { - // Two points of precision - write!(&mut output, "{secs_portion:.2}s").unwrap(); - } else if secs_portion > 0.0 { - // Display as ms with two units of precision - write!(&mut output, "{:.2}ms", secs_portion * 1000.0).unwrap(); - } - output.trim().to_string() -} - #[cfg(test)] mod tests; diff --git a/zstd/src/bin/structured-zstd/progress/tests.rs b/zstd/src/bin/structured-zstd/progress/tests.rs index 01cc1ea36..59cef6d07 100644 --- a/zstd/src/bin/structured-zstd/progress/tests.rs +++ b/zstd/src/bin/structured-zstd/progress/tests.rs @@ -1,57 +1,74 @@ -use std::time::Duration; +use std::io::{self, Read}; -use super::{fmt_duration, fmt_size}; +use super::ProgressMonitor; +/// The monitor is done when the reader is done, and "done" is the reader +/// saying so, not the byte count matching a number from a directory entry. A +/// FIFO reports zero, a file can shrink after it was measured, and in both +/// cases a monitor that waits for the two to meet waits forever. #[test] -fn human_readable_filesize() { - // Bytes - assert_eq!(&fmt_size(100.0), "100B"); - // Kibibytes - assert_eq!(&fmt_size(12.0 * 2.0_f64.powi(10)), "12.00KiB"); - // Mebibytes - assert_eq!(&fmt_size(7.0 * 2.0_f64.powi(20)), "7.00MiB"); - // Gibibytes - assert_eq!(&fmt_size(123.0 * 2.0_f64.powi(30)), "123.00GiB"); +fn progress_finishes_when_the_reader_does_not_when_the_count_matches() { + // A reader with more bytes than the total it was created with. + let mut monitor = ProgressMonitor::new(&b"bytes that were not counted"[..], 0, false); + let mut sink = Vec::new(); + io::copy(&mut monitor, &mut sink).expect("copying must succeed"); + assert!( + monitor.finished, + "the reader reached its end, so the monitor has to be finished too" + ); + assert_eq!(monitor.read, sink.len() as u64, "and count what it read"); } +/// `Read::read` answers `Ok(0)` for an empty buffer as well as for the end of +/// the stream, as the contract says. Treating the first as the second finishes +/// the monitor before any bytes have moved. #[test] -fn human_readable_duration() { - assert_eq!(&fmt_duration(Duration::from_millis(7)), "7.00ms"); - assert_eq!(&fmt_duration(Duration::from_millis(1500)), "1.50s"); - assert_eq!(&fmt_duration(Duration::from_secs(30)), "30.0s"); - assert_eq!(&fmt_duration(Duration::from_secs(90)), "1m 30s"); - assert_eq!(&fmt_duration(Duration::from_secs(5 * 60)), "5m"); - assert_eq!(&fmt_duration(Duration::from_secs(3 * 60 * 60)), "3h"); - assert_eq!( - &fmt_duration(Duration::from_secs(60 * 60 + 20 * 60 + 30)), - "1h 20m 30s" +fn an_empty_buffer_read_is_not_the_end_of_the_stream() { + let mut monitor = ProgressMonitor::new(&b"payload"[..], 7, false); + assert_eq!(monitor.read(&mut []).unwrap(), 0); + assert!( + !monitor.finished, + "an empty buffer says nothing about the reader" + ); + + let mut buf = [0u8; 7]; + assert_eq!(monitor.read(&mut buf).unwrap(), 7); + assert!( + !monitor.finished, + "a total taken from a directory entry does not end the stream" ); + assert_eq!(monitor.read(&mut buf).unwrap(), 0); + assert!(monitor.finished, "the reader saying so does"); } -/// Hours are the largest unit this prints, so they count on rather than wrap. -/// Wrapping them at 60 leaves a two-and-a-half-day run with nothing at all to -/// show: no hours, no minutes, no seconds, an empty summary. +/// Both directions stream, so a file only has to fit the window, never memory. +/// Measuring its length in `usize` puts a 4 GiB ceiling on 32-bit targets that +/// has nothing to do with what the work needs: the progress counter would be +/// deciding which archives the tool can open. #[test] -fn hours_do_not_wrap() { - assert_eq!(&fmt_duration(Duration::from_secs(60 * 60 * 60)), "60h"); +fn a_file_length_is_not_narrowed_to_the_pointer_width() { + let huge = u64::from(u32::MAX) + 1; + let monitor = ProgressMonitor::new(&b""[..], huge, false); assert_eq!( - &fmt_duration(Duration::from_secs(61 * 60 * 60 + 5 * 60)), - "61h 5m" + monitor.total, huge, + "a length larger than a 32-bit pointer must survive" ); } -/// The seconds are shown rounded, and a value that rounds up to a full minute -/// belongs in the minutes: printed as it stands it reads `1m 60s`, a duration -/// no clock shows. The carry runs all the way up, so 59m 59.6s is an hour. +/// A read error is the reader's to report; the monitor passes it through +/// without counting bytes that never arrived or declaring the stream done. #[test] -fn rounded_seconds_carry_into_the_next_minute() { - assert_eq!(&fmt_duration(Duration::from_millis(119_500)), "2m"); - // Under a minute the seconds carry a decimal, so the carry is what that - // decimal rounds to: 59.96 shown to one place is a minute. - assert_eq!(&fmt_duration(Duration::from_millis(59_960)), "1m"); - assert_eq!( - &fmt_duration(Duration::from_millis(3_599_600)), - "1h", - "the carry runs past the minutes as well" - ); +fn a_read_error_passes_through_uncounted() { + struct Broken; + impl Read for Broken { + fn read(&mut self, _buf: &mut [u8]) -> io::Result { + Err(io::Error::other("disk on fire")) + } + } + let mut monitor = ProgressMonitor::new(Broken, 10, false); + let mut buf = [0u8; 4]; + let err = monitor.read(&mut buf).expect_err("the error must surface"); + assert_eq!(err.to_string(), "disk on fire"); + assert_eq!(monitor.read, 0, "nothing was read"); + assert!(!monitor.finished, "an error is not the end of the stream"); } diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 72569787c..961a1c4a8 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -34,14 +34,78 @@ fn prepared_dict(raw: &[u8]) -> Dictionaries { Dictionaries::prepare(Some(raw), true, true).expect("the fixture dictionary must parse") } +/// What a plain `zstd` invocation presets, before any flag. +fn plain() -> ProgramPreset { + program_preset("zstd") +} + +/// Parse `args` as the plain `zstd` command with the built-in default level. fn parse(args: &[&str]) -> Result { - let owned: Vec = args.iter().map(std::ffi::OsString::from).collect(); - match parse_args(&owned, Mode::Compress, false)? { - Parsed::Run(opts) => Ok(opts), + parse_as(&plain(), CompressionLevel::DEFAULT_LEVEL, args) +} + +/// Parse `args` under `preset` (an `argv[0]` dispatch) with `default_level` +/// standing in for the built-in default (`ZSTD_CLEVEL`). +fn parse_as(preset: &ProgramPreset, default_level: i32, args: &[&str]) -> Result { + let owned: Vec = args.iter().map(OsString::from).collect(); + match parse_args(&owned, preset, default_level).map_err(|failure| failure.error)? { + Parsed::Run(opts) => Ok(*opts), Parsed::Handled => bail!("parse handled (help/version) unexpectedly"), } } +/// A scratch directory unique to the test, removed when dropped. +struct Scratch(PathBuf); + +impl Scratch { + fn new(tag: &str) -> Self { + let dir = std::env::temp_dir().join(format!("szstd-cli-{tag}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + Self(dir) + } + + fn path(&self) -> &Path { + &self.0 + } + + fn file(&self, relative: &str, content: &[u8]) -> PathBuf { + let path = self.0.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, content).unwrap(); + path + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// A frame compressed the way the tool compresses by default. +fn frame_of(payload: &[u8]) -> Vec { + let mut frame = Vec::new(); + compress_stream( + payload, + &mut frame, + &FrameSettings { + level: 3, + ..FrameSettings::default() + }, + &no_dict(), + ) + .expect("compressing the fixture must succeed"); + frame +} + +/// Decode `stream` with the default decode settings. +fn decoded(stream: &[u8]) -> Result> { + let mut out = Vec::new(); + decompress_stream(stream, &mut out, &no_dict(), &DecodeSettings::default())?; + Ok(out) +} + /// A filename is bytes, and on Unix those bytes need not be UTF-8. Reading the /// command line as text rejects such a name before any of the byte-preserving /// path handling can run — and does it by panicking, which is not an answer. @@ -53,8 +117,11 @@ fn a_non_utf8_argument_survives_parsing() { let name = OsStr::from_bytes(b"weird\xffname.zst"); let args: Vec = vec![OsString::from("-d"), name.to_os_string()]; - let opts = match parse_args(&args, Mode::Compress, false).expect("parsing must not fail") { - Parsed::Run(opts) => opts, + let opts = match parse_args(&args, &plain(), CompressionLevel::DEFAULT_LEVEL) + .map_err(|failure| failure.error) + .expect("parsing must not fail") + { + Parsed::Run(opts) => *opts, Parsed::Handled => panic!("unexpected help/version"), }; assert_eq!( @@ -81,8 +148,11 @@ fn attached_path_options_keep_their_bytes() { let mut arg = flag.to_vec(); arg.extend_from_slice(name.as_bytes()); let args = vec![OsString::from(OsStr::from_bytes(&arg)), OsString::from("f")]; - match parse_args(&args, Mode::Compress, false).expect("parsing must not fail") { - Parsed::Run(opts) => opts, + match parse_args(&args, &plain(), CompressionLevel::DEFAULT_LEVEL) + .map_err(|failure| failure.error) + .expect("parsing must not fail") + { + Parsed::Run(opts) => *opts, Parsed::Handled => panic!("unexpected help/version"), } }; @@ -93,6 +163,22 @@ fn attached_path_options_keep_their_bytes() { attached(b"--use-dict=").dict.as_deref(), Some(expected.as_path()) ); + // The options that take a directory or a list name are paths too. + assert_eq!( + attached(b"--output-dir-flat=").output_dir.as_deref(), + Some(expected.as_path()) + ); + assert_eq!( + attached(b"--output-dir-mirror=") + .output_dir_mirror + .as_deref(), + Some(expected.as_path()) + ); + assert_eq!( + attached(b"--filelist=").filelists, + vec![expected.clone()], + "a list name keeps its bytes as well" + ); } #[test] @@ -119,9 +205,37 @@ fn list_file_walks_multi_frame_archive_by_seeking() { let dir = std::env::temp_dir(); let path = dir.join(format!("szstd-list-test-{}.zst", std::process::id())); fs::write(&path, &archive).unwrap(); - let result = list_file(&path); + let result = list_file(&path, false, 0); let _ = fs::remove_file(&path); - result.expect("list_file must walk both frames without error"); + let summary = result.expect("list_file must walk both frames without error"); + assert_eq!(summary.frames, 2); + assert_eq!(summary.decompressed, Some(4096 + 38)); + assert!( + !summary.check && summary.checksum.is_none(), + "library-default frames carry no checksum, so the archive reports none" + ); + + // The tool's own frames do, and the stored value of the last frame that + // has one is kept for `-lv` to print. + let mut checked = frame_of(&[7u8; 4096]); + let trailer: [u8; 4] = checked[checked.len() - 4..].try_into().unwrap(); + checked.extend_from_slice(&compress_slice_to_vec( + b"a second frame without a checksum", + CompressionLevel::Default, + )); + fs::write(&path, &checked).unwrap(); + let result = list_file(&path, false, 0); + let _ = fs::remove_file(&path); + let summary = result.expect("a mixed archive lists"); + assert!( + summary.check, + "one checksummed frame makes the archive checked" + ); + assert_eq!( + summary.checksum, + Some(trailer), + "the checksummed frame's trailer is kept" + ); } /// The `DictID` column holds one id, and a concatenated archive can need more @@ -743,9 +857,9 @@ fn a_new_output_inherits_the_source_permissions() { fs::write(&input, b"secret payload").unwrap(); fs::set_permissions(&input, fs::Permissions::from_mode(0o600)).unwrap(); - let mut opts = parse(&["-3", "f"]).unwrap(); + let mut opts = parse(&["-3", "-q", "f"]).unwrap(); opts.inputs = vec![input.clone()]; - let result = process_file(&opts, &input, &no_dict()); + let result = process_file(&opts, &input, &no_dict(), 1); let output = PathBuf::from(format!("{}.zst", input.display())); let mode = fs::metadata(&output).map(|m| m.permissions().mode() & 0o777); @@ -917,10 +1031,10 @@ fn compressing_a_setuid_file_does_not_produce_a_setuid_archive() { return; } - let mut opts = parse(&["-3", "-f", "s"]).unwrap(); + let mut opts = parse(&["-3", "-f", "-q", "s"]).unwrap(); opts.inputs = vec![source.clone()]; opts.output = Some(archive.clone()); - let compressed = process_file(&opts, &source, &no_dict()); + let compressed = process_file(&opts, &source, &no_dict(), 1); let mode = fs::metadata(&archive).map(|m| m.permissions().mode() & 0o7777); let _ = fs::remove_file(&source); @@ -989,10 +1103,10 @@ fn replacing_a_file_does_not_restore_its_old_permissions() { let archive = dir.join(format!("szstd-replace-arch-{}", std::process::id())); fs::write(&archive, b"the archive that was here before").unwrap(); fs::set_permissions(&archive, fs::Permissions::from_mode(0o644)).unwrap(); - let mut opts = parse(&["-3", "-f", "s"]).unwrap(); + let mut opts = parse(&["-3", "-f", "-q", "s"]).unwrap(); opts.inputs = vec![sample.clone()]; opts.output = Some(archive.clone()); - let compressed = process_file(&opts, &sample, &no_dict()); + let compressed = process_file(&opts, &sample, &no_dict(), 1); let archive_mode = fs::metadata(&archive).map(|m| m.permissions().mode() & 0o777); let _ = fs::remove_file(&sample); @@ -1012,46 +1126,6 @@ fn replacing_a_file_does_not_restore_its_old_permissions() { ); } -/// The summary is printed when the reader is done, and "done" is the reader -/// saying so — not the byte count matching a number from a directory entry. A -/// FIFO reports zero, a file can shrink after it was measured, and in both -/// cases a monitor that waits for the two to meet waits forever. -#[test] -fn progress_finishes_when_the_reader_does_not_when_the_count_matches() { - // A reader with more bytes than the total it was created with. - let mut monitor = ProgressMonitor::new(&b"bytes that were not counted"[..], 0); - let mut sink = Vec::new(); - io::copy(&mut monitor, &mut sink).expect("copying must succeed"); - assert!( - monitor.finished, - "the reader reached its end, so the monitor has to be finished too" - ); - assert_eq!(monitor.read, sink.len() as u64, "and count what it read"); -} - -/// `Read::read` answers `Ok(0)` for an empty buffer as well as for the end of -/// the stream — the contract says so. Treating the first as the second finishes -/// the monitor before any bytes have moved, and the summary for the work that -/// followed is then never printed. -#[test] -fn an_empty_buffer_read_is_not_the_end_of_the_stream() { - let mut monitor = ProgressMonitor::new(&b"payload"[..], 7); - assert_eq!(monitor.read(&mut []).unwrap(), 0); - assert!( - !monitor.finished, - "an empty buffer says nothing about the reader" - ); - - let mut buf = [0u8; 7]; - assert_eq!(monitor.read(&mut buf).unwrap(), 7); - assert!( - !monitor.finished, - "a total taken from a directory entry does not end the stream" - ); - assert_eq!(monitor.read(&mut buf).unwrap(), 0); - assert!(monitor.finished, "the reader saying so does"); -} - /// Listing walks frame headers and training builds a dictionary from samples; /// neither reads the one `-D` names. Loading it anyway fails a listing over a /// missing file that has nothing to do with it, and spends time and memory on a @@ -1084,20 +1158,6 @@ fn a_dictionary_is_loaded_only_where_it_is_used() { assert!(load_dictionary(&decompressing).is_err()); } -/// Both directions stream, so a file only has to fit the window, never memory. -/// Measuring its length in `usize` puts a 4 GiB ceiling on 32-bit targets that -/// has nothing to do with what the work needs — the progress counter would be -/// deciding which archives the tool can open. -#[test] -fn a_file_length_is_not_narrowed_to_the_pointer_width() { - let huge = u64::from(u32::MAX) + 1; - let monitor = ProgressMonitor::new(&b""[..], huge); - assert_eq!( - monitor.total, huge, - "a length larger than a 32-bit pointer must survive" - ); -} - /// `--stream-size` exists precisely for inputs whose length cannot be stat'd. /// A named FIFO is one of those, so the per-file size being unavailable is the /// moment the option matters most — dropping it there leaves the pledge @@ -1112,8 +1172,16 @@ fn an_explicit_stream_size_survives_an_unstattable_input() { let mut frame = Vec::new(); // `None` is what a FIFO or device yields: no reliable size from metadata. - run_stream_core(&opts, &payload[..], &mut frame, None, &no_dict()) - .expect("compressing must succeed"); + let processed = stream( + &opts, + &no_dict(), + ProgressMonitor::new(&payload[..], 0, false), + None, + &mut frame, + ) + .expect("compressing must succeed"); + assert_eq!(processed.read, payload.len() as u64); + assert_eq!(processed.written, frame.len() as u64); let info = read_frame_header_info(&frame, false).expect("the frame header must parse"); assert_eq!( @@ -1257,8 +1325,19 @@ fn zero_means_unset_where_the_api_says_it_does() { /// an archive; upstream calls it an unexpected end of file. #[test] fn an_empty_stream_is_not_a_valid_archive() { - decompress_stream(&b""[..], io::sink(), &no_dict()) + decompress_stream(&b""[..], io::sink(), &no_dict(), &DecodeSettings::default()) .expect_err("an empty input carries no frame to decode"); + // Even under pass-through: an empty file is not an archive either. + decompress_stream( + &b""[..], + io::sink(), + &no_dict(), + &DecodeSettings { + verify_checksum: true, + pass_through: true, + }, + ) + .expect_err("nothing to pass through is still nothing"); } /// Skippable frames sit inside ordinary archives — seekable-zstd puts its index @@ -1282,9 +1361,10 @@ fn list_file_walks_past_skippable_frames() { let dir = std::env::temp_dir(); let path = dir.join(format!("szstd-list-skip-{}.zst", std::process::id())); fs::write(&path, &archive).unwrap(); - let result = list_file(&path); + let result = list_file(&path, true, 0); let _ = fs::remove_file(&path); - result.expect("a skippable frame between two frames must not fail the listing"); + let summary = result.expect("a skippable frame between two frames must not fail the listing"); + assert_eq!((summary.frames, summary.skips), (3, 1)); } /// Frame_Content_Size is a declaration, not a measurement: a few bytes of @@ -1315,21 +1395,58 @@ fn list_file_refuses_a_content_size_total_that_overflows() { let dir = std::env::temp_dir(); let path = dir.join(format!("szstd-list-overflow-{}.zst", std::process::id())); fs::write(&path, &archive).unwrap(); - let result = list_file(&path); + let result = list_file(&path, false, 0); let _ = fs::remove_file(&path); result.expect_err("a total that cannot be represented must be reported, not wrapped"); } #[test] fn argv0_unzstd_defaults_to_decompress() { - assert_eq!(program_mode("unzstd"), (Mode::Decompress, false)); - assert_eq!(program_mode("/usr/bin/unzstd"), (Mode::Decompress, false)); -} - -#[test] -fn argv0_zstdcat_decompresses_to_stdout() { - assert_eq!(program_mode("zstdcat"), (Mode::Decompress, true)); - assert_eq!(program_mode("zstd"), (Mode::Compress, false)); + let preset = program_preset("unzstd"); + assert_eq!(preset.mode, Mode::Decompress); + assert!(!preset.to_stdout); + assert!(!preset.force); + assert_eq!(program_preset("/usr/bin/unzstd").mode, Mode::Decompress); + assert_eq!(program_preset("unzstd.exe").mode, Mode::Decompress); +} + +/// `zstdcat` is `zstd -dcf` with pass-through and the quiet level, as the +/// reference command sets it up: a `zcat` replacement copies a plain file +/// through rather than refusing it, and says nothing on success. +#[test] +fn argv0_zstdcat_decompresses_to_stdout_passing_plain_input_through() { + for name in ["zstdcat", "zcat", "/usr/local/bin/zstdcat"] { + let preset = program_preset(name); + assert_eq!(preset.mode, Mode::Decompress, "{name}"); + assert!(preset.to_stdout, "{name}"); + assert!(preset.force, "{name}"); + assert_eq!(preset.pass_through, Some(true), "{name}"); + assert_eq!(preset.verbosity, 1, "{name}"); + } + let plain = program_preset("zstd"); + assert_eq!(plain.mode, Mode::Compress); + assert!(!plain.to_stdout && !plain.force); + assert_eq!(plain.pass_through, None); + assert_eq!(plain.verbosity, DEFAULT_LEVEL); + // `zstdmt` compresses like `zstd`; its worker count has no effect here. + assert_eq!(program_preset("zstdmt").mode, Mode::Compress); +} + +/// The preset is the starting point, not the last word: `zstdcat -v` still +/// raises the level, and `--no-pass-through` still turns pass-through off. +#[test] +fn flags_adjust_the_argv0_preset() { + let cat = program_preset("zstdcat"); + let quiet = parse_as(&cat, 3, &["a.zst"]).unwrap(); + assert_eq!(quiet.verbosity, 1); + assert!(quiet.force && quiet.follow_links); + assert!( + DecodeSettings::from_options(&quiet).pass_through, + "zstdcat passes unknown input through" + ); + let louder = parse_as(&cat, 3, &["-v", "--no-pass-through", "a.zst"]).unwrap(); + assert_eq!(louder.verbosity, 2); + assert!(!DecodeSettings::from_options(&louder).pass_through); } #[test] @@ -1433,9 +1550,13 @@ fn dict_and_output_take_values() { assert_eq!(opts.dict, Some(PathBuf::from("dict.bin"))); } +/// Several inputs into one `-o` is a concatenation, which the reference +/// command permits after a warning; the command line itself is not wrong. #[test] -fn output_rejects_multiple_inputs() { - assert!(parse(&["-o", "out.zst", "a.txt", "b.txt"]).is_err()); +fn output_accepts_multiple_inputs_for_concatenation() { + let opts = parse(&["-o", "out.zst", "a.txt", "b.txt"]).unwrap(); + assert_eq!(opts.inputs.len(), 2); + assert_eq!(opts.output, Some(PathBuf::from("out.zst"))); } #[test] @@ -1761,50 +1882,307 @@ fn fast_and_long_match_exactly_not_by_prefix() { assert!(parse(&["-19", "--long=27"]).unwrap().long); } +/// The wire-format switches reach the frame: `--no-check` drops the +/// checksum, `--no-content-size` the Frame_Content_Size field, and the later +/// of `--no-check` / `-C` wins, as the last flag does elsewhere. +#[test] +fn wire_format_flags_reach_the_frame_header() { + use structured_zstd::decoding::{FrameContentSize, read_frame_header_info}; + + let plain = parse(&["f"]).unwrap(); + assert!(plain.checksum && plain.content_size_flag && plain.dict_id_flag); + let stripped = parse(&["--no-check", "--no-content-size", "--no-dictID", "f"]).unwrap(); + assert!(!stripped.checksum && !stripped.content_size_flag && !stripped.dict_id_flag); + assert!(parse(&["--no-check", "-C", "f"]).unwrap().checksum); + assert!(parse(&["--no-check", "--check", "f"]).unwrap().checksum); + assert!( + parse(&["--no-content-size", "--content-size", "f"]) + .unwrap() + .content_size_flag + ); + + let payload = b"payload whose header is inspected"; + let mut bare = Vec::new(); + compress_stream( + &payload[..], + &mut bare, + &FrameSettings { + level: 3, + pledged_size: Some(payload.len() as u64), + checksum: false, + content_size_flag: false, + ..FrameSettings::default() + }, + &no_dict(), + ) + .unwrap(); + let info = read_frame_header_info(&bare, false).unwrap(); + assert!(!info.content_checksum, "--no-check leaves the checksum out"); + assert_eq!( + info.content_size, + FrameContentSize::Unknown, + "--no-content-size leaves the size out of the header" + ); + + let mut full = Vec::new(); + compress_stream( + &payload[..], + &mut full, + &FrameSettings { + level: 3, + pledged_size: Some(payload.len() as u64), + ..FrameSettings::default() + }, + &no_dict(), + ) + .unwrap(); + let info = read_frame_header_info(&full, false).unwrap(); + assert!(info.content_checksum, "the default frame is checksummed"); + assert_eq!( + info.content_size, + FrameContentSize::Known(payload.len() as u64) + ); +} + +/// `--no-dictID` keeps the dictionary's ID out of a dictionary frame: the +/// decoder then has to be told which dictionary to use, and cannot check. #[test] -fn unsupported_format_flags_are_rejected_not_ignored() { - // These change the wire format but are not wired through yet — accepting - // them silently would hand the caller the wrong frame layout. - assert!(parse(&["--no-check"]).is_err()); - assert!(parse(&["--no-content-size"]).is_err()); - assert!(parse(&["--no-dictID"]).is_err()); - // Verbosity aliases stay honest no-ops. - assert!(parse(&["--quiet"]).is_ok()); - assert!(parse(&["--verbose"]).is_ok()); +fn no_dict_id_leaves_the_id_out_of_a_dictionary_frame() { + use structured_zstd::decoding::read_frame_header_info; + + let raw = include_bytes!("../../../dict_tests/dictionary"); + let dicts = prepared_dict(raw); + let payload: Vec = (0..4000u32).map(|i| (i % 97) as u8).collect(); + let mut with_id = Vec::new(); + compress_stream( + payload.as_slice(), + &mut with_id, + &FrameSettings { + level: 3, + ..FrameSettings::default() + }, + &dicts, + ) + .unwrap(); + assert!( + read_frame_header_info(&with_id, false) + .unwrap() + .dictionary_id + .is_some(), + "a dictionary frame names its dictionary by default" + ); + let mut anonymous = Vec::new(); + compress_stream( + payload.as_slice(), + &mut anonymous, + &FrameSettings { + level: 3, + dict_id_flag: false, + ..FrameSettings::default() + }, + &dicts, + ) + .unwrap(); + assert!( + read_frame_header_info(&anonymous, false) + .unwrap() + .dictionary_id + .is_none(), + "--no-dictID keeps the id out" + ); + // Told the dictionary outright, the decoder still reads the frame. + let mut out = Vec::new(); + decompress_stream( + anonymous.as_slice(), + &mut out, + &dicts, + &DecodeSettings::default(), + ) + .expect("an explicit dictionary decodes an anonymous frame"); + assert_eq!(out, payload); +} + +/// `-q` and `-v` move the display level one step per occurrence, from the +/// default of 2, so `-qq` reaches the level that silences errors too. +#[test] +fn quiet_and_verbose_move_the_display_level() { + assert_eq!(parse(&["f"]).unwrap().verbosity, DEFAULT_LEVEL); + assert_eq!(parse(&["-q", "f"]).unwrap().verbosity, 1); + assert_eq!(parse(&["-qq", "f"]).unwrap().verbosity, 0); + assert_eq!(parse(&["--quiet", "--quiet", "f"]).unwrap().verbosity, 0); + assert_eq!(parse(&["-v", "f"]).unwrap().verbosity, 3); + assert_eq!(parse(&["-vvv", "f"]).unwrap().verbosity, 5); + assert_eq!(parse(&["--verbose", "-q", "f"]).unwrap().verbosity, 2); +} + +/// A mistaken command line reports the level the flags before it had reached, +/// so `-q --bogus` gets the error alone while the default level adds usage. +#[test] +fn a_parse_failure_carries_the_display_level_reached() { + let owned: Vec = ["-q", "--bogus"].iter().map(OsString::from).collect(); + let failure = parse_args(&owned, &plain(), CompressionLevel::DEFAULT_LEVEL) + .err() + .expect("an unknown option fails"); + assert_eq!(failure.verbosity, 1); + assert!(failure.error.to_string().contains("--bogus")); +} + +/// `ZSTD_CLEVEL` replaces the default level only: a level on the command line +/// still wins, and the benchmark's default start follows it too. +#[test] +fn the_environment_level_is_the_default_the_command_line_overrides() { + assert_eq!(parse_as(&plain(), 11, &["f"]).unwrap().level, 11); + assert_eq!(parse_as(&plain(), 11, &["-3", "f"]).unwrap().level, 3); + assert_eq!( + parse_as(&plain(), 11, &["-b", "f"]).unwrap().bench_start, + 11 + ); + // An environment level above the ceiling is reduced like a typed one. + assert_eq!(parse_as(&plain(), 22, &["f"]).unwrap().level, 19); +} + +/// `ZSTD_CLEVEL` is read the way the reference command reads it: a sign, +/// digits, an optional `K`/`M`; anything else is ignored with a warning and +/// the built-in default stands. +#[test] +fn the_environment_level_is_read_like_the_reference_reads_it() { + let read = |value: &str| level_from_env(Some(OsStr::new(value)), 0); + assert_eq!(read("11"), 11); + assert_eq!(read("-3"), -3); + assert_eq!(read("+2"), 2); + assert_eq!(read("1K"), 1024); + assert_eq!( + read("-999999999"), + CompressionLevel::MIN_LEVEL, + "a level below the scale is clamped, as the library clamps it" + ); + assert_eq!(read("abc"), CompressionLevel::DEFAULT_LEVEL); + assert_eq!(read(""), CompressionLevel::DEFAULT_LEVEL); + assert_eq!(read("3x"), CompressionLevel::DEFAULT_LEVEL); + assert_eq!(read("99999999999"), CompressionLevel::DEFAULT_LEVEL); + assert_eq!(read("-"), CompressionLevel::DEFAULT_LEVEL); + assert_eq!(level_from_env(None, 0), CompressionLevel::DEFAULT_LEVEL); + // The thread count is validated the same way and never fails the run. + check_threads_env(Some(OsStr::new("4")), 0); + check_threads_env(Some(OsStr::new("nope")), 0); + check_threads_env(None, 0); +} + +/// Options that take a value take it attached or as the next argument, the +/// way the reference command's `NEXT_FIELD` reads them; a missing value, or +/// another option where the value should be, is a broken command line. +#[test] +fn valued_long_options_take_the_next_argument_or_an_attached_value() { + let separated = parse(&[ + "--filelist", + "list.txt", + "--output-dir-flat", + "out", + "--maxdict", + "4096", + "f", + ]) + .unwrap(); + assert_eq!(separated.filelists, vec![PathBuf::from("list.txt")]); + assert_eq!(separated.output_dir, Some(PathBuf::from("out"))); + assert_eq!(separated.max_dict, 4096); + assert_eq!(separated.inputs, vec![PathBuf::from("f")]); + + let attached = parse(&[ + "--filelist=a.txt", + "--filelist=b.txt", + "--output-dir-mirror=tree", + "--stream-size", + "4K", + "f", + ]) + .unwrap(); + assert_eq!( + attached.filelists, + vec![PathBuf::from("a.txt"), PathBuf::from("b.txt")] + ); + assert_eq!(attached.output_dir_mirror, Some(PathBuf::from("tree"))); + assert_eq!(attached.pledged_size, Some(4096)); + + assert!(parse(&["--filelist"]).is_err(), "a value is required"); + assert!( + parse(&["--output-dir-flat", "-f", "f"]).is_err(), + "an option is not a value" + ); + assert!( + parse(&["--output-dir-flat="]).is_err(), + "an empty directory" + ); + assert!(parse(&["--output-dir-mirror", "", "f"]).is_err()); + assert!( + parse(&["--maxdict", "big", "f"]).is_err(), + "a number is a number" + ); +} + +/// The remaining file-selection and decode flags parse and land in the +/// options they steer. +#[test] +fn file_selection_flags_parse() { + let opts = parse(&[ + "-r", + "--exclude-compressed", + "--pass-through", + "--progress", + "dir", + ]) + .unwrap(); + assert!(opts.recursive && opts.exclude_compressed); + assert_eq!(opts.pass_through, Some(true)); + assert_eq!(opts.progress, Progress::Always); + let opts = parse(&["--no-pass-through", "--no-progress", "-C", "f"]).unwrap(); + assert_eq!(opts.pass_through, Some(false)); + assert_eq!(opts.progress, Progress::Never); + assert!(opts.checksum); + // `-f` follows links and admits a terminal on stdin, as the reference + // command's does; without it neither happens. + let forced = parse(&["-f", "f"]).unwrap(); + assert!(forced.force && forced.follow_links && forced.force_stdin); + let plain_run = parse(&["f"]).unwrap(); + assert!(!plain_run.follow_links && !plain_run.force_stdin); +} + +/// The help and version texts carry no typographic dash and the version +/// line names both the reference version followed and this build's own. +#[test] +fn help_and_version_texts_are_plain_ascii_punctuation() { + assert!(!HELP_ADVANCED.contains('\u{2014}')); + let mut usage = Vec::new(); + write_short_usage(&mut usage, "zstd").unwrap(); + let usage = String::from_utf8(usage).unwrap(); + assert!(!usage.contains('\u{2014}')); + assert!(usage.contains("Usage: zstd [OPTIONS...] [INPUT... | -] [-o OUTPUT]")); + assert!(usage.contains("-H, --help")); + assert_eq!(UPSTREAM_VERSION, "1.5.7"); } #[test] fn decompress_suffix_stripping() { - let opts = Options { - mode: Mode::Decompress, - level: 3, - store: false, - dict: None, - to_stdout: false, - output: None, - force: false, - keep: false, - remove_source: false, - inputs: vec![PathBuf::from("archive.tar.zst")], - max_dict: DEFAULT_MAX_DICT, - dict_id: None, - bench: false, - bench_start: 3, - bench_end: 0, - bench_secs: 1.0, - bench_separately: false, - long: false, - long_window_log: None, - memory_limit: None, - target_block_size: None, - pledged_size: None, - size_hint: None, - }; + let opts = parse(&["-d", "archive.tar.zst"]).unwrap(); assert_eq!( derive_output_path(&opts, Path::new("archive.tar.zst")).unwrap(), PathBuf::from("archive.tar") ); - assert!(derive_output_path(&opts, Path::new("noext")).is_err()); + // The reference command's suffix list: `.zstd` is dropped like `.zst`, + // and a `.tzst` tarball comes back as `.tar`. + assert_eq!( + derive_output_path(&opts, Path::new("data.zstd")).unwrap(), + PathBuf::from("data") + ); + assert_eq!( + derive_output_path(&opts, Path::new("backup.tzst")).unwrap(), + PathBuf::from("backup.tar") + ); + let err = derive_output_path(&opts, Path::new("noext")) + .expect_err("no suffix, no derived name") + .to_string(); + assert!(err.contains("unknown suffix"), "{err}"); + assert!(derive_output_path(&opts, Path::new("archive.gz")).is_err()); // A path is bytes, not text. Rebuilding it through a lossy conversion // renames what it decompresses — and two different inputs can end up @@ -2154,16 +2532,22 @@ fn listing_refuses_inputs_that_are_not_regular_files() { return; } - let mut opts = parse(&["-l", "f"]).unwrap(); + let refused = list_file(&fifo, false, 0); + let mut opts = parse(&["-l", "-qq", "f"]).unwrap(); opts.inputs = vec![fifo.clone()]; - let refused = run(opts); + let failed = run(opts); let _ = fs::remove_file(&fifo); let err = refused.expect_err("a FIFO cannot be listed").to_string(); assert!( - err.contains("regular files"), + err.contains("is not a file"), "the refusal must name what is wrong with the input: {err}" ); + assert_eq!( + failed.expect("a listing reports per file and goes on"), + 1, + "and the run counts it as failed" + ); } /// A size that is not an allocation — a sparse file's apparent length — can be @@ -2330,7 +2714,7 @@ fn listing_refuses_the_stdin_marker() { .expect_err("the marker means stdin, which listing cannot walk") .to_string(); assert!( - err.contains("stdin"), + err.contains("standard input"), "the refusal must say what is wrong with it: {err}" ); } @@ -2470,21 +2854,312 @@ fn the_memory_limit_counts_the_dictionary_and_the_benchmark_together() { ); } -/// Flags whose whole purpose is to change which files are touched, or what -/// happens to input that is not compressed, cannot be accepted as no-ops: the -/// caller would get compression where they asked for a skip, or an error where -/// they asked for a copy. +/// Outputs land where the directory flags say: `--output-dir-flat` beside +/// nothing but the file name, `--output-dir-mirror` under the replayed source +/// directory, and the mirror wins when both are given, as it does in the +/// reference command. A source the mirror cannot place is an error for that +/// input. #[test] -fn unimplemented_behaviour_flags_are_rejected() { - for args in [ - &["--exclude-compressed", "f"][..], - &["-d", "--pass-through", "f"][..], - ] { - assert!( - parse(args).is_err(), - "{args:?} changes which files are processed and is not implemented" - ); - } +fn output_directories_place_the_derived_name() { + let flat = parse(&["--output-dir-flat", "out", "f"]).unwrap(); + assert_eq!( + derive_output_path(&flat, Path::new("a/b/c.txt")).unwrap(), + PathBuf::from("out/c.txt.zst") + ); + let flat_decompress = parse(&["-d", "--output-dir-flat", "out", "f"]).unwrap(); + assert_eq!( + derive_output_path(&flat_decompress, Path::new("a/b/c.txt.zst")).unwrap(), + PathBuf::from("out/c.txt") + ); + let mirror = parse(&["--output-dir-mirror", "tree", "f"]).unwrap(); + assert_eq!( + derive_output_path(&mirror, Path::new("a/b/c.txt")).unwrap(), + PathBuf::from("tree/a/b/c.txt.zst") + ); + assert_eq!( + derive_output_path(&mirror, Path::new("/abs/c.txt")).unwrap(), + PathBuf::from("tree/abs/c.txt.zst") + ); + let err = derive_output_path(&mirror, Path::new("../c.txt")) + .expect_err("a source that climbs cannot be mirrored") + .to_string(); + assert!(err.contains("--output-dir-mirror cannot compress"), "{err}"); + let both = parse(&[ + "--output-dir-flat", + "out", + "--output-dir-mirror", + "tree", + "f", + ]) + .unwrap(); + assert_eq!( + derive_output_path(&both, Path::new("a/c.txt")).unwrap(), + PathBuf::from("tree/a/c.txt.zst"), + "the mirror takes precedence" + ); + // `-o` names the destination outright, whatever directory flags say. + let named = parse(&["-o", "exact.zst", "--output-dir-flat", "out", "f"]).unwrap(); + assert_eq!( + derive_output_path(&named, Path::new("a/c.txt")).unwrap(), + PathBuf::from("exact.zst") + ); +} + +/// `--exclude-compressed` leaves a file whose extension says it is already +/// compressed alone: no output, no failure. A directory named without `-r` is +/// refused as the reference command refuses it. +#[test] +fn already_compressed_inputs_are_skipped_and_directories_refused() { + let scratch = Scratch::new("exclude"); + let archive = scratch.file("data.gz", b"pretend gzip"); + let plain = scratch.file("data.txt", b"plain text to compress"); + + let mut opts = parse(&["--exclude-compressed", "-q", "f"]).unwrap(); + opts.inputs = vec![archive.clone(), plain.clone()]; + let skipped = process_file(&opts, &archive, &no_dict(), 2).expect("skipping is not an error"); + assert!(matches!(skipped, Outcome::Skipped)); + assert!( + !scratch.path().join("data.gz.zst").exists(), + "nothing is written for a skipped input" + ); + let done = process_file(&opts, &plain, &no_dict(), 2).expect("the plain file compresses"); + assert!(matches!(done, Outcome::Done(_))); + assert!(scratch.path().join("data.txt.zst").exists()); + + let err = open_input(&opts, scratch.path()) + .expect_err("a directory is not an input without -r") + .to_string(); + assert!(err.contains("is a directory"), "{err}"); + let err = open_input(&opts, &scratch.path().join("missing")) + .expect_err("a missing input is reported") + .to_string(); + assert!(err.contains("can't stat"), "{err}"); +} + +/// One failing input does not end the run: the rest are processed and the +/// failure count comes back to become the exit status, as the reference +/// command's does. +#[test] +fn a_failing_input_is_reported_and_the_rest_are_processed() { + let scratch = Scratch::new("continue"); + let good = scratch.file("good.txt", b"good bytes to compress"); + let missing = scratch.path().join("missing.txt"); + + let mut opts = parse(&["-qq", "f", "g"]).unwrap(); + opts.inputs = vec![missing, good.clone()]; + let failed = run(opts).expect("the run itself completes"); + assert_eq!(failed, 1, "one input failed"); + assert!( + scratch.path().join("good.txt.zst").exists(), + "the good input was still compressed" + ); +} + +/// Without `-f` an existing output is not replaced. Below the default display +/// level no question can be asked, so the input is refused and counted as +/// failed, with the existing file untouched. +#[test] +fn an_existing_output_is_not_replaced_quietly() { + let scratch = Scratch::new("overwrite"); + let input = scratch.file("in.txt", b"new content"); + let existing = scratch.file("in.txt.zst", b"precious bytes"); + + let mut opts = parse(&["-q", "f"]).unwrap(); + opts.inputs = vec![input.clone()]; + let outcome = process_file(&opts, &input, &no_dict(), 1).expect("a refusal is not an error"); + assert!(matches!(outcome, Outcome::Refused)); + assert_eq!(fs::read(&existing).unwrap(), b"precious bytes"); + + let mut forced = parse(&["-q", "-f", "f"]).unwrap(); + forced.inputs = vec![input.clone()]; + let outcome = process_file(&forced, &input, &no_dict(), 1).expect("-f replaces it"); + assert!(matches!(outcome, Outcome::Done(_))); + assert_ne!(fs::read(&existing).unwrap(), b"precious bytes"); +} + +/// Several inputs into one `-o` are concatenated as frames into that file, +/// which decodes to the inputs in order. It is a destructive shape, so +/// `--rm` is set aside and the sources stay; without `-f` and with no way to +/// ask, the run refuses and writes nothing. +#[test] +fn several_inputs_into_one_output_are_concatenated_and_keep_their_sources() { + let scratch = Scratch::new("concat"); + let a = scratch.file("a.txt", b"first part, "); + let b = scratch.file("b.txt", b"second part"); + let output = scratch.path().join("both.zst"); + + let mut refused = parse(&["-q", "--rm", "-o", "x", "a", "b"]).unwrap(); + refused.inputs = vec![a.clone(), b.clone()]; + refused.output = Some(output.clone()); + assert_eq!( + run(refused).expect("a refusal is a failed run, not an error"), + 2, + "every input counts as failed" + ); + assert!(!output.exists(), "nothing is written without -f"); + + let mut opts = parse(&["-q", "-f", "--rm", "-o", "x", "a", "b"]).unwrap(); + opts.inputs = vec![a.clone(), b.clone()]; + opts.output = Some(output.clone()); + assert_eq!(run(opts).expect("the concatenation runs"), 0); + assert_eq!( + decoded(&fs::read(&output).unwrap()).unwrap(), + b"first part, second part" + ); + assert!( + a.exists() && b.exists(), + "--rm is set aside for a concatenation" + ); +} + +/// A single input into `-o` keeps the reference command's single-file path: +/// `--rm` applies, and the output takes the source's permissions. +#[test] +fn a_single_input_into_a_named_output_is_removed_on_request() { + let scratch = Scratch::new("single"); + let input = scratch.file("only.txt", b"only bytes"); + let output = scratch.path().join("only.zst"); + + let mut opts = parse(&["-q", "--rm", "-o", "x", "a"]).unwrap(); + opts.inputs = vec![input.clone()]; + opts.output = Some(output.clone()); + assert_eq!(run(opts).unwrap(), 0); + assert_eq!(decoded(&fs::read(&output).unwrap()).unwrap(), b"only bytes"); + assert!(!input.exists(), "--rm removes the one source"); +} + +/// A run pointed only at empty directories has nothing to do: it says so and +/// succeeds, rather than falling back to reading stdin. +#[test] +fn empty_directories_are_nothing_to_do_not_a_request_for_stdin() { + let scratch = Scratch::new("emptydir"); + fs::create_dir_all(scratch.path().join("empty")).unwrap(); + let mut opts = parse(&["-r", "-q", "d"]).unwrap(); + opts.inputs = vec![scratch.path().join("empty")]; + assert_eq!(run(opts).expect("nothing to do is not an error"), 0); + + // And a directory named without `-r` is one failed input. + let mut opts = parse(&["-qq", "d"]).unwrap(); + opts.inputs = vec![scratch.path().join("empty")]; + assert_eq!(run(opts).unwrap(), 1); +} + +/// Decompression reports how many bytes came out, which is what `-t` and the +/// summaries print; a corrupted checksum is ignored under `--no-check`. +#[test] +fn decoding_counts_its_output_and_no_check_ignores_the_checksum() { + let payload = b"payload whose checksum will be corrupted"; + let mut frame = frame_of(payload); + let written = decompress_stream( + frame.as_slice(), + io::sink(), + &no_dict(), + &DecodeSettings::default(), + ) + .unwrap(); + assert_eq!(written, payload.len() as u64); + + let last = frame.len() - 1; + frame[last] ^= 0xFF; + decompress_stream( + frame.as_slice(), + io::sink(), + &no_dict(), + &DecodeSettings::default(), + ) + .expect_err("verified by default"); + let ignored = decompress_stream( + frame.as_slice(), + io::sink(), + &no_dict(), + &DecodeSettings { + verify_checksum: false, + pass_through: false, + }, + ) + .expect("--no-check decodes it regardless"); + assert_eq!(ignored, payload.len() as u64); + assert!( + !DecodeSettings::from_options(&parse(&["-d", "--no-check", "f"]).unwrap()).verify_checksum + ); +} + +/// Input that is not a zstd stream is copied through under `--pass-through`, +/// bytes intact, and refused otherwise; a stream too short to hold a magic +/// number is treated the same way, as the reference command treats it. +#[test] +fn plain_input_is_passed_through_or_refused() { + let pass = DecodeSettings { + verify_checksum: true, + pass_through: true, + }; + let mut out = Vec::new(); + let written = decompress_stream(&b"plain text, not a frame"[..], &mut out, &no_dict(), &pass) + .expect("pass-through copies it"); + assert_eq!(out, b"plain text, not a frame"); + assert_eq!(written, out.len() as u64); + + let mut short = Vec::new(); + decompress_stream(&b"ab"[..], &mut short, &no_dict(), &pass).unwrap(); + assert_eq!(short, b"ab", "fewer than four bytes are passed through too"); + + let err = decompress_stream( + &b"plain text, not a frame"[..], + io::sink(), + &no_dict(), + &DecodeSettings::default(), + ) + .expect_err("refused without pass-through") + .to_string(); + assert!(err.contains("unsupported format"), "{err}"); + let err = decompress_stream( + &b"ab"[..], + io::sink(), + &no_dict(), + &DecodeSettings::default(), + ) + .expect_err("a stump is refused too") + .to_string(); + assert!(err.contains("unknown header"), "{err}"); + + // A real frame followed by plain bytes: the frame decodes, and the tail + // is passed through after it, as the reference command's loop does. + let mut mixed = frame_of(b"framed"); + mixed.extend_from_slice(b" then plain"); + let mut out = Vec::new(); + decompress_stream(mixed.as_slice(), &mut out, &no_dict(), &pass).unwrap(); + assert_eq!(out, b"framed then plain"); + + // The default follows the reference command: on when forced and writing + // to stdout (`zstd -dcf`), off otherwise. + assert!(DecodeSettings::from_options(&parse(&["-dcf", "f"]).unwrap()).pass_through); + assert!(!DecodeSettings::from_options(&parse(&["-dc", "f"]).unwrap()).pass_through); + assert!(!DecodeSettings::from_options(&parse(&["-df", "f"]).unwrap()).pass_through); + assert!( + DecodeSettings::from_options(&parse(&["-df", "--pass-through", "f"]).unwrap()).pass_through + ); +} + +/// Data to stdout silences the result summary and sets `--rm` aside; the +/// verbosity and the removal flag are what the run computes from the inputs +/// and destination together. +#[test] +fn stdout_output_is_recognised_from_the_inputs_and_destination() { + assert!(writes_stdout(&parse(&["-c", "f"]).unwrap())); + assert!( + writes_stdout(&parse(&[]).unwrap()), + "stdin in, nothing named: stdout out" + ); + assert!(writes_stdout(&parse(&["-"]).unwrap())); + assert!(!writes_stdout(&parse(&["-o", "out"]).unwrap())); + assert!(!writes_stdout(&parse(&["f"]).unwrap())); + assert!( + !writes_stdout(&parse(&["f", "-"]).unwrap()), + "a file among them gets its own" + ); + assert!(reads_stdin(&[])); + assert!(reads_stdin(&[PathBuf::from("a"), PathBuf::from("-")])); + assert!(!reads_stdin(&[PathBuf::from("a")])); } /// `--adapt` also comes parameterised upstream (`--adapt=min=1,max=9`). @@ -2576,8 +3251,13 @@ fn corrupted_checksum_is_reported_not_passed() { let last = frame.len() - 1; frame[last] ^= 0xFF; - let err = decompress_stream(frame.as_slice(), io::sink(), &no_dict()) - .expect_err("a corrupted checksum must fail the decode"); + let err = decompress_stream( + frame.as_slice(), + io::sink(), + &no_dict(), + &DecodeSettings::default(), + ) + .expect_err("a corrupted checksum must fail the decode"); let text = err.to_string(); assert!( text.to_ascii_lowercase().contains("checksum"), @@ -2653,8 +3333,7 @@ fn concatenated_frames_are_all_decoded() { .expect("compressing a fixture frame must succeed"); } - let mut out = Vec::new(); - decompress_stream(stream.as_slice(), &mut out, &no_dict()).expect("both frames must decode"); + let out = decoded(&stream).expect("both frames must decode"); assert_eq!( out, b"first frame payloadsecond frame payload", "every frame in the stream has to reach the output" @@ -2682,9 +3361,7 @@ fn skippable_frames_are_stepped_over() { compress_stream(&b" and more"[..], &mut stream, &level_only, &no_dict()) .expect("compressing the trailing fixture must succeed"); - let mut out = Vec::new(); - decompress_stream(stream.as_slice(), &mut out, &no_dict()) - .expect("a skippable frame must not fail the decode"); + let out = decoded(&stream).expect("a skippable frame must not fail the decode"); assert_eq!( out, b"payload and more", "skippable content is stepped over, not emitted" From 4e59a6fc36a41015bbf5d78f1d7d9539082e54b7 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 11 Sep 2026 02:20:19 +0300 Subject: [PATCH 2/5] feat(encode): add the literal compression mode parameter LiteralCompressionMode (Auto / Enable / Disable) on the compression parameters builder, the drop-in equivalent of ZSTD_c_literalCompressionMode: Disable stores every literal section raw, Enable entropy-codes literals on the negative levels too, Auto keeps the level's own choice. FrameCompressor and StreamingEncoder carry the mode beside the target-length override and feed it to the raw-literals gate on every frame. Part of #128 --- zstd/src/encoding/frame_compressor.rs | 35 ++++++++--- zstd/src/encoding/mod.rs | 4 +- zstd/src/encoding/parameters.rs | 60 ++++++++++++++++++ zstd/src/encoding/parameters/tests.rs | 87 ++++++++++++++++++++++++++ zstd/src/encoding/streaming_encoder.rs | 6 ++ 5 files changed, 181 insertions(+), 11 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index ff8b5062b..511d0d9cb 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -9,8 +9,8 @@ use twox_hash::XxHash64; use core::hash::Hasher; use super::{ - CompressionLevel, Matcher, block_header::BlockHeader, frame_header::FrameHeader, levels::*, - match_generator::MatchGeneratorDriver, + CompressionLevel, LiteralCompressionMode, Matcher, block_header::BlockHeader, + frame_header::FrameHeader, levels::*, match_generator::MatchGeneratorDriver, }; use crate::common::MAX_BLOCK_SIZE; use crate::fse::fse_encoder::{FSETable, default_ll_table, default_ml_table, default_of_table}; @@ -233,6 +233,10 @@ pub struct FrameCompressor< /// after `set_parameters` flips whether the override applies (the /// matcher drops it on a dictionary frame). target_length_override: Option, + /// Public literal-compression mode (upstream `ZSTD_c_literalCompressionMode`), + /// persisted beside the target-length override for the same per-frame + /// recomputation of the raw-literals gate. + literal_compression_mode: LiteralCompressionMode, } #[derive(Clone, Default)] @@ -1029,8 +1033,9 @@ pub(crate) fn sync_effective_strategy( } } -/// Upstream `ZSTD_literalsCompressionIsDisabled` (`ps_auto`): raw literals -/// iff the EFFECTIVE cParams are the fast strategy with `targetLength > 0`. +/// Upstream `ZSTD_literalsCompressionIsDisabled`: an explicit +/// [`LiteralCompressionMode`] decides outright; under `Auto`, raw literals iff +/// the EFFECTIVE cParams are the fast strategy with `targetLength > 0`. /// The effective strategy tag gates this (a strategy override can move a /// negative level off fast). For the fast strategy the level table sets /// `targetLength > 0` exactly on the negative (acceleration) rows, so absent @@ -1041,12 +1046,19 @@ pub(crate) fn literal_compression_disabled( strategy_tag: crate::encoding::strategy::StrategyTag, level: CompressionLevel, target_length_override: Option, + mode: LiteralCompressionMode, ) -> bool { - strategy_tag == crate::encoding::strategy::StrategyTag::Fast - && target_length_override.map_or_else( - || matches!(level, CompressionLevel::Level(n) if n < 0), - |tl| tl > 0, - ) + match mode { + LiteralCompressionMode::Disable => true, + LiteralCompressionMode::Enable => false, + LiteralCompressionMode::Auto => { + strategy_tag == crate::encoding::strategy::StrategyTag::Fast + && target_length_override.map_or_else( + || matches!(level, CompressionLevel::Level(n) if n < 0), + |tl| tl > 0, + ) + } + } } /// The level params the matcher's reset resolves for a frame: through the @@ -1476,6 +1488,7 @@ impl FrameCompressor { block_decompressed_sizes: alloc::vec::Vec::new(), strategy_override: None, target_length_override: None, + literal_compression_mode: LiteralCompressionMode::Auto, } } @@ -1506,6 +1519,7 @@ impl FrameCompressor { let overrides = params.overrides(); self.strategy_override = overrides.strategy.map(|s| (s.tag(), s.lazy_depth())); self.target_length_override = overrides.target_length; + self.literal_compression_mode = overrides.literal_compression; // Keep `state.strategy_tag` consistent immediately so the borrowed // one-shot eligibility gate (`borrowed_eligible`) and literal gates // are correct even before the next `compress()` re-sync. Resolve it @@ -1526,6 +1540,7 @@ impl FrameCompressor { self.state.strategy_tag, self.compression_level, overrides.target_length.filter(|_| !dict_frame), + self.literal_compression_mode, ); self.state.matcher.set_param_overrides(Some(overrides)); } @@ -1890,6 +1905,7 @@ impl FrameCompressor { block_decompressed_sizes: alloc::vec::Vec::new(), strategy_override: None, target_length_override: None, + literal_compression_mode: LiteralCompressionMode::Auto, } } @@ -2225,6 +2241,7 @@ impl FrameCompressor { self.state.strategy_tag, self.compression_level, self.target_length_override.filter(|_| !planned), + self.literal_compression_mode, ); let cached_entropy = if use_dictionary_state { self.dictionary_entropy_cache.as_ref() diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index d230fc57d..003e4788f 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -102,8 +102,8 @@ pub use levels::config::{ }; pub use match_generator::MatchGeneratorDriver; pub use parameters::{ - Bounds, CParameter, CompressionParameters, CompressionParametersBuilder, ParameterError, - Strategy, + Bounds, CParameter, CompressionParameters, CompressionParametersBuilder, + LiteralCompressionMode, ParameterError, Strategy, }; pub use streaming_encoder::StreamingEncoder; diff --git a/zstd/src/encoding/parameters.rs b/zstd/src/encoding/parameters.rs index c050a611e..033d7b118 100644 --- a/zstd/src/encoding/parameters.rs +++ b/zstd/src/encoding/parameters.rs @@ -138,6 +138,23 @@ impl Strategy { } } +/// Whether literals are entropy-coded, the drop-in equivalent of C zstd's +/// `ZSTD_c_literalCompressionMode` (`ZSTD_ps_auto` / `ZSTD_ps_enable` / +/// `ZSTD_ps_disable`). +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub enum LiteralCompressionMode { + /// The level decides: literals are stored raw on the fast strategy's + /// acceleration (negative) levels, where the Huffman pass costs more + /// speed than it saves, and compressed everywhere else. + #[default] + Auto, + /// Compress literals on every level, the negative ones included. A + /// block whose literals do not shrink still stores them raw. + Enable, + /// Never compress literals: every literal section is stored raw. + Disable, +} + /// One tunable compression parameter — the analogue of a C zstd /// `ZSTD_cParameter`. Used to query bounds via [`CParameter::bounds`]. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -288,6 +305,8 @@ pub(crate) struct ParamOverrides { /// `Some` when `enable_long_distance_matching(true)` was set; carries /// the (possibly empty) LDM knob overrides. pub(crate) ldm: Option, + /// Whether literals are entropy-coded; `Auto` leaves it to the level. + pub(crate) literal_compression: LiteralCompressionMode, } impl ParamOverrides { @@ -303,6 +322,7 @@ impl ParamOverrides { && self.target_length.is_none() && self.strategy.is_none() && self.ldm.is_none() + && self.literal_compression == LiteralCompressionMode::Auto } } @@ -335,6 +355,7 @@ impl CompressionParameters { strategy: None, enable_ldm: false, ldm: LdmOverride::default(), + literal_compression: LiteralCompressionMode::Auto, } } @@ -348,6 +369,11 @@ impl CompressionParameters { self.overrides.ldm.is_some() } + /// How literals are entropy-coded (see [`LiteralCompressionMode`]). + pub fn literal_compression_mode(&self) -> LiteralCompressionMode { + self.overrides.literal_compression + } + pub(crate) fn overrides(&self) -> ParamOverrides { self.overrides } @@ -367,9 +393,42 @@ pub struct CompressionParametersBuilder { strategy: Option, enable_ldm: bool, ldm: LdmOverride, + literal_compression: LiteralCompressionMode, } impl CompressionParametersBuilder { + /// Decide whether literals are entropy-coded, overriding the level's own + /// choice. C `ZSTD_c_literalCompressionMode`. + /// + /// ```rust + /// use structured_zstd::encoding::{ + /// compress_with_parameters, CompressionLevel, CompressionParameters, + /// LiteralCompressionMode, + /// }; + /// + /// // Literal-heavy input: 32 distinct symbols and nothing for the match + /// // finder to repeat, so only entropy-coding the literals shrinks it. + /// let text: Vec = (0..8192u32) + /// .map(|i| b'a' + (i.wrapping_mul(2_654_435_761) >> 27) as u8) + /// .collect(); + /// // Level -3 stores literals raw by default; asking for compression + /// // makes the frame smaller on input like this. + /// let compressed_literals = CompressionParameters::builder(CompressionLevel::Level(-3)) + /// .literal_compression(LiteralCompressionMode::Enable) + /// .build() + /// .unwrap(); + /// let plain = compress_with_parameters( + /// &text[..], + /// &CompressionParameters::builder(CompressionLevel::Level(-3)).build().unwrap(), + /// ); + /// let coded = compress_with_parameters(&text[..], &compressed_literals); + /// assert!(coded.len() < plain.len()); + /// ``` + pub fn literal_compression(mut self, mode: LiteralCompressionMode) -> Self { + self.literal_compression = mode; + self + } + /// Override the maximum back-reference distance (`log2`). C /// `ZSTD_c_windowLog`. pub fn window_log(mut self, value: u32) -> Self { @@ -495,6 +554,7 @@ impl CompressionParametersBuilder { target_length: self.target_length, strategy: self.strategy, ldm, + literal_compression: self.literal_compression, }, }) } diff --git a/zstd/src/encoding/parameters/tests.rs b/zstd/src/encoding/parameters/tests.rs index 99a867660..448759fae 100644 --- a/zstd/src/encoding/parameters/tests.rs +++ b/zstd/src/encoding/parameters/tests.rs @@ -43,6 +43,93 @@ fn builder_records_each_knob() { assert!(!o.is_empty()); } +/// The literal mode is a knob like the others: recorded by the builder, read +/// back from the parameters, and an override of the level when it is not +/// `Auto`, since a non-empty override set is what the reset path acts on. +#[test] +fn literal_compression_mode_is_recorded_and_counts_as_an_override() { + let auto = CompressionParameters::builder(CompressionLevel::Level(3)) + .build() + .unwrap(); + assert_eq!( + auto.literal_compression_mode(), + LiteralCompressionMode::Auto + ); + assert!(auto.overrides().is_empty()); + + for mode in [ + LiteralCompressionMode::Enable, + LiteralCompressionMode::Disable, + ] { + let p = CompressionParameters::builder(CompressionLevel::Level(3)) + .literal_compression(mode) + .build() + .unwrap(); + assert_eq!(p.literal_compression_mode(), mode); + assert!(!p.overrides().is_empty(), "{mode:?} overrides the level"); + } +} + +/// `Disable` stores every literal raw, so text compresses worse than the +/// level's default; `Enable` on a negative level, where the default is raw, +/// compresses better. Both frames still decode to the input. +#[test] +fn literal_compression_mode_changes_the_frame() { + use crate::decoding::StreamingDecoder; + use crate::encoding::compress_with_parameters; + use crate::io::Read; + + // Literal-heavy input: 32 distinct symbols in a sequence with no repeats + // for the match finder, so the frame is all literals and only their + // entropy coding can shrink it (5 bits a symbol against 8 raw). + let text: alloc::vec::Vec = (0..8192u32) + .map(|i| b'a' + (i.wrapping_mul(2_654_435_761) >> 27) as u8) + .collect(); + let frame_with = |level: i32, mode: LiteralCompressionMode| { + let params = CompressionParameters::builder(CompressionLevel::Level(level)) + .literal_compression(mode) + .build() + .unwrap(); + compress_with_parameters(&text[..], ¶ms) + }; + let decoded = |frame: &[u8]| { + let mut source = frame; + let mut out = alloc::vec::Vec::new(); + StreamingDecoder::new(&mut source) + .unwrap() + .read_to_end(&mut out) + .unwrap(); + out + }; + + let auto_l3 = frame_with(3, LiteralCompressionMode::Auto); + let raw_l3 = frame_with(3, LiteralCompressionMode::Disable); + assert!( + raw_l3.len() > auto_l3.len(), + "raw literals cost bytes on text: {} vs {}", + raw_l3.len(), + auto_l3.len() + ); + assert_eq!(decoded(&raw_l3), text); + + let auto_fast = frame_with(-3, LiteralCompressionMode::Auto); + let coded_fast = frame_with(-3, LiteralCompressionMode::Enable); + assert!( + coded_fast.len() < auto_fast.len(), + "compressed literals save bytes at a negative level: {} vs {}", + coded_fast.len(), + auto_fast.len() + ); + assert_eq!(decoded(&coded_fast), text); + // `Auto` at a negative level is raw literals, which is what `Disable` + // spells out, so the two agree there. + assert_eq!( + auto_fast, + frame_with(-3, LiteralCompressionMode::Disable), + "the negative level's default is raw literals" + ); +} + #[test] fn enable_ldm_sets_override_block() { let p = CompressionParameters::builder(CompressionLevel::Level(19)) diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index 474a1bc70..4cd593ed6 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -65,6 +65,9 @@ pub struct StreamingEncoder { /// resolved at frame start reads the value the matcher runs (dropped on a /// dictionary frame, where the CDict's targetLength applies). target_length_override: Option, + /// Public literal-compression mode (upstream `ZSTD_c_literalCompressionMode`), + /// read by the same gate; mirrors `FrameCompressor`'s field. + literal_compression_mode: crate::encoding::LiteralCompressionMode, /// `ZSTD_f_zstd1_magicless` — omit the 4-byte magic number prefix. /// Default false. See [`Self::set_magicless`]. magicless: bool, @@ -127,6 +130,7 @@ impl StreamingEncoder { // resync does not discard it (matching `FrameCompressor::set_parameters`). self.strategy_override = overrides.strategy.map(|s| (s.tag(), s.lazy_depth())); self.target_length_override = overrides.target_length; + self.literal_compression_mode = overrides.literal_compression; self.state.strategy_tag = self.strategy_override.map_or_else( || { crate::encoding::strategy::StrategyTag::for_compression_level( @@ -189,6 +193,7 @@ impl StreamingEncoder { savings: 0, strategy_override: None, target_length_override: None, + literal_compression_mode: crate::encoding::LiteralCompressionMode::Auto, magicless: false, content_checksum: false, dictionary: None, @@ -658,6 +663,7 @@ impl StreamingEncoder { self.state.strategy_tag, self.compression_level, self.target_length_override.filter(|_| !dict_frame), + self.literal_compression_mode, ); self.savings = 0; #[cfg(feature = "hash")] From 4923908b4c3270406ae3daf0940d4803f8cd7db3 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 11 Sep 2026 02:20:49 +0300 Subject: [PATCH 3/5] feat(cli): --zstd=, --patch-from, trainer tuning, bench layout - --zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#,lhlog=#,lmml=#, lblog=#,lhrlog=#: every knob through the compression-parameters builder, validated at the command line; a --zstd window wins over --long's, the LDM knobs apply with --long, and --long is admitted below level 16 when --zstd=strat= moves the level onto the optimal parser - --[no-]compress-literals wired to the literal compression mode - --patch-from REF: the reference as raw content on both sides, the window sized to the input (highbit + 1, within what this build decodes), ultra levels unlocked, long-distance matching on where the optimal parser runs; refused with -D, on several inputs, and on stdin without --stream-size - --train-cover runs the COVER trainer (its reference-side tuning is refused); --train-fastcover=k=,d=,f=,steps=,split=,accel= tunes FastCOVER the way the reference checks it; shrink is refused - -b prints upstream's result layout, with the -q machine line and header; -i defaults to 3 seconds; several files are named by their count - --rsyncable refused as incompatible with single-thread mode Part of #128 --- README.md | 19 +- zstd/src/bin/structured-zstd/main.rs | 733 ++++++++++++++++++++++---- zstd/src/bin/structured-zstd/tests.rs | 444 +++++++++++++++- 3 files changed, 1070 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index dabd3da0b..3e57ee196 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,11 @@ others, and the exit status is 1 when any input failed and 2 on an interrupt, which also removes the partial output. The wire-format switches take effect: `--[no-]check` (`--no-check` also skips -checksum verification when decoding), `--[no-]content-size` and `--no-dictID`. +checksum verification when decoding), `--[no-]content-size`, `--no-dictID` and +`--[no-]compress-literals`. `--zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#` +overrides the level's parameters knob by knob (the `ldm*` knobs apply with +`--long`). `--patch-from REF` compresses against a reference as raw content +with the window sized to the input, and applies the patch back on `-d`. `--[no-]pass-through` copies non-zstd input through unchanged when decompressing, on by default for `zstdcat` and `zstd -dcf` as upstream has it, and `--exclude-compressed` skips inputs whose extension names an @@ -84,17 +88,18 @@ source can fill, so a small file compressed with `--long` does not ask its decoders to reserve 128 MiB. Flags that would change the result are refused instead: `--format=` for -anything but zstd, `--patch-from`, `--rsyncable` and -`--[no-]compress-literals`. `-M` is treated as the safety promise it is: on the +anything but zstd and `--rsyncable`, which needs the worker threads this build +does not have. `-M` is treated as the safety promise it is: on the runs that decode, a limit covering the 128 MiB window, the decoder's buffers and the `-D` dictionary is kept and a tighter one is refused rather than ignored. Compressing, listing and training allocate no decoder, so the flag is accepted there and describes nothing, as upstream has it. -`--train` and `--train-fastcover` both train with FastCOVER, the algorithm -upstream also defaults to. `--train-cover` and `--train-legacy` name algorithms -this build does not have, so they are refused rather than quietly served by -FastCOVER. `-D` takes either a dictionary produced by `--train` or any file at +`--train` and `--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#]` train +with FastCOVER, the algorithm upstream also defaults to, and `--train-cover` +with the COVER trainer (whose reference-side tuning does not apply here, so it +is refused rather than misread). `--train-legacy` names an algorithm this build +does not have and is refused. `-D` takes either a dictionary produced by `--train` or any file at all, which is then used as raw content the way upstream does — such a dictionary has no ID, so the same bytes must be supplied when decoding. diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 84dfc036b..ce3e4c122 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -14,7 +14,9 @@ use std::fs::{self, File, OpenOptions}; use std::io::{self, BufReader, ErrorKind, IsTerminal, Read, Write}; use std::path::{Path, PathBuf}; -use structured_zstd::encoding::CompressionLevel; +use structured_zstd::encoding::{ + CompressionLevel, CompressionParameters, LiteralCompressionMode, Strategy, +}; /// Error type for the tool: a boxed message, which is all a command-line /// program does with an error — print it and exit non-zero. Written against @@ -198,11 +200,83 @@ struct Options { output_dir_mirror: Option, /// Whether the progress counter is drawn (`--[no-]progress`). progress: Progress, + /// Per-knob compression parameters from `--zstd=...`. + advanced: AdvancedParams, + /// Whether literals are entropy-coded (`--[no-]compress-literals`). + literals: LiteralCompressionMode, + /// Reference file for `--patch-from`: raw-content dictionary compression + /// with the window sized to the input, so the whole reference is + /// reachable. + patch_from: Option, + /// Which dictionary trainer `--train*` runs. + trainer: Trainer, + /// The trainer's tuning from `--train-fastcover=...` / `--train-cover=...`. + trainer_params: TrainerParams, +} + +/// The dictionary trainers `--train` selects between. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Trainer { + /// `--train` / `--train-fastcover`: FastCOVER, the reference default. + FastCover, + /// `--train-cover`: the segment-scoring COVER trainer. + Cover, +} + +/// Tuning from `--train-fastcover=k=#,d=#,f=#,steps=#,split=#,accel=#` and +/// `--train-cover=k=#,d=#,steps=#,split=#`, each knob `None` until given. +/// `shrink` is parsed so the command line is validated, and refused at +/// training time: no trainer here shrinks the dictionary afterwards. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +struct TrainerParams { + k: Option, + d: Option, + f: Option, + steps: Option, + split_percent: Option, + accel: Option, + shrink: bool, +} + +impl TrainerParams { + /// Whether any tuning was given at all. + fn is_default(&self) -> bool { + *self == Self::default() + } +} + +/// Per-knob compression parameters from `--zstd=wlog=#,clog=#,...` +/// (upstream `parseCompressionParameters`). Every knob is optional and +/// overrides the level's own value when set; zero, as there, means "the +/// level's value". +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +struct AdvancedParams { + window_log: Option, + chain_log: Option, + hash_log: Option, + search_log: Option, + min_match: Option, + target_length: Option, + strategy: Option, + ldm_hash_log: Option, + ldm_min_match: Option, + ldm_bucket_size_log: Option, + ldm_hash_rate_log: Option, +} + +impl AdvancedParams { + /// Whether any knob overrides the level. + fn is_default(&self) -> bool { + *self == Self::default() + } } /// Upstream `zstd --maxdict` default (110 KiB). const DEFAULT_MAX_DICT: usize = 112_640; +/// Least time `-b` measures each level for (upstream `BMK_TIMETEST_DEFAULT_S`). +const DEFAULT_BENCH_SECONDS: f64 = 3.0; + /// Window log a bare `--long` selects, as upstream documents (128 MiB). const DEFAULT_LONG_WINDOW_LOG: u32 = 27; @@ -378,7 +452,7 @@ fn check_window_log(log: u32) -> Result<()> { let upper = bounds.upper_bound.min(i64::from(decodable)); if i64::from(log) < bounds.lower_bound || i64::from(log) > upper { bail!( - "--long window log {log} is outside the supported range {}..={upper} \ + "window log {log} is outside the supported range {}..={upper} \ (above {decodable} the frame would declare a window this build \ refuses to decode)", bounds.lower_bound, @@ -411,6 +485,99 @@ fn parse_adapt_params(params: &str) -> Result<()> { Ok(()) } +/// Parse `--zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#,...` the +/// way the reference command does (`zstdcli.c`, `parseCompressionParameters`): +/// each key in its long or short spelling, a value read as a leading number +/// with an optional `K` / `M`, commas between. `overlapLog` / `ovlog` is a +/// multi-threading knob, accepted and without effect. Zero leaves the knob at +/// the level's value, as it does there. +fn parse_advanced_params(text: &str) -> Result { + let mut params = AdvancedParams::default(); + if text.is_empty() { + return Ok(params); + } + for field in text.split(',') { + let (key, value) = field + .split_once('=') + .ok_or_else(|| eyre!("--zstd parameter `{field}` is not `key=value`"))?; + let (number, tail) = read_leading_u32(value) + .wrap_err_with(|| format!("--zstd parameter `{key}` has an invalid value"))?; + if !tail.is_empty() { + bail!("--zstd parameter `{key}` has an invalid value `{value}`"); + } + let set = (number != 0).then_some(number); + match key { + "windowLog" | "wlog" => params.window_log = set, + "chainLog" | "clog" => params.chain_log = set, + "hashLog" | "hlog" => params.hash_log = set, + "searchLog" | "slog" => params.search_log = set, + "minMatch" | "mml" => params.min_match = set, + "targetLength" | "tlen" => params.target_length = set, + "strategy" | "strat" => { + params.strategy = match set { + None => None, + Some(ordinal) => { + Some(Strategy::from_ordinal(ordinal).ok_or_else(|| { + eyre!("--zstd strategy {ordinal} is out of range 1..=9") + })?) + } + } + } + "overlapLog" | "ovlog" => {} + "ldmHashLog" | "lhlog" => params.ldm_hash_log = set, + "ldmMinMatch" | "lmml" => params.ldm_min_match = set, + "ldmBucketSizeLog" | "lblog" => params.ldm_bucket_size_log = set, + "ldmHashRateLog" | "lhrlog" => params.ldm_hash_rate_log = set, + _ => bail!("--zstd has no `{key}` parameter"), + } + } + Ok(params) +} + +/// Parse the tuning of `--train-fastcover=...` (`fastcover` true: `f=` and +/// `accel=` are accepted as well) or `--train-cover=...`, the way the +/// reference command's `parseFastCoverParameters` / `parseCoverParameters` +/// read them. `shrink` takes an optional `=#` regression bound. +fn parse_trainer_params(text: &str, fastcover: bool) -> Result { + let flag = if fastcover { + "--train-fastcover" + } else { + "--train-cover" + }; + let mut params = TrainerParams::default(); + for field in text.split(',') { + if field == "shrink" || field.starts_with("shrink=") { + if let Some(bound) = field.strip_prefix("shrink=") { + let (_, tail) = read_leading_u32(bound) + .wrap_err_with(|| format!("{flag} shrink bound `{bound}` is invalid"))?; + if !tail.is_empty() { + bail!("{flag} shrink bound `{bound}` is invalid"); + } + } + params.shrink = true; + continue; + } + let (key, value) = field + .split_once('=') + .ok_or_else(|| eyre!("{flag} parameter `{field}` is not `key=value`"))?; + let (number, tail) = read_leading_u32(value) + .wrap_err_with(|| format!("{flag} parameter `{key}` has an invalid value"))?; + if !tail.is_empty() { + bail!("{flag} parameter `{key}` has an invalid value `{value}`"); + } + match key { + "k" => params.k = Some(number), + "d" => params.d = Some(number), + "steps" => params.steps = Some(number), + "split" => params.split_percent = Some(number), + "f" if fastcover => params.f = Some(number), + "accel" if fastcover => params.accel = Some(number), + _ => bail!("{flag} has no `{key}` parameter"), + } + } + Ok(params) +} + /// Refuse to write binary output into an interactive terminal unless forced. /// /// A compressed frame painted into a terminal scrambles the session and the @@ -702,7 +869,7 @@ fn parse_args_into( bench: false, bench_start: default_level, bench_end: 0, - bench_secs: 1.0, + bench_secs: DEFAULT_BENCH_SECONDS, bench_separately: false, long: false, long_window_log: None, @@ -723,6 +890,11 @@ fn parse_args_into( output_dir: None, output_dir_mirror: None, progress: Progress::Auto, + advanced: AdvancedParams::default(), + literals: LiteralCompressionMode::Auto, + patch_from: None, + trainer: Trainer::FastCover, + trainer_params: TrainerParams::default(), }; let mut ultra = false; let mut iter = args.iter().enumerate().peekable(); @@ -750,15 +922,24 @@ fn parse_args_into( "test" => select_mode(&mut opts, Mode::Test), "list" => select_mode(&mut opts, Mode::List), // Plain `--train` selects the same default upstream does, - // FastCOVER, so the two spellings agree. - "train" | "train-fastcover" => select_mode(&mut opts, Mode::Train), - // The other trainers produce different dictionaries. Accepting - // the flag and running FastCOVER anyway would hand back a - // dictionary the caller did not ask for, with nothing to say so. - "train-cover" | "train-legacy" => { - bail!( - "--{long} is not implemented; --train / --train-fastcover trains with FastCOVER" - ) + // FastCOVER, so the two spellings agree. Bare `--train-fastcover` + // resets the tuning, as upstream's does. + "train" => select_mode(&mut opts, Mode::Train), + "train-fastcover" => { + select_mode(&mut opts, Mode::Train); + opts.trainer = Trainer::FastCover; + opts.trainer_params = TrainerParams::default(); + } + "train-cover" => { + select_mode(&mut opts, Mode::Train); + opts.trainer = Trainer::Cover; + opts.trainer_params = TrainerParams::default(); + } + // The legacy trainer produces a different dictionary. Accepting + // the flag and running another trainer would hand back one the + // caller did not ask for, with nothing to say so. + "train-legacy" => { + bail!("--{long} is not implemented; --train-cover and --train-fastcover are") } // `-c` and `-o` name competing destinations, so each clears the // other and the later one on the command line wins, as upstream @@ -818,12 +999,8 @@ fn parse_args_into( | "no-mmap-dict" | "row-match-finder" | "no-row-match-finder" => {} - // Forces literals compressed or stored, which changes the - // frame that comes out. The encoder has no such switch here, - // so accepting the flag would hand back the other layout. - "compress-literals" | "no-compress-literals" => { - bail!("--{long} is not implemented"); - } + "compress-literals" => opts.literals = LiteralCompressionMode::Enable, + "no-compress-literals" => opts.literals = LiteralCompressionMode::Disable, _ => { if long == "fast" { // `--fast` is the level -1 alias. @@ -928,10 +1105,28 @@ fn parse_args_into( if v != "zstd" { bail!("--format={v} is not supported; this build only writes zstd"); } - } else if long == "rsyncable" || long.starts_with("patch-from") { - // Both change the emitted frame, so silence would be a - // wrong answer rather than a slower one. - bail!("--{long} is not implemented"); + } else if let Some(v) = long.strip_prefix("zstd=") { + opts.advanced = parse_advanced_params(v)?; + } else if let Some(v) = long.strip_prefix("train-cover=") { + select_mode(&mut opts, Mode::Train); + opts.trainer = Trainer::Cover; + opts.trainer_params = parse_trainer_params(v, false)?; + } else if let Some(v) = long.strip_prefix("train-fastcover=") { + select_mode(&mut opts, Mode::Train); + opts.trainer = Trainer::FastCover; + opts.trainer_params = parse_trainer_params(v, true)?; + } else if let Some(reference) = + option_value(long, "patch-from", arg_os, &mut iter)? + { + // A patch needs the levels that reach far back, so the + // reference command unlocks the ultra levels with it. + opts.patch_from = Some(reference); + ultra = true; + } else if long == "rsyncable" { + // Synchronisation points are cut between the jobs of a + // multi-threaded run, which this build does not have; + // the reference command refuses the pair too. + bail!("--rsyncable is not compatible with single-thread mode"); } else if long == "long" { // Bare `--long` is `--long=27` upstream. The window is // the point of the flag, so leaving the level's own one @@ -1161,12 +1356,33 @@ fn parse_args_into( } else { opts.level }; - if opts.long && compresses(&opts) && long_level < MIN_LONG_LEVEL { + // A `--zstd=strat=` override onto the optimal parser carries the matcher + // whatever the level says. + let optimal_strategy = opts + .advanced + .strategy + .is_some_and(|strategy| strategy >= Strategy::Btopt); + if opts.long && compresses(&opts) && long_level < MIN_LONG_LEVEL && !optimal_strategy { bail!( "--long needs level {MIN_LONG_LEVEL} or above, where long-distance \ matching runs; at level {long_level} it would only widen the window", ); } + // `--zstd=` knobs are validated here, before any file is opened: a window + // the decoder cannot read back is refused like `--long=N` is, and a knob + // out of its range is a broken command line. + if let Some(log) = opts.advanced.window_log { + check_window_log(log)?; + } + if compresses(&opts) { + frame_parameters( + CompressionLevel::from_level(opts.level), + &FrameSettings::from_options(&opts), + )?; + } + if opts.patch_from.is_some() && opts.dict.is_some() { + bail!("error : can't use -D and --patch-from=# at the same time"); + } Ok(Parsed::Run(Box::new(opts))) } @@ -1353,8 +1569,12 @@ Advanced compression options: --ultra Enable levels beyond 19, up to 22; requires more memory. --fast[=#] Use to very fast compression levels. [Default: 1] --long[=#] Enable long distance matching with window log #. [Default: 27] - Available from level 16 up, where long-distance matching runs; - capped at 27, the window this build can read back. + Available from level 16 up (or with --zstd=strat=7..9), where + long-distance matching runs; capped at 27, the window this + build can read back. + --patch-from=REF Use REF as the reference point for Zstandard's diff engine. + --zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#[,lhlog=#,lmml=#,lblog=#,lhrlog=#] + Override the level's compression parameters knob by knob. --exclude-compressed Only compress files that are not already compressed. --stream-size=# Specify size of streaming input from STDIN. @@ -1364,6 +1584,7 @@ Advanced compression options: --no-dictID Don't write `dictID` into the header (dictionary compression only). --[no-]content-size Write the input size into the frame header when it is known. [Default: Write] + --[no-]compress-literals Force (un)compressed literals. --format=zstd Compress files to the `.zst` format. [Default] @@ -1375,7 +1596,9 @@ Advanced decompression options: Dictionary builder: --train Create a dictionary from a training set of files. - --train-fastcover Use the fast cover algorithm (the trainer --train also runs). + --train-cover Use the cover algorithm (takes no tuning here). + --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] + Use the fast cover algorithm (with optional arguments). -o NAME Use NAME as dictionary name. [Default: dictionary] --maxdict=# Limit dictionary to specified size #. [Default: 112640] --dictID=# Force dictionary ID to #. [Default: Random] @@ -1383,22 +1606,51 @@ Dictionary builder: Benchmark options: -b# Perform benchmarking with compression level #. [Default: 3] -e# Test all compression levels up to #; starting level is `-b#`. [Default: 1] - -i# Set the minimum evaluation to time # seconds. [Default: 1] + -i# Set the minimum evaluation to time # seconds. [Default: 3] -S Output one benchmark result per input file. [Default: Consolidated result] -D dictionary Benchmark using dictionary Environment: ZSTD_CLEVEL sets the default compression level; ZSTD_NBTHREADS is read and validated. Accepted for compatibility, with no effect here: -T#/--threads=#, --single-thread, ---auto-threads, -B#, --block-size=#, --adapt, --[no-]sparse, --[no-]asyncio, ---[no-]mmap-dict, --[no-]row-match-finder (compression runs single-threaded). +--auto-threads, -B#, --block-size=#, --adapt, --zstd=ovlog=#, --[no-]sparse, +--[no-]asyncio, --[no-]mmap-dict, --[no-]row-match-finder (compression runs +single-threaded). Rejected rather than ignored, because they would change the result: --format= -other than zstd, --patch-from, --rsyncable, --[no-]compress-literals, ---train-cover, --train-legacy, and -M/--memory below the enforced ceiling when -decoding. A new output file keeps its source's permissions. +other than zstd, --rsyncable (needs worker threads), --train-legacy, shrink in +the trainer tuning, and -M/--memory below the enforced ceiling when decoding. +A new output file keeps its source's permissions. "; +/// The file the run's dictionary comes from: `-D`, or the `--patch-from` +/// reference, which is a dictionary by another name. The command line refuses +/// both at once, so at most one is set. +fn dictionary_path(opts: &Options) -> Option<&Path> { + opts.dict.as_deref().or(opts.patch_from.as_deref()) +} + +/// The window a `--patch-from` compression runs with: wide enough to reach +/// back over the whole input (`highbit(size) + 1`, as the reference command +/// sizes it), within what this build can read back. A larger input cannot be +/// patched here, since the frame would declare a window the decoder refuses. +fn patch_window_log(source_size: u64) -> Result { + use structured_zstd::encoding::CParameter; + + let file_window_log = u64::BITS - source_size.max(1).leading_zeros(); + let lower = u32::try_from(CParameter::WindowLog.bounds().lower_bound) + .expect("the window log lower bound is a small positive number"); + let decodable = structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE.ilog2(); + if file_window_log > decodable { + bail!( + "Can't handle files larger than {} MiB with --patch-from: the patch would \ + declare a window this build refuses to decode", + structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE >> 20 + ); + } + Ok(file_window_log.max(lower)) +} + /// Read the `-D` dictionary, if there is one, without breaking `-M` to do it. /// /// The limit was already weighed against what decoding alone needs; the @@ -1408,7 +1660,7 @@ decoding. A new output file keeps its source's permissions. /// then bounded by that same size, and a file that grew in between is an error /// rather than a silent truncation, which would corrupt the dictionary. fn load_dictionary(opts: &Options) -> Result>> { - let Some(path) = &opts.dict else { + let Some(path) = dictionary_path(opts) else { return Ok(None); }; // Listing walks frame headers and training builds a dictionary from its @@ -1506,7 +1758,19 @@ impl Dictionaries { /// An empty file is no dictionary rather than a broken one — loading a /// zero-size dictionary returns to no-dictionary mode — so `-D` on an empty /// file compresses plainly instead of failing. - fn prepare(raw: Option<&[u8]>, for_compression: bool, for_decoding: bool) -> Result { + /// + /// `raw_content` is `--patch-from`: the reference is content whatever it + /// starts with, the way `ZSTD_CCtx_refPrefix` takes it, so a reference that + /// happens to begin with the dictionary magic is not parsed as one. + fn prepare( + raw: Option<&[u8]>, + raw_content: bool, + for_compression: bool, + for_decoding: bool, + ) -> Result { + use structured_zstd::decoding::{Dictionary, DictionaryHandle}; + use structured_zstd::encoding::EncoderDictionary; + let Some(raw) = raw.filter(|raw| !raw.is_empty()) else { return Ok(Self::default()); }; @@ -1517,18 +1781,25 @@ impl Dictionaries { // parsing first and handing over the content would key it on the // wrong size. Whatever `-D` was pointed at: a trained dictionary, // or any file at all, taken as raw content the way upstream does. - prepared.encoder = Some( - structured_zstd::encoding::EncoderDictionary::from_serialized_or_raw_content(raw) - .map_err(|err| eyre!("invalid dictionary: {err:?}"))?, - ); + // Raw content has no tables, so its content length is its length. + let dictionary = if raw_content { + Dictionary::from_raw_content(0, raw.to_vec()) + .map(EncoderDictionary::from_dictionary) + } else { + EncoderDictionary::from_serialized_or_raw_content(raw) + }; + prepared.encoder = + Some(dictionary.map_err(|err| eyre!("invalid dictionary: {err:?}"))?); } if for_decoding { - prepared.decoder = Some( - structured_zstd::decoding::DictionaryHandle::from_dictionary( - structured_zstd::decoding::Dictionary::from_serialized_or_raw_content(raw) - .map_err(|err| eyre!("failed to parse dictionary: {err:?}"))?, - ), - ); + let dictionary = if raw_content { + Dictionary::from_raw_content(0, raw.to_vec()) + } else { + Dictionary::from_serialized_or_raw_content(raw) + }; + prepared.decoder = Some(DictionaryHandle::from_dictionary( + dictionary.map_err(|err| eyre!("failed to parse dictionary: {err:?}"))?, + )); } Ok(prepared) } @@ -1604,6 +1875,35 @@ fn run(mut opts: Options) -> Result { if opts.mode == Mode::Test { opts.remove_source = false; } + if opts.patch_from.is_some() { + // A patch is one input against one reference: the reference command + // refuses several, and stdin only with a declared length, since the + // window is sized from it. + if opts.inputs.len() > 1 { + bail!("error : can't use --patch-from=# on multiple files"); + } + if compresses(&opts) { + let source_size = match (opts.pledged_size, opts.inputs.first()) { + (Some(size), _) => size, + (None, Some(input)) if input != Path::new("-") => fs::metadata(input) + .map_err(|err| eyre!("can't stat {} : {err}", input.display()))? + .len(), + _ => bail!("Using --patch-from with stdin requires --stream-size"), + }; + opts.advanced.window_log = Some(patch_window_log(source_size)?); + // Long-distance matching is what finds the reference across a + // window this wide; it runs on the optimal parser here, so it is + // switched on where that parser runs. + let optimal = opts.level >= MIN_LONG_LEVEL + || opts + .advanced + .strategy + .is_some_and(|strategy| strategy >= Strategy::Btopt); + if optimal { + opts.long = true; + } + } + } let dict_bytes = load_dictionary(&opts)?; @@ -1615,7 +1915,12 @@ fn run(mut opts: Options) -> Result { run_benchmark(&opts, dict_bytes)?; return Ok(0); } - let dicts = Dictionaries::prepare(dict_bytes.as_deref(), compresses(&opts), decodes(&opts))?; + let dicts = Dictionaries::prepare( + dict_bytes.as_deref(), + opts.patch_from.is_some(), + compresses(&opts), + decodes(&opts), + )?; // Everything from here on primes from the parsed form, so the blob it was // parsed out of is released rather than held for the length of the run // beside the thing that replaced it. @@ -1644,7 +1949,7 @@ fn run(mut opts: Options) -> Result { // by the time anything is written, and what it writes is plaintext that // never needed it: the reference command permits that, and refusing would // break a working script to protect nothing. - if let (Some(output), Some(dict)) = (&opts.output, &opts.dict) + if let (Some(output), Some(dict)) = (&opts.output, dictionary_path(&opts)) && !opts.to_stdout && opts.mode == Mode::Compress && names_the_same_file(output, dict)? @@ -1852,7 +2157,7 @@ fn process_separately(opts: &Options, dicts: &Dictionaries, total: usize) -> Res // `./foo.zst` and `dir/../dir/foo.zst` name one file, and a match // on the string alone would miss two of the three. Compression // only, for the reason given at the `-o` check in `run`. - if let Some(dict) = &opts.dict + if let Some(dict) = dictionary_path(opts) && opts.mode == Mode::Compress && names_the_same_file(&output, dict)? { @@ -1870,7 +2175,7 @@ fn process_separately(opts: &Options, dicts: &Dictionaries, total: usize) -> Res // there to be compared, so identity is asked of the filesystem as // well: a hard link is a second name for one file, and no amount of // resolving either name tells them apart. - if let Some(dict) = &opts.dict + if let Some(dict) = dictionary_path(opts) && opts.remove_source && !opts.keep && (names_the_same_file(input, dict)? @@ -2182,7 +2487,7 @@ fn run_benchmark(opts: &Options, dict: Option>) -> Result<()> { // here, once, rather than inside the timed loops below. The blob is then // done with: it is released before the measuring starts rather than held // beside the two forms parsed out of it for the rest of the run. - let dicts = &Dictionaries::prepare(dict.as_deref(), true, true)?; + let dicts = &Dictionaries::prepare(dict.as_deref(), opts.patch_from.is_some(), true, true)?; drop(dict); if opts.bench_separately { @@ -2195,12 +2500,12 @@ fn run_benchmark(opts: &Options, dict: Option>) -> Result<()> { } let data = read_inputs_bounded(&opts.inputs, &sizes)?; - let label = opts - .inputs - .iter() - .map(|input| input.display().to_string()) - .collect::>() - .join(", "); + // Several inputs are one measurement, named by their count as the + // reference command names it. + let label = match opts.inputs.as_slice() { + [only] => only.display().to_string(), + many => format!(" {} files", many.len()), + }; benchmark_one(opts, dicts, &label, &data) } @@ -2259,12 +2564,23 @@ fn benchmark_one(opts: &Options, dicts: &Dictionaries, label: &str, data: &[u8]) } // Per-level time budget; best (fastest) pass wins, like upstream's -i loop. let mb = data.len() as f64 / 1e6; - println!( - "benchmarking {label} ({}) levels {}..={}", - HumanSize::new(data.len() as u64, false), + let name = bench_display_name(label); + display!( + opts.verbosity, + 3, + "Benchmarking {label} from level {} to {}", opts.bench_start, - opts.bench_end, + opts.bench_end ); + if opts.verbosity == 1 { + // The reference command's machine-readable header, for scripts that + // drive `-b -q`. + println!( + "bench {UPSTREAM_VERSION} : input {} bytes, {} seconds, 0 KB blocks", + data.len(), + opts.bench_secs as u64 + ); + } // The two buffers the measurement fills, sized once from what they will // hold: the frame can be no larger than `compress_bound` says, and the @@ -2317,7 +2633,6 @@ fn benchmark_one(opts: &Options, dicts: &Dictionaries, label: &str, data: &[u8]) } } - let ratio = data.len() as f64 / compressed.len() as f64; let c_speed = if best_compress > 0.0 { mb / best_compress } else { @@ -2328,20 +2643,93 @@ fn benchmark_one(opts: &Options, dicts: &Dictionaries, label: &str, data: &[u8]) } else { f64::INFINITY }; - println!( - "{level:>3} {:>10} {ratio:>7.3} {c_speed:>7.1} MB/s comp {d_speed:>8.1} MB/s decomp", - HumanSize::new(compressed.len() as u64, false), - ); + let result = BenchResult { + level, + input: data.len() as u64, + output: compressed.len() as u64, + compress_mb_s: c_speed, + decompress_mb_s: d_speed, + }; + if opts.verbosity >= DEFAULT_LEVEL { + println!("{}", result.line(&name)); + } else if opts.verbosity == 1 { + println!("{}", result.quiet_line(&name)); + } } Ok(()) } +/// What one level of a benchmark measured. +struct BenchResult { + level: i32, + input: u64, + output: u64, + compress_mb_s: f64, + decompress_mb_s: f64, +} + +impl BenchResult { + /// Compression ratio, as the reference command computes it. + fn ratio(&self) -> f64 { + self.input as f64 / self.output as f64 + } + + /// The reference command's result line at the default display level: + /// `%2i#%-17.17s :%10u ->%10u (x%5.*f), %6.*f MB/s, %6.1f MB/s`, the ratio + /// shown to three significant figures and the compression speed to two + /// decimals below 10 MB/s. + fn line(&self, name: &str) -> String { + let ratio = self.ratio(); + let ratio_digits = 1 + usize::from(ratio < 100.0) + usize::from(ratio < 10.0); + let speed_digits = if self.compress_mb_s < 10.0 { 2 } else { 1 }; + format!( + "{:>2}#{:<17}:{:>10} ->{:>10} (x{:>5.ratio_digits$}), {:>6.speed_digits$} MB/s, {:>6.1} MB/s", + self.level, + name, + self.input, + self.output, + ratio, + self.compress_mb_s, + self.decompress_mb_s, + ) + } + + /// The reference command's line under `-q`, which its own speed scripts + /// parse: `-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s`. + fn quiet_line(&self, name: &str) -> String { + format!( + "-{:<3}{:>11} ({:>5.3}) {:>6.2} MB/s {:>6.1} MB/s {}", + self.level, + self.output, + self.ratio(), + self.compress_mb_s, + self.decompress_mb_s, + name, + ) + } +} + +/// The name a benchmark line carries: the file's own name, cut to its last +/// 17 characters as the reference command cuts it. +fn bench_display_name(label: &str) -> String { + let name = Path::new(label) + .file_name() + .map_or(label, |name| name.to_str().unwrap_or(label)); + let chars = name.chars().count(); + if chars > 17 { + name.chars().skip(chars - 17).collect() + } else { + name.to_string() + } +} + /// `--train`: build a FastCOVER dictionary from the concatenated sample files /// and write it to `-o` (default `dictionary`). Mirrors upstream /// `zstd --train FILEs -o dict --maxdict=N [--dictID=N]`. fn train_dictionary(opts: &Options) -> Result<()> { use structured_zstd::dictionary::{ - FastCoverOptions, FinalizeOptions, create_fastcover_dict_from_slice, + FinalizeOptions, create_fastcover_dict_from_slice, create_raw_dict_from_source, + finalize_raw_dict, }; if opts.inputs.iter().any(|input| input == Path::new("-")) { @@ -2445,19 +2833,50 @@ fn train_dictionary(opts: &Options) -> Result<()> { samples.push(metadata); } + let finalize = FinalizeOptions { + dict_id: opts.dict_id, + }; let mut dict = Vec::new(); - // From the slice, not through a reader: the corpus is the largest thing - // this run holds, and the reader path buffers it a second time inside. - create_fastcover_dict_from_slice( - corpus.as_slice(), - &mut dict, - opts.max_dict, - &FastCoverOptions::default(), - FinalizeOptions { - dict_id: opts.dict_id, - }, - ) - .map_err(|err| eyre!("dictionary training failed: {err}"))?; + match opts.trainer { + Trainer::FastCover => { + let options = fastcover_options(&opts.trainer_params)?; + // From the slice, not through a reader: the corpus is the largest + // thing this run holds, and the reader path buffers it a second + // time inside. + create_fastcover_dict_from_slice( + corpus.as_slice(), + &mut dict, + opts.max_dict, + &options, + finalize, + ) + .map_err(|err| eyre!("dictionary training failed: {err}"))?; + } + Trainer::Cover => { + // The COVER trainer here scores segments by k-mer frequency, as the + // reference's does, but is not parameterised the same way: `k`, + // `d`, `steps` and `split` name knobs it does not have, and + // `shrink` a pass it does not run. Running it anyway would return + // a dictionary trained under different terms than the ones typed. + if !opts.trainer_params.is_default() { + bail!( + "--train-cover takes no tuning here (k, d, steps, split, shrink); \ + use --train-fastcover=... for a tunable trainer" + ); + } + let mut raw = Vec::new(); + create_raw_dict_from_source(corpus.as_slice(), corpus.len(), &mut raw, opts.max_dict) + .map_err(|err| eyre!("dictionary training failed: {err}"))?; + if raw.is_empty() { + bail!("dictionary training failed: the samples yield no dictionary content"); + } + // The trainer writes its most valuable segment last, and + // finalizing keeps the tail when the header leaves less room than + // was asked for, so the best content survives the cut. + dict = finalize_raw_dict(raw.as_slice(), corpus.as_slice(), opts.max_dict, finalize) + .map_err(|err| eyre!("dictionary training failed: {err}"))?; + } + } // A trained dictionary is an output file like any other, so it is written // through a temporary that is renamed into place: an interrupted run @@ -2502,6 +2921,74 @@ fn train_dictionary(opts: &Options) -> Result<()> { Ok(()) } +/// The FastCOVER tuning `--train-fastcover=...` asked for, checked the way the +/// reference trainer checks it: `d` is 6 or 8, `f` lies in `1..=31`, `accel` +/// in `1..=10`, `k` is at least `d`, `split` is a percentage. Naming both `k` +/// and `d` fixes them and skips the parameter search; naming `steps` widens +/// or narrows the search over `k` instead. +fn fastcover_options( + params: &TrainerParams, +) -> Result { + use structured_zstd::dictionary::FastCoverOptions; + + if params.shrink { + bail!("--train-fastcover shrink is not implemented"); + } + let mut options = FastCoverOptions::default(); + if let Some(d) = params.d { + if d != 6 && d != 8 { + bail!("--train-fastcover d must be 6 or 8, got {d}"); + } + options.d = d as usize; + options.d_candidates = vec![d as usize]; + } + if let Some(f) = params.f { + if f == 0 || f > 31 { + bail!("--train-fastcover f must be in 1..=31, got {f}"); + } + options.f = f; + options.f_candidates = vec![f]; + } + if let Some(accel) = params.accel { + if accel == 0 || accel > 10 { + bail!("--train-fastcover accel must be in 1..=10, got {accel}"); + } + options.accel = accel as usize; + } + if let Some(split) = params.split_percent { + if split > 100 { + bail!("--train-fastcover split is a percentage, got {split}"); + } + // Zero asks for the default, as it does there. + if split > 0 { + options.split_point = f64::from(split) / 100.0; + } + } + match (params.k, params.steps) { + (Some(k), _) => { + if (k as usize) < options.d { + bail!( + "--train-fastcover k must be at least d, got k={k} d={}", + options.d + ); + } + options.k = k as usize; + options.k_candidates = vec![k as usize]; + } + (None, Some(steps)) => { + // The reference searches `k` over 50..=2000 in `steps` strides. + const K_MIN: usize = 50; + const K_MAX: usize = 2000; + let stride = ((K_MAX - K_MIN) / steps.max(1) as usize).max(1); + options.k_candidates = (K_MIN..=K_MAX).step_by(stride).collect(); + } + (None, None) => {} + } + // With both `k` and `d` given there is nothing left to search for. + options.optimize = !(params.k.is_some() && params.d.is_some()); + Ok(options) +} + /// The permissions a file made from all of `samples` may carry: every bit that /// each of them grants, and no other. /// @@ -3437,6 +3924,10 @@ struct FrameSettings { content_size_flag: bool, /// Whether a dictionary frame records the dictionary's ID (`--no-dictID`). dict_id_flag: bool, + /// Per-knob overrides from `--zstd=...`. + advanced: AdvancedParams, + /// Whether literals are entropy-coded (`--[no-]compress-literals`). + literals: LiteralCompressionMode, } impl Default for FrameSettings { @@ -3455,6 +3946,8 @@ impl Default for FrameSettings { checksum: true, content_size_flag: true, dict_id_flag: true, + advanced: AdvancedParams::default(), + literals: LiteralCompressionMode::Auto, } } } @@ -3474,8 +3967,71 @@ impl FrameSettings { checksum: opts.checksum, content_size_flag: opts.content_size_flag, dict_id_flag: opts.dict_id_flag, + advanced: opts.advanced, + literals: opts.literals, + } + } +} + +/// The fine-grained parameters a frame runs under, or `None` when nothing +/// overrides the level: `--zstd=` knobs, `--long` (whose window yields to a +/// `--zstd=wlog=`, as the reference command has it), and the literal mode. +/// The long-distance knobs from `--zstd=` reach the encoder only with +/// `--long`, since there they tune a matcher that otherwise does not run. +fn frame_parameters( + level: CompressionLevel, + settings: &FrameSettings, +) -> Result> { + let advanced = settings.advanced; + if !settings.long && advanced.is_default() && settings.literals == LiteralCompressionMode::Auto + { + return Ok(None); + } + let mut builder = CompressionParameters::builder(level); + let window_log = advanced + .window_log + .or(settings.long.then_some(settings.long_window_log).flatten()); + if let Some(log) = window_log { + builder = builder.window_log(log); + } + if let Some(log) = advanced.chain_log { + builder = builder.chain_log(log); + } + if let Some(log) = advanced.hash_log { + builder = builder.hash_log(log); + } + if let Some(log) = advanced.search_log { + builder = builder.search_log(log); + } + if let Some(length) = advanced.min_match { + builder = builder.min_match(length); + } + if let Some(length) = advanced.target_length { + builder = builder.target_length(length); + } + if let Some(strategy) = advanced.strategy { + builder = builder.strategy(strategy); + } + if settings.long { + builder = builder.enable_long_distance_matching(true); + if let Some(log) = advanced.ldm_hash_log { + builder = builder.ldm_hash_log(log); + } + if let Some(length) = advanced.ldm_min_match { + builder = builder.ldm_min_match(length); + } + if let Some(log) = advanced.ldm_bucket_size_log { + builder = builder.ldm_bucket_size_log(log); + } + if let Some(log) = advanced.ldm_hash_rate_log { + builder = builder.ldm_hash_rate_log(log); } } + builder = builder.literal_compression(settings.literals); + builder + .build() + .map(Some) + .map_err(|err| eyre!("invalid compression parameters: {err}")) } /// How a stream is decoded: whether a stored checksum is compared, and what @@ -3524,12 +4080,11 @@ fn compress_stream( store, pledged_size, size_hint, - long, - long_window_log, target_block_size, checksum, content_size_flag, dict_id_flag, + .. } = settings; let compression_level = if store { CompressionLevel::Uncompressed @@ -3558,23 +4113,13 @@ fn compress_stream( .set_target_block_size(Some(target)) .wrap_err("failed to set the block-size target")?; } - // Long-distance matching (`--long`) is a per-knob override applied via the - // compression-parameters API; skip it for `--store` (raw frames don't match). - if long && !store { - let mut builder = - structured_zstd::encoding::CompressionParameters::builder(compression_level) - .enable_long_distance_matching(true); - // `--long=N` asked for a specific back-reference distance; without it - // the level's own window stands. - if let Some(log) = long_window_log { - builder = builder.window_log(log); - } - let params = builder - .build() - .map_err(|err| eyre!("failed to build LDM parameters: {err:?}"))?; + // `--long`, `--zstd=` and the literal mode are per-knob overrides applied + // via the compression-parameters API; skipped for `--store`, whose raw + // frames match nothing. + if !store && let Some(params) = frame_parameters(compression_level, settings)? { encoder .set_parameters(¶ms) - .wrap_err("failed to enable long-distance matching")?; + .wrap_err("failed to apply the compression parameters")?; } if let Some(size) = pledged_size { // The size is known exactly (a regular file, or `--stream-size`), so diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 961a1c4a8..a9df59a3a 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -31,7 +31,7 @@ fn benchmark_budget(input_len: u64, levels: std::ops::RangeInclusive) -> u6 /// The same blob a `-D` run would hand the codecs, parsed for both directions /// so one helper serves a compressing test and a decoding one alike. fn prepared_dict(raw: &[u8]) -> Dictionaries { - Dictionaries::prepare(Some(raw), true, true).expect("the fixture dictionary must parse") + Dictionaries::prepare(Some(raw), false, true, true).expect("the fixture dictionary must parse") } /// What a plain `zstd` invocation presets, before any flag. @@ -1691,18 +1691,19 @@ fn training_refuses_to_overwrite_without_force() { } /// The trainer flags name algorithms, and the algorithm decides what the -/// dictionary contains. Only FastCOVER is implemented here, so the flags that -/// ask for COVER or the legacy trainer have to say no — running FastCOVER under -/// their name returns a dictionary the caller did not ask for. +/// dictionary contains. FastCOVER and COVER are here; the legacy trainer is +/// not, so its flag has to say no rather than run another trainer under its +/// name. #[test] -fn unimplemented_trainers_are_refused_not_substituted() { +fn the_legacy_trainer_is_refused_not_substituted() { assert_eq!(parse(&["--train", "s1"]).unwrap().mode, Mode::Train); assert_eq!( parse(&["--train-fastcover", "s1"]).unwrap().mode, Mode::Train ); - assert!(parse(&["--train-cover", "s1"]).is_err()); + assert_eq!(parse(&["--train-cover", "s1"]).unwrap().mode, Mode::Train); assert!(parse(&["--train-legacy", "s1"]).is_err()); + assert!(parse(&["--train-legacy=s=8", "s1"]).is_err()); } /// A window is a promise about how much memory decoding will need, so it is @@ -2264,7 +2265,6 @@ fn unimplemented_output_changing_flags_are_rejected() { for args in [ &["--format=gzip", "f"][..], &["--format=xz", "f"][..], - &["--patch-from=ref", "f"][..], &["--rsyncable", "f"][..], ] { assert!( @@ -2427,7 +2427,7 @@ fn the_memory_limit_counts_what_a_benchmark_holds() { let path = dir.join(format!("szstd-benchmem-{}.bin", std::process::id())); fs::write(&path, vec![0u8; 64 * 1024]).unwrap(); - let mut opts = parse(&["-b3", "f"]).unwrap(); + let mut opts = parse(&["-b3", "-i1", "f"]).unwrap(); opts.inputs = vec![path.clone()]; // 64 KiB in and 64 KiB back out, against a limit with 32 KiB of headroom // above the decoder's own floor. @@ -2467,7 +2467,7 @@ fn separate_benchmarking_measures_one_file_at_a_time() { // Measured one at a time, only one file is in memory at once — so a limit // that fits a single file is enough, while the concatenation needs both. // Sized the way the run sizes what it holds, for one file. - let mut opts = parse(&["-b3", "-S", "f"]).unwrap(); + let mut opts = parse(&["-b3", "-S", "-i1", "f"]).unwrap(); opts.inputs = vec![one.clone(), two.clone()]; opts.memory_limit = Some(benchmark_budget(32 * 1024, 3..=3)); let separately = run_benchmark(&opts, None); @@ -2500,7 +2500,7 @@ fn benchmarking_refuses_inputs_that_are_not_regular_files() { return; } - let mut opts = parse(&["-b3", "f"]).unwrap(); + let mut opts = parse(&["-b3", "-i1", "f"]).unwrap(); opts.inputs = vec![fifo.clone()]; let refused = run_benchmark(&opts, None); let _ = fs::remove_file(&fifo); @@ -2578,7 +2578,7 @@ fn the_memory_limit_counts_the_compressed_benchmark_buffer() { .collect(); fs::write(&path, &payload).unwrap(); - let mut opts = parse(&["-b3", "f"]).unwrap(); + let mut opts = parse(&["-b3", "-i1", "f"]).unwrap(); opts.inputs = vec![path.clone()]; // Room for two 64 KiB buffers above the decoder's floor, not for three. opts.memory_limit = @@ -2606,7 +2606,7 @@ fn benchmarking_refuses_the_stdin_marker() { let dash = dir.join("-"); fs::write(&dash, vec![1u8; 4096]).unwrap(); - let mut opts = parse(&["-b3", "f"]).unwrap(); + let mut opts = parse(&["-b3", "-i1", "f"]).unwrap(); // Exactly as the command line spells it, with the file there to be found. opts.inputs = vec![PathBuf::from("-")]; let previous = std::env::current_dir().unwrap(); @@ -2780,7 +2780,7 @@ fn the_memory_limit_counts_the_encoder_the_benchmark_builds() { "a compression pass allocates something to match with" ); - let mut opts = parse(&["-b3", "f"]).unwrap(); + let mut opts = parse(&["-b3", "-i1", "f"]).unwrap(); opts.inputs = vec![input.clone()]; // Room for everything but the encoder: it still has to fit beside them. @@ -2809,7 +2809,7 @@ fn the_memory_limit_counts_every_copy_of_the_dictionary() { let buffers = benchmark_budget(32 * 1024, 3..=3); - let mut opts = parse(&["-b3", "f"]).unwrap(); + let mut opts = parse(&["-b3", "-i1", "f"]).unwrap(); opts.inputs = vec![input.clone()]; // Room for the buffers and two dictionaries: still one short. @@ -2835,7 +2835,7 @@ fn the_memory_limit_counts_the_dictionary_and_the_benchmark_together() { fs::write(&input, vec![0u8; 32 * 1024]).unwrap(); let dictionary = vec![0u8; 32 * 1024]; - let mut opts = parse(&["-b3", "f"]).unwrap(); + let mut opts = parse(&["-b3", "-i1", "f"]).unwrap(); opts.inputs = vec![input.clone()]; // Room for exactly what the benchmark holds, and so none to spare for a // dictionary beside it. @@ -3304,15 +3304,6 @@ fn stdout_and_output_follow_last_option_wins() { assert_eq!(stdout_last.output, None); } -/// `--[no-]compress-literals` forces literals compressed or stored, which -/// changes the emitted frame. The encoder has no such switch here, so -/// accepting the flag would hand back a frame laid out the other way. -#[test] -fn literal_mode_flags_are_rejected_until_wired() { - assert!(parse(&["--compress-literals", "f"]).is_err()); - assert!(parse(&["--no-compress-literals", "f"]).is_err()); -} - /// Concatenating frames is a documented property of the format: `cat a.zst /// b.zst` decodes to `a` followed by `b`, which is how `tar` archives and /// append-style logs are built. Stopping at the first frame loses the rest @@ -3402,7 +3393,7 @@ fn the_memory_limit_counts_the_encoder_the_dictionary_asks_for() { 64 KiB file: {with_dict} vs {plain}" ); - let mut opts = parse(&["-b5", "f"]).unwrap(); + let mut opts = parse(&["-b5", "-i1", "f"]).unwrap(); opts.inputs = vec![input.clone()]; // Everything the run holds, with the encoder weighed on the file alone: @@ -3421,3 +3412,406 @@ fn the_memory_limit_counts_the_encoder_the_dictionary_asks_for() { .expect_err("a ceiling weighed on the file alone does not cover the dictionary's tables"); accepted.expect("weighed on the dictionary's own parameters, the run fits"); } + +/// `--zstd=` takes the reference command's keys in both spellings, reads each +/// value as a leading number, treats zero as "the level's value", and refuses +/// a key it does not have or a value that is not a number. +#[test] +fn advanced_parameters_parse_the_reference_spellings() { + let short = + parse_advanced_params("wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6").unwrap(); + assert_eq!( + short, + AdvancedParams { + window_log: Some(23), + chain_log: Some(23), + hash_log: Some(22), + search_log: Some(6), + min_match: Some(3), + target_length: Some(48), + strategy: Some(Strategy::Btlazy2), + ..AdvancedParams::default() + } + ); + let long = parse_advanced_params( + "windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6", + ) + .unwrap(); + assert_eq!(short, long, "the long spellings are the same knobs"); + let ldm = parse_advanced_params("lhlog=20,lmml=64,lblog=3,lhrlog=7,ovlog=5").unwrap(); + assert_eq!(ldm.ldm_hash_log, Some(20)); + assert_eq!(ldm.ldm_min_match, Some(64)); + assert_eq!(ldm.ldm_bucket_size_log, Some(3)); + assert_eq!(ldm.ldm_hash_rate_log, Some(7)); + assert!( + parse_advanced_params("wlog=0,strat=0") + .unwrap() + .is_default(), + "zero is the level's own value" + ); + assert_eq!( + parse_advanced_params("tlen=1K").unwrap().target_length, + Some(1024), + "the reference reader's K multiplier applies" + ); + assert!(parse_advanced_params("").unwrap().is_default()); + assert!( + parse_advanced_params("strat=10").is_err(), + "no tenth strategy" + ); + assert!(parse_advanced_params("nope=1").is_err(), "unknown key"); + assert!(parse_advanced_params("wlog").is_err(), "no value"); + assert!(parse_advanced_params("wlog=abc").is_err(), "not a number"); + assert!(parse_advanced_params("wlog=23x").is_err(), "trailing junk"); + assert!(parse_advanced_params("wlog=23,").is_err(), "trailing comma"); +} + +/// The knobs reach the encoder: a `--zstd=wlog=` window is what the frame +/// declares, it wins over the window `--long` would set, and a window the +/// decoder cannot read back is refused at the command line like `--long=N`. +#[test] +fn advanced_parameters_reach_the_frame() { + use structured_zstd::decoding::read_frame_header_info; + + let opts = parse(&["--zstd=wlog=20,strat=7", "f"]).unwrap(); + assert_eq!(opts.advanced.window_log, Some(20)); + assert_eq!(opts.advanced.strategy, Some(Strategy::Btopt)); + assert!( + parse(&["--zstd=wlog=28", "f"]).is_err(), + "beyond what decodes" + ); + assert!( + parse(&["--zstd=mml=9", "f"]).is_err(), + "out of the knob's range" + ); + + // Big enough that the window is not capped by the source. + let payload = vec![0u8; 3 << 20]; + let mut frame = Vec::new(); + compress_stream( + payload.as_slice(), + &mut frame, + &FrameSettings { + level: 3, + pledged_size: Some(payload.len() as u64), + advanced: AdvancedParams { + window_log: Some(20), + ..AdvancedParams::default() + }, + ..FrameSettings::default() + }, + &no_dict(), + ) + .unwrap(); + assert_eq!( + read_frame_header_info(&frame, false).unwrap().window_size, + 1 << 20, + "the frame declares the window --zstd asked for" + ); + let mut frame = Vec::new(); + compress_stream( + payload.as_slice(), + &mut frame, + &FrameSettings { + level: 16, + long: true, + long_window_log: Some(27), + pledged_size: Some(payload.len() as u64), + advanced: AdvancedParams { + window_log: Some(21), + ..AdvancedParams::default() + }, + ..FrameSettings::default() + }, + &no_dict(), + ) + .unwrap(); + assert_eq!( + read_frame_header_info(&frame, false).unwrap().window_size, + 1 << 21, + "--zstd=wlog wins over the window --long would set" + ); +} + +/// `--long` below level 16 is refused because the matcher does not run there, +/// unless `--zstd=strat=` moves the level onto a parser where it does. +#[test] +fn a_strategy_override_onto_the_optimal_parser_admits_long() { + assert!(parse(&["-3", "--long", "f"]).is_err()); + assert!(parse(&["-3", "--long", "--zstd=strat=7", "f"]).is_ok()); + assert!(parse(&["-3", "--long", "--zstd=strat=9", "f"]).is_ok()); + assert!(parse(&["-3", "--long", "--zstd=strat=6", "f"]).is_err()); +} + +/// `--[no-]compress-literals` decides whether literal sections are +/// entropy-coded: forced raw, a literal-heavy input compresses worse than the +/// level's default; forced on at a negative level, where the default is raw, +/// it compresses better. +#[test] +fn literal_compression_flags_reach_the_frame() { + assert_eq!( + parse(&["--no-compress-literals", "f"]).unwrap().literals, + LiteralCompressionMode::Disable + ); + assert_eq!( + parse(&["--compress-literals", "f"]).unwrap().literals, + LiteralCompressionMode::Enable + ); + assert_eq!( + parse(&["f"]).unwrap().literals, + LiteralCompressionMode::Auto + ); + + let payload: Vec = (0..8192u32) + .map(|i| b'a' + (i.wrapping_mul(2_654_435_761) >> 27) as u8) + .collect(); + let frame_with = |level: i32, literals: LiteralCompressionMode| { + let mut frame = Vec::new(); + compress_stream( + payload.as_slice(), + &mut frame, + &FrameSettings { + level, + literals, + ..FrameSettings::default() + }, + &no_dict(), + ) + .unwrap(); + frame + }; + let auto = frame_with(3, LiteralCompressionMode::Auto); + let raw = frame_with(3, LiteralCompressionMode::Disable); + assert!(raw.len() > auto.len(), "{} vs {}", raw.len(), auto.len()); + assert_eq!(decoded(&raw).unwrap(), payload); + let fast_auto = frame_with(-3, LiteralCompressionMode::Auto); + let fast_coded = frame_with(-3, LiteralCompressionMode::Enable); + assert!( + fast_coded.len() < fast_auto.len(), + "{} vs {}", + fast_coded.len(), + fast_auto.len() + ); + assert_eq!(decoded(&fast_coded).unwrap(), payload); +} + +/// The trainer flags take their tuning the way the reference command reads +/// it, and the FastCOVER options built from it follow the reference's rules: +/// both `k` and `d` fix the parameters, `steps` widens the search over `k`, +/// and a value the trainer cannot take is refused. +#[test] +fn trainer_parameters_parse_and_build_options() { + let params = parse_trainer_params("k=200,d=8,f=20,steps=4,split=75,accel=2", true).unwrap(); + assert_eq!( + params, + TrainerParams { + k: Some(200), + d: Some(8), + f: Some(20), + steps: Some(4), + split_percent: Some(75), + accel: Some(2), + shrink: false, + } + ); + assert!(parse_trainer_params("shrink", false).unwrap().shrink); + assert!(parse_trainer_params("k=50,shrink=2", false).unwrap().shrink); + assert!( + parse_trainer_params("f=20", false).is_err(), + "cover has no f" + ); + assert!(parse_trainer_params("accel=2", false).is_err(), "nor accel"); + assert!(parse_trainer_params("k", true).is_err(), "no value"); + assert!(parse_trainer_params("k=x", true).is_err(), "not a number"); + assert!(parse_trainer_params("zzz=1", true).is_err(), "unknown key"); + + let fixed = fastcover_options(¶ms).unwrap(); + assert!(!fixed.optimize, "k and d given: nothing to search"); + assert_eq!((fixed.k, fixed.d, fixed.f, fixed.accel), (200, 8, 20, 2)); + assert_eq!(fixed.split_point, 0.75); + + let searched = fastcover_options(&TrainerParams { + steps: Some(10), + ..TrainerParams::default() + }) + .unwrap(); + assert!(searched.optimize); + assert_eq!( + searched.k_candidates.len(), + 11, + "50..=2000 in strides of 195" + ); + assert_eq!(searched.k_candidates[0], 50); + + let bad = |params: TrainerParams| fastcover_options(¶ms).is_err(); + assert!(bad(TrainerParams { + d: Some(7), + ..TrainerParams::default() + })); + assert!(bad(TrainerParams { + f: Some(32), + ..TrainerParams::default() + })); + assert!(bad(TrainerParams { + accel: Some(0), + ..TrainerParams::default() + })); + assert!(bad(TrainerParams { + k: Some(4), + d: Some(8), + ..TrainerParams::default() + })); + assert!(bad(TrainerParams { + split_percent: Some(101), + ..TrainerParams::default() + })); + assert!(bad(TrainerParams { + shrink: true, + ..TrainerParams::default() + })); + + let opts = parse(&["--train-fastcover=k=200,d=8", "s1"]).unwrap(); + assert_eq!(opts.mode, Mode::Train); + assert_eq!(opts.trainer, Trainer::FastCover); + assert_eq!(opts.trainer_params.k, Some(200)); + let opts = parse(&["--train-cover", "s1"]).unwrap(); + assert_eq!(opts.trainer, Trainer::Cover); + assert!(opts.trainer_params.is_default()); + assert!(parse(&["--train-legacy", "s1"]).is_err()); +} + +/// `--train-cover` trains with the COVER trainer and writes a real dictionary; +/// its reference-side tuning names knobs this trainer does not have, so a +/// tuned request is refused rather than trained under other terms. +#[test] +fn cover_training_writes_a_dictionary_and_refuses_tuning() { + let scratch = Scratch::new("cover"); + let corpus: Vec = (0..60_000u32) + .flat_map(|i| format!("record {} value {}\n", i % 500, (i * 7919) % 1000).into_bytes()) + .collect(); + let sample = scratch.file("samples.txt", &corpus); + let output = scratch.path().join("cover.dict"); + + let mut opts = parse(&["--train-cover", "-q", "--maxdict=8192", "s"]).unwrap(); + opts.inputs = vec![sample.clone()]; + opts.output = Some(output.clone()); + train_dictionary(&opts).expect("COVER training succeeds"); + let dictionary = fs::read(&output).unwrap(); + assert!(dictionary.len() <= 8192); + structured_zstd::decoding::Dictionary::decode_dict(&dictionary) + .expect("the output is a finalized dictionary"); + + let mut tuned = parse(&["--train-cover=k=50", "-q", "-f", "s"]).unwrap(); + tuned.inputs = vec![sample]; + tuned.output = Some(output); + let err = train_dictionary(&tuned) + .expect_err("tuning the reference's COVER has no meaning here") + .to_string(); + assert!(err.contains("takes no tuning"), "{err}"); +} + +/// `--patch-from REF` compresses against the reference as raw content with a +/// window wide enough to reach all of it, unlocks the ultra levels, and takes +/// the reference back on decompression. A patch of a file against a near +/// copy of itself is far smaller than the file compressed alone. +#[test] +fn patch_from_compresses_against_the_reference() { + let scratch = Scratch::new("patch"); + let old: Vec = (0..4000u32) + .flat_map(|i| format!("line {i}: the quick brown fox {}\n", i * 31 % 977).into_bytes()) + .collect(); + let mut new = old.clone(); + new.extend_from_slice(b"an appended line that the reference lacks\n"); + new[1000..1010].copy_from_slice(b"EDITEDHERE"); + let reference = scratch.file("old.txt", &old); + let input = scratch.file("new.txt", &new); + let patch = scratch.path().join("new.patch"); + + let opts = parse(&["--patch-from", "old", "-22", "f"]).unwrap(); + assert_eq!(opts.patch_from, Some(PathBuf::from("old"))); + assert_eq!(opts.level, 22, "--patch-from unlocks the ultra levels"); + assert!(parse(&["--patch-from=old", "-D", "dict", "f"]).is_err()); + + for level in [3, 19] { + let mut opts = parse(&["-q", "-f", "--patch-from", "r", "f"]).unwrap(); + opts.level = level; + opts.patch_from = Some(reference.clone()); + opts.inputs = vec![input.clone()]; + opts.output = Some(patch.clone()); + assert_eq!(run(opts).expect("patching runs"), 0, "level {level}"); + let patch_bytes = fs::read(&patch).unwrap(); + assert!( + patch_bytes.len() * 4 < frame_of(&new).len(), + "level {level}: a patch ({} bytes) is far smaller than the file compressed alone ({})", + patch_bytes.len(), + frame_of(&new).len() + ); + + let restored = scratch.path().join("restored.txt"); + let mut opts = parse(&["-d", "-q", "-f", "--patch-from", "r", "f"]).unwrap(); + opts.patch_from = Some(reference.clone()); + opts.inputs = vec![patch.clone()]; + opts.output = Some(restored.clone()); + assert_eq!(run(opts).expect("applying the patch runs"), 0); + assert_eq!(fs::read(&restored).unwrap(), new, "level {level}"); + } + + // One input against one reference, and stdin only with a declared length. + let mut two = parse(&["-q", "--patch-from", "r", "a", "b"]).unwrap(); + two.patch_from = Some(reference.clone()); + two.inputs = vec![input.clone(), input.clone()]; + assert!(run(two).is_err()); + let mut stdin = parse(&["-q", "--patch-from", "r"]).unwrap(); + stdin.patch_from = Some(reference); + let err = run(stdin) + .expect_err("stdin needs --stream-size") + .to_string(); + assert!(err.contains("--stream-size"), "{err}"); +} + +/// The patch window covers the input (`highbit(size) + 1`), is never below the +/// smallest window the format allows, and stops where the decoder would refuse +/// the frame. +#[test] +fn the_patch_window_covers_the_input_within_what_decodes() { + assert_eq!(patch_window_log(0).unwrap(), 10); + assert_eq!(patch_window_log(1).unwrap(), 10); + assert_eq!(patch_window_log(2000).unwrap(), 11); + assert_eq!(patch_window_log(1 << 20).unwrap(), 21); + assert_eq!(patch_window_log((1 << 27) - 1).unwrap(), 27); + assert!(patch_window_log(1 << 27).is_err()); +} + +/// `-b` prints its result in the reference command's layout, at the default +/// level and under `-q`, with the file name cut to its last 17 characters. +#[test] +fn benchmark_lines_follow_the_reference_layout() { + let result = BenchResult { + level: 3, + input: 7692, + output: 255, + compress_mb_s: 123.456, + decompress_mb_s: 1234.5, + }; + assert_eq!( + result.line("a.txt"), + " 3#a.txt : 7692 -> 255 (x30.16), 123.5 MB/s, 1234.5 MB/s" + ); + assert_eq!( + result.quiet_line("a.txt"), + "-3 255 (30.165) 123.46 MB/s 1234.5 MB/s a.txt" + ); + let slow = BenchResult { + compress_mb_s: 1.5, + output: 7000, + ..result + }; + assert!(slow.line("a.txt").contains("(x1.099), 1.50 MB/s")); + assert_eq!(bench_display_name("dir/a.txt"), "a.txt"); + assert_eq!( + bench_display_name("a-very-long-file-name.txt"), + "ong-file-name.txt", + "the last 17 characters" + ); + assert_eq!(bench_display_name(" 3 files"), " 3 files"); +} From 3692b034df6315b022d7dcce594d5209fbfa19bf Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 11 Sep 2026 03:27:01 +0300 Subject: [PATCH 4/5] fix(encode): reset the literal mode on a level switch - `set_compression_level` cleared the strategy, target-length and matcher overrides installed by `set_parameters` but kept the literal compression mode, so a compressor switched back to a bare level still wrote the previous parameters' raw (or forced) literals. - Reset it to `Auto` with the other overrides; the regression test compares the frame after such a switch with a fresh compressor's. Part of #128 --- zstd/src/encoding/frame_compressor.rs | 6 ++- zstd/src/encoding/frame_compressor/tests.rs | 41 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 511d0d9cb..8efaa62e4 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -3088,7 +3088,7 @@ impl FrameCompressor { /// This also clears any fine-grained parameter overrides installed via /// [`set_parameters`](Self::set_parameters): reverting to a bare level /// means plain level-based tuning, not the previous frame's customized - /// strategy / LDM / log overrides. To keep overriding, call + /// strategy / LDM / log / literal-mode overrides. To keep overriding, call /// [`set_parameters`](Self::set_parameters) again with the new base level. pub fn set_compression_level( &mut self, @@ -3104,9 +3104,11 @@ impl FrameCompressor { compression_level, CompressionLevel::Level(n) if n < 0 ); - // Drop sticky overrides so the level switch yields plain geometry. + // Drop sticky overrides so the level switch yields plain geometry, + // the literal mode included: the gate above is the bare level's rule. self.strategy_override = None; self.target_length_override = None; + self.literal_compression_mode = LiteralCompressionMode::Auto; self.state.matcher.clear_param_overrides(); old } diff --git a/zstd/src/encoding/frame_compressor/tests.rs b/zstd/src/encoding/frame_compressor/tests.rs index 8659cebd4..cd27555ae 100644 --- a/zstd/src/encoding/frame_compressor/tests.rs +++ b/zstd/src/encoding/frame_compressor/tests.rs @@ -2517,6 +2517,47 @@ fn set_compression_level_resyncs_literal_disable_for_negatives() { ); } +/// Regression: `set_compression_level` must forget a literal compression mode +/// installed by `set_parameters` along with the other overrides, so a +/// compressor switched back to a bare level emits the frame that level emits +/// on its own, not one carrying the previous parameters' raw literals. +#[cfg(feature = "std")] +#[test] +fn set_compression_level_forgets_the_literal_compression_mode() { + use super::CompressionLevel; + use crate::encoding::{CompressionParameters, LiteralCompressionMode}; + + // Literal-heavy input: 32 symbols with nothing for the match finder, so + // whether the literals are Huffman-coded decides the frame size. + let text: Vec = (0..8192u32) + .map(|i| b'a' + (i.wrapping_mul(2_654_435_761) >> 27) as u8) + .collect(); + let level = CompressionLevel::Level(3); + let raw_literals = CompressionParameters::builder(level) + .literal_compression(LiteralCompressionMode::Disable) + .build() + .unwrap(); + + let mut reused: FrameCompressor = FrameCompressor::new(level); + reused.set_parameters(&raw_literals); + let with_raw = reused.compress_independent_frame(&text); + reused.set_compression_level(level); + let after_switch = reused.compress_independent_frame(&text); + + let mut fresh: FrameCompressor = FrameCompressor::new(level); + let plain = fresh.compress_independent_frame(&text); + assert!( + with_raw.len() > plain.len(), + "the fixture must make the mode visible: {} vs {} bytes", + with_raw.len(), + plain.len() + ); + assert_eq!( + after_switch, plain, + "a bare level after set_parameters must compress as that level alone does" + ); +} + /// Regression: `set_compression_level` followed by `compress()` must /// refresh `state.strategy_tag` through the reset-time sync so the /// literal-compression gates (`min_literals_to_compress`, From 1b7f6df8ae1211ab3d3b0089aa2fc619446e4a28 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 11 Sep 2026 03:27:17 +0300 Subject: [PATCH 5/5] fix(cli): symlink loops, SIG_IGN, Windows Ctrl-C - `-r` under `-f` followed a link back to an ancestor round and round, listing each file once per nesting level until the path ran out of room. The walk now keeps the device and inode of the directories it is inside and reports a link that leads back into one instead of descending; the reference command (util.c, UTIL_prepareFileList) has the same loop. - The "every named input is a symbolic link" refusal fired before the `--filelist` entries were merged, failing a run whose list still named inputs. It is judged on the merged set now; upstream judges the named inputs alone (zstdcli.c, the followLinks block), a deliberate departure. - The SIGINT handler declared `signal` with a function-pointer type, so a returned SIG_IGN marker became an invalid `Option` value, and it replaced an inherited ignore. The disposition is an integer now, as C's sighandler_t is; a SIGINT the process inherited as ignored stays so, as gzip and xz keep it and as a background job of a non-interactive shell expects; `clear` restores what the first `guard` found. Upstream installs its handler unconditionally, another departure. - The same handler now runs on Windows through the C runtime's `signal`, `_wunlink` and `_exit`, removing the partial output and exiting with status 2 as on POSIX; only platforms with neither keep the no-op. Compiled and unit-tested for the Windows target; Ctrl-C delivery itself is not exercised by the tests. - `create_dir_if_missing` no longer declares a `mut` that only the Unix branch uses. Part of #128 --- zstd/src/bin/structured-zstd/inputs.rs | 117 ++++++-- zstd/src/bin/structured-zstd/inputs/tests.rs | 37 +++ zstd/src/bin/structured-zstd/interrupt.rs | 270 +++++++++++++++--- .../bin/structured-zstd/interrupt/tests.rs | 55 +++- 4 files changed, 418 insertions(+), 61 deletions(-) diff --git a/zstd/src/bin/structured-zstd/inputs.rs b/zstd/src/bin/structured-zstd/inputs.rs index 3e7f2a025..4e35c6ef1 100644 --- a/zstd/src/bin/structured-zstd/inputs.rs +++ b/zstd/src/bin/structured-zstd/inputs.rs @@ -6,7 +6,9 @@ //! `--output-dir-mirror`. The reference command does this in `zstdcli.c` and //! `util.c`, and the order of the steps is kept: symbolic links are dropped //! from the NAMED inputs before the file lists are merged, and directories are -//! expanded after. +//! expanded after. One departure: whether anything is left to do is judged +//! once the file lists are in. The reference command judges the named inputs +//! alone, which fails a run whose list still names usable inputs. use std::ffi::OsString; use std::fs; @@ -56,20 +58,29 @@ pub fn select_inputs( } files.push(input); } - if files.is_empty() && named_count > 0 { - bail!("every named input is a symbolic link; pass -f to follow them"); - } for list in filelists { files.extend(read_filelist(list)?); } + if files.is_empty() && named_count > 0 { + bail!("every named input is a symbolic link; pass -f to follow them"); + } let named = files.len(); if recursive { let mut expanded = Vec::with_capacity(files.len()); for input in files { - if fs::metadata(&input).is_ok_and(|m| m.is_dir()) { - walk_directory(&input, follow_links, verbosity, &mut expanded); - } else { - expanded.push(input); + match fs::metadata(&input) { + Ok(metadata) if metadata.is_dir() => { + let mut ancestors = Vec::new(); + descend( + &input, + &metadata, + follow_links, + verbosity, + &mut expanded, + &mut ancestors, + ); + } + _ => expanded.push(input), } } files = expanded; @@ -77,6 +88,60 @@ pub fn select_inputs( Ok(Selection { files, named }) } +/// What identifies a directory whatever name reaches it, so a walk notices +/// when a link has led it back to a directory it is already inside. +#[cfg(unix)] +type DirId = (u64, u64); +#[cfg(not(unix))] +type DirId = PathBuf; + +/// The identity of the directory at `path`, whose `metadata` (links followed) +/// is already in hand; the device and inode where the file system has them, +/// the canonical path elsewhere. +#[cfg(unix)] +fn dir_id(path: &Path, metadata: &fs::Metadata) -> Option { + use std::os::unix::fs::MetadataExt; + let _ = path; + Some((metadata.dev(), metadata.ino())) +} + +#[cfg(not(unix))] +fn dir_id(path: &Path, metadata: &fs::Metadata) -> Option { + let _ = metadata; + fs::canonicalize(path).ok() +} + +/// Walk `dir` unless the walk is already inside it. A link followed under +/// `-f`, or a bind mount, can lead back to an ancestor; entering it again +/// would list the tree once more per nesting level until the path ran out of +/// room, so the loop is reported and not descended. `ancestors` holds the +/// directories on the way down to `dir`. +fn descend( + dir: &Path, + metadata: &fs::Metadata, + follow_links: bool, + verbosity: i32, + out: &mut Vec, + ancestors: &mut Vec, +) { + let Some(id) = dir_id(dir, metadata) else { + walk_directory(dir, follow_links, verbosity, out, ancestors); + return; + }; + if ancestors.contains(&id) { + display!( + verbosity, + 2, + "Warning : {} leads back into a directory being walked, ignoring", + dir.display() + ); + return; + } + ancestors.push(id); + walk_directory(dir, follow_links, verbosity, out, ancestors); + ancestors.pop(); +} + /// Whether `path` itself is a symbolic link, whatever it points at. fn is_symlink(path: &Path) -> bool { fs::symlink_metadata(path).is_ok_and(|m| m.file_type().is_symlink()) @@ -160,8 +225,15 @@ fn bytes_to_path(bytes: &[u8]) -> PathBuf { /// Entries are taken in name order so two runs over one tree process it the /// same way; the reference command takes them in directory order, which the /// filesystem does not promise to keep. A directory that cannot be read is -/// reported and contributes nothing, as there too. -fn walk_directory(dir: &Path, follow_links: bool, verbosity: i32, out: &mut Vec) { +/// reported and contributes nothing, as there too. Subdirectories go through +/// [`descend`], which keeps a link from leading the walk round in a circle. +fn walk_directory( + dir: &Path, + follow_links: bool, + verbosity: i32, + out: &mut Vec, + ancestors: &mut Vec, +) { let entries = match fs::read_dir(dir) { Ok(entries) => entries, Err(err) => { @@ -196,10 +268,11 @@ fn walk_directory(dir: &Path, follow_links: bool, verbosity: i32, out: &mut Vec< ); continue; } - if fs::metadata(&path).is_ok_and(|m| m.is_dir()) { - walk_directory(&path, follow_links, verbosity, out); - } else { - out.push(path); + match fs::metadata(&path) { + Ok(metadata) if metadata.is_dir() => { + descend(&path, &metadata, follow_links, verbosity, out, ancestors); + } + _ => out.push(path), } } } @@ -286,14 +359,20 @@ fn create_dir_if_missing(dir: &Path, permissions: Option) -> st Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} Err(err) => return Err(err), } - let mut builder = fs::DirBuilder::new(); #[cfg(unix)] - if let Some(permissions) = &permissions { + let builder = { use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; - builder.mode(permissions.mode() & 0o7777); - } + let mut builder = fs::DirBuilder::new(); + if let Some(permissions) = &permissions { + builder.mode(permissions.mode() & 0o7777); + } + builder + }; #[cfg(not(unix))] - let _ = permissions; + let builder = { + let _ = permissions; + fs::DirBuilder::new() + }; builder.create(dir) } diff --git a/zstd/src/bin/structured-zstd/inputs/tests.rs b/zstd/src/bin/structured-zstd/inputs/tests.rs index 23c46de62..eaa017c21 100644 --- a/zstd/src/bin/structured-zstd/inputs/tests.rs +++ b/zstd/src/bin/structured-zstd/inputs/tests.rs @@ -90,6 +90,43 @@ fn named_symlinks_are_skipped_unless_links_are_followed() { assert!(err.contains("symbolic link"), "the refusal says why: {err}"); } +/// A run whose named inputs were all links still has the inputs its +/// `--filelist` names: the "nothing left" refusal is judged on the merged set, +/// not on the command line alone. +#[cfg(unix)] +#[test] +fn a_filelist_keeps_the_run_alive_when_every_named_input_is_a_link() { + let scratch = Scratch::new("linklist"); + let target = scratch.file("target.txt"); + let link = scratch.path().join("link.txt"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + let list = scratch.path().join("list.txt"); + fs::write(&list, format!("{}\n", target.display())).unwrap(); + + let selection = select_inputs(vec![link], &[list], false, false, 0) + .expect("the list still supplies an input"); + assert_eq!(selection.files, vec![target]); +} + +/// Under `-f` a link back to an ancestor directory is entered once and then +/// recognised: the walk reports the loop and does not descend again, so each +/// file is listed once instead of once per nesting level until the path runs +/// out of room. +#[cfg(unix)] +#[test] +fn a_link_back_into_the_tree_is_not_walked_twice_under_f() { + let scratch = Scratch::new("loop"); + let leaf = scratch.file("tree/inner/leaf.txt"); + std::os::unix::fs::symlink( + scratch.path().join("tree"), + scratch.path().join("tree/inner/up"), + ) + .unwrap(); + + let selection = select_inputs(vec![scratch.path().join("tree")], &[], true, true, 0).unwrap(); + assert_eq!(selection.files, vec![leaf]); +} + /// The same rule inside a walked tree: a link found by `-r` is skipped without /// `-f`, so a tree with a link back into itself does not loop, and a link to a /// directory is not descended into. diff --git a/zstd/src/bin/structured-zstd/interrupt.rs b/zstd/src/bin/structured-zstd/interrupt.rs index c4a53c2f4..ecd0c506f 100644 --- a/zstd/src/bin/structured-zstd/interrupt.rs +++ b/zstd/src/bin/structured-zstd/interrupt.rs @@ -4,39 +4,135 @@ //! temporary beside the source. The reference command installs a `SIGINT` //! handler that unlinks the artefact and exits with status 2; this does the //! same, through the C library the standard library already links, so the -//! tool takes on no dependency for it. Platforms without POSIX signals get -//! the no-op version and keep the temporary on interruption. +//! tool takes on no dependency for it. On Windows the C runtime's `signal` +//! delivers the console's `Ctrl-C` as `SIGINT` the same way. A `SIGINT` the +//! process inherited as ignored stays ignored, as a background job of a +//! non-interactive shell or a `nohup` run expects; the reference command +//! replaces it. Platforms with neither get the no-op version and keep the +//! temporary on interruption. -#[cfg(unix)] +#[cfg(any(unix, windows))] mod imp { - use core::ffi::{c_char, c_int, c_void}; - use std::os::unix::ffi::OsStrExt; + use core::ffi::c_int; use std::path::Path; use std::ptr; - use std::sync::atomic::{AtomicPtr, Ordering}; + use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; - type Handler = extern "C" fn(c_int); + use sys::PathUnit; - unsafe extern "C" { - fn signal(signum: c_int, handler: Option) -> Option; - fn unlink(path: *const c_char) -> c_int; - fn write(fd: c_int, buf: *const c_void, count: usize) -> isize; - fn _exit(status: c_int) -> !; - } + /// The C `sighandler_t`: `SIG_DFL`, `SIG_IGN`, `SIG_ERR` or a handler's + /// address. An integer rather than a function pointer, since the first + /// three are markers, not addresses of anything callable. + type SigHandler = usize; + #[cfg(test)] + const SIG_DFL: SigHandler = 0; + const SIG_IGN: SigHandler = 1; + const SIG_ERR: SigHandler = usize::MAX; - /// `SIGINT` has this number on every POSIX system. + /// `SIGINT` has this number on every POSIX system and in the Windows C + /// runtime. const SIGINT: c_int = 2; - /// Longest path the guard covers, NUL included. A longer temporary is not - /// guarded rather than truncated to a name that is not the file's. + #[cfg(unix)] + mod sys { + use core::ffi::{c_char, c_int, c_void}; + use std::os::unix::ffi::OsStrExt; + use std::path::Path; + + /// A unit of the path as `unlink` takes it: a byte. + pub type PathUnit = u8; + + unsafe extern "C" { + pub fn signal(signum: c_int, handler: usize) -> usize; + fn unlink(path: *const c_char) -> c_int; + fn write(fd: c_int, buf: *const c_void, count: usize) -> isize; + pub fn _exit(status: c_int) -> !; + } + + /// The path's units in the order `unlink` reads them. + pub fn units(path: &Path) -> impl Iterator + '_ { + path.as_os_str().as_bytes().iter().copied() + } + + /// # Safety + /// `path` points at a NUL-terminated string. + pub unsafe fn remove(path: *const PathUnit) { + unsafe { + unlink(path.cast()); + } + } + + /// A newline on stderr, as the reference command prints one before + /// exiting. + pub fn newline() { + // SAFETY: a plain libc call on a valid buffer. + unsafe { + write(2, b"\n".as_ptr().cast(), 1); + } + } + } + + #[cfg(windows)] + mod sys { + use core::ffi::{c_int, c_uint, c_void}; + use std::os::windows::ffi::OsStrExt; + use std::path::Path; + + /// A unit of the path as `_wunlink` takes it: a UTF-16 code unit. + pub type PathUnit = u16; + + #[cfg_attr( + all(target_env = "msvc", not(target_feature = "crt-static")), + link(name = "msvcrt") + )] + #[cfg_attr( + all(target_env = "msvc", target_feature = "crt-static"), + link(name = "libcmt") + )] + unsafe extern "C" { + pub fn signal(signum: c_int, handler: usize) -> usize; + fn _wunlink(path: *const u16) -> c_int; + fn _write(fd: c_int, buf: *const c_void, count: c_uint) -> c_int; + pub fn _exit(status: c_int) -> !; + } + + /// The path's units in the order `_wunlink` reads them. + pub fn units(path: &Path) -> impl Iterator + '_ { + path.as_os_str().encode_wide() + } + + /// # Safety + /// `path` points at a NUL-terminated wide string. + pub unsafe fn remove(path: *const PathUnit) { + unsafe { + _wunlink(path); + } + } + + /// A newline on stderr, as the reference command prints one before + /// exiting. + pub fn newline() { + // SAFETY: a plain C runtime call on a valid buffer. + unsafe { + _write(2, b"\n".as_ptr().cast(), 1); + } + } + } + + /// Longest path the guard covers, terminator included. A longer temporary + /// is not guarded rather than truncated to a name that is not the file's. pub const PATH_CAPACITY: usize = 4096; - /// The guarded path as a C string, or all zeros. Written only while + /// The guarded path, NUL-terminated, or all zeros. Written only while /// `ARTEFACT` is null, so the handler never reads a half-written name. - static mut PATH: [u8; PATH_CAPACITY] = [0; PATH_CAPACITY]; + static mut PATH: [PathUnit; PATH_CAPACITY] = [0; PATH_CAPACITY]; /// Points into `PATH` while a file is guarded, null otherwise. - static ARTEFACT: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + static ARTEFACT: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + + /// What `SIGINT` was before the handler first went in, `SIG_ERR` until + /// then: the disposition `clear` puts back. + static INHERITED: AtomicUsize = AtomicUsize::new(SIG_ERR); /// Async-signal-safe by construction: `unlink`, `write` and `_exit` /// only, no allocation, no locks, no formatting. @@ -47,42 +143,88 @@ mod imp { // and is not rewritten until the pointer has been cleared. if !path.is_null() { unsafe { - unlink(path); + sys::remove(path); } } - // SAFETY: plain libc calls on a valid buffer and a constant status. + sys::newline(); + // SAFETY: a plain exit with a constant status. unsafe { - write(2, b"\n".as_ptr().cast(), 1); - _exit(2); + sys::_exit(2); + } + } + + /// Put `on_interrupt` in place, and say whether it is. A `SIGINT` the + /// process inherited as ignored is left ignored: `signal` is the only + /// portable way to learn the current disposition and replaces it while + /// doing so, so an ignore found on the first call is put straight back + /// (the classic idiom) and remembered, and later calls do not touch it. + fn install() -> bool { + let handler = on_interrupt as *const () as SigHandler; + match INHERITED.load(Ordering::SeqCst) { + SIG_IGN => false, + SIG_ERR => { + // SAFETY: plain libc calls with a valid handler address. + let previous = unsafe { sys::signal(SIGINT, handler) }; + if previous == SIG_IGN { + unsafe { + sys::signal(SIGINT, SIG_IGN); + } + } + INHERITED.store(previous, Ordering::SeqCst); + previous != SIG_IGN && previous != SIG_ERR + } + _ => { + // SAFETY: as above. + let previous = unsafe { sys::signal(SIGINT, handler) }; + previous != SIG_ERR + } } } /// Remove `path` if the process is interrupted before [`clear`] is called. pub fn guard(path: &Path) { - let bytes = path.as_os_str().as_bytes(); - if bytes.is_empty() || bytes.len() >= PATH_CAPACITY || bytes.contains(&0) { + ARTEFACT.store(ptr::null_mut(), Ordering::SeqCst); + let buffer = (&raw mut PATH).cast::(); + let mut len = 0; + for unit in sys::units(path) { + // A terminator inside the name, or a name that would not leave + // room for one, is not a name the handler can be given. + if unit == 0 || len >= PATH_CAPACITY - 1 { + return; + } + // SAFETY: the handler reads `PATH` only through `ARTEFACT`, which + // is null for the length of this write; raw pointer access keeps + // no reference to the static alive, and `len` is in range. + unsafe { + *buffer.add(len) = unit; + } + len += 1; + } + if len == 0 { return; } - ARTEFACT.store(ptr::null_mut(), Ordering::SeqCst); - // SAFETY: the handler reads `PATH` only through `ARTEFACT`, which is - // null for the length of this write; raw pointer access keeps no - // reference to the static alive. + // SAFETY: `len < PATH_CAPACITY`, and as above. unsafe { - let buffer = (&raw mut PATH).cast::(); - ptr::copy_nonoverlapping(bytes.as_ptr(), buffer, bytes.len()); - *buffer.add(bytes.len()) = 0; - ARTEFACT.store(buffer.cast::(), Ordering::SeqCst); - signal(SIGINT, Some(on_interrupt)); + *buffer.add(len) = 0; + } + if !install() { + return; } + ARTEFACT.store(buffer, Ordering::SeqCst); } /// Stop guarding: an interruption from here on keeps the file and takes - /// the default action. + /// the action the process started with. pub fn clear() { ARTEFACT.store(ptr::null_mut(), Ordering::SeqCst); - // SAFETY: restoring the default disposition is always valid. - unsafe { - signal(SIGINT, None); + // An inherited ignore was never replaced, and an unknown disposition + // (no guard yet, or a failed install) has nothing to put back. + let inherited = INHERITED.load(Ordering::SeqCst); + if inherited != SIG_IGN && inherited != SIG_ERR { + // SAFETY: restoring a disposition `signal` itself returned. + unsafe { + sys::signal(SIGINT, inherited); + } } } @@ -91,9 +233,59 @@ mod imp { pub fn is_guarded() -> bool { !ARTEFACT.load(Ordering::SeqCst).is_null() } + + /// Forget what an earlier guard found, as a fresh process would not know + /// it (for tests). + #[cfg(test)] + pub fn forget_inherited() { + INHERITED.store(SIG_ERR, Ordering::SeqCst); + } + + /// Make the process ignore `SIGINT`, as a parent may have left it (for + /// tests). + #[cfg(test)] + pub fn ignore_interrupts() { + // SAFETY: a plain libc call with a marker value. + unsafe { + sys::signal(SIGINT, SIG_IGN); + } + } + + /// Give `SIGINT` its default action (for tests). + #[cfg(test)] + pub fn take_default_action() { + // SAFETY: a plain libc call with a marker value. + unsafe { + sys::signal(SIGINT, SIG_DFL); + } + } + + /// The current disposition of `SIGINT`, read by replacing it and putting + /// it back (for tests). + #[cfg(test)] + fn disposition() -> SigHandler { + // SAFETY: plain libc calls; the second restores what the first found. + unsafe { + let current = sys::signal(SIGINT, SIG_IGN); + sys::signal(SIGINT, current); + current + } + } + + /// Whether `SIGINT` is ignored right now (for tests). + #[cfg(test)] + pub fn interrupts_ignored() -> bool { + disposition() == SIG_IGN + } + + /// Whether `SIGINT` takes its default action right now (for tests). + #[cfg(test)] + pub fn default_action() -> bool { + disposition() == SIG_DFL + } } -#[cfg(not(unix))] +#[cfg(not(any(unix, windows)))] mod imp { use std::path::Path; diff --git a/zstd/src/bin/structured-zstd/interrupt/tests.rs b/zstd/src/bin/structured-zstd/interrupt/tests.rs index 6ce358e5e..5872dcc2c 100644 --- a/zstd/src/bin/structured-zstd/interrupt/tests.rs +++ b/zstd/src/bin/structured-zstd/interrupt/tests.rs @@ -6,9 +6,11 @@ use super::{clear, guard}; /// The guard is a window: it opens when a temporary is being written and /// closes when the file is in place. A guard left open past `clear` would /// delete a finished output on the next interruption. -#[cfg(unix)] +#[cfg(any(unix, windows))] #[test] fn a_guard_is_set_by_guard_and_removed_by_clear() { + super::imp::forget_inherited(); + super::imp::take_default_action(); clear(); assert!(!is_guarded()); guard(Path::new("/tmp/szstd-guard-test")); @@ -19,9 +21,11 @@ fn a_guard_is_set_by_guard_and_removed_by_clear() { /// A path the buffer cannot hold is not guarded rather than guarded under a /// truncated name, which would be some other file's. -#[cfg(unix)] +#[cfg(any(unix, windows))] #[test] fn a_path_that_does_not_fit_is_left_unguarded() { + super::imp::forget_inherited(); + super::imp::take_default_action(); clear(); let long = "x".repeat(super::imp::PATH_CAPACITY); guard(Path::new(&long)); @@ -34,7 +38,52 @@ fn a_path_that_does_not_fit_is_left_unguarded() { clear(); } -#[cfg(not(unix))] +/// A process started with `SIGINT` ignored (a background job of a +/// non-interactive shell, a `nohup` run) keeps ignoring it: installing the +/// handler over the inherited disposition would make such a run die on a +/// `Ctrl-C` meant for the foreground, and would hand an ignore marker to a +/// function-pointer binding on the way. +#[cfg(any(unix, windows))] +#[test] +fn an_inherited_ignore_of_sigint_is_kept() { + super::imp::forget_inherited(); + super::imp::ignore_interrupts(); + guard(Path::new("/tmp/szstd-guard-test")); + assert!( + !is_guarded(), + "with interruptions ignored there is nothing to guard against" + ); + assert!( + super::imp::interrupts_ignored(), + "SIGINT must stay ignored after guard" + ); + clear(); + assert!( + super::imp::interrupts_ignored(), + "and after clear, which restores what guard found" + ); +} + +/// `clear` puts back the disposition the first `guard` replaced, the default +/// action for a process started normally, rather than a fixed value. +#[cfg(any(unix, windows))] +#[test] +fn clear_restores_the_action_guard_replaced() { + super::imp::forget_inherited(); + super::imp::take_default_action(); + guard(Path::new("/tmp/szstd-guard-test")); + assert!( + !super::imp::default_action(), + "the handler is in place while guarded" + ); + clear(); + assert!( + super::imp::default_action(), + "the default action is back once cleared" + ); +} + +#[cfg(not(any(unix, windows)))] #[test] fn the_no_op_guard_never_reports_a_guard() { guard(Path::new("anything"));