feat(cli): upstream v1.5.7 file selection, display levels and encoder switches

README.md

@@ -53,12 +53,31 @@ in nothing extra.
 
 The binary speaks the upstream `zstd`
 command line: levels (`-1`..`-19`, `--ultra` for `-20`..`-22`, `--fast[=N]`),
-`-d`, `-c`, `-o`, `-t`, `-l`, `-D`, `--train`, `-b`, and the usual
-`-f`/`-k`/`--rm` file handling.
-
-Flags that only steer how the work is done (`-T`, `-B`, `--adapt`,
-`--[no-]progress`, …) are accepted and ignored — their values are still
-validated, so a typo is an error rather than silence.
+`-d`, `-c`, `-o`, `-t`, `-l` (`-lv` for the per-archive block), `-D`,
+`--train`, `-b`, the usual `-f`/`-k`/`--rm` file handling, the `-q`/`-v`
+display levels, and file selection with `-r`, `--filelist`,
+`--output-dir-flat` and `--output-dir-mirror`. `ZSTD_CLEVEL` sets the default
+level and `ZSTD_NBTHREADS` is validated, both read as upstream reads them.
+Several inputs into one `-o` or `-c` are concatenated after upstream's
+warning, an existing output is asked about unless `-f` is given (and refused
+under `-q`, where nothing can be asked), one failing input does not stop the
+others, and the exit status is 1 when any input failed and 2 on an interrupt,
+which also removes the partial output.
+
+The wire-format switches take effect: `--[no-]check` (`--no-check` also skips
+checksum verification when decoding), `--[no-]content-size`, `--no-dictID` and
+`--[no-]compress-literals`. `--zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#`
+overrides the level's parameters knob by knob (the `ldm*` knobs apply with
+`--long`). `--patch-from REF` compresses against a reference as raw content
+with the window sized to the input, and applies the patch back on `-d`.
+`--[no-]pass-through` copies non-zstd input through unchanged when
+decompressing, on by default for `zstdcat` and `zstd -dcf` as upstream has it,
+and `--exclude-compressed` skips inputs whose extension names an
+already-compressed format.
+
+Flags that only steer how the work is done (`-T`, `-B`, `--adapt`, ...) are
+accepted and ignored; their values are still validated, so a typo is an error
+rather than silence.
 `--target-compressed-block-size` does take effect: it bounds what goes into a
 block, so blocks flush sooner. `--long` means `--long=27`, as upstream
 documents, and is capped there: a larger window would produce frames this
@@ -69,18 +88,18 @@ source can fill, so a small file compressed with `--long` does not ask its
 decoders to reserve 128 MiB.
 
 Flags that would change the result are refused instead: `--format=` for
-anything but zstd, `--patch-from`, `--rsyncable`, `--no-check`,
-`--[no-]compress-literals`, and the not-yet-implemented `--pass-through` /
-`--exclude-compressed`. `-M` is treated as the safety promise it is: on the
+anything but zstd and `--rsyncable`, which needs the worker threads this build
+does not have. `-M` is treated as the safety promise it is: on the
 runs that decode, a limit covering the 128 MiB window, the decoder's buffers
 and the `-D` dictionary is kept and a tighter one is refused rather than
 ignored. Compressing, listing and training allocate no decoder, so the flag is
 accepted there and describes nothing, as upstream has it.
 
-`--train` and `--train-fastcover` both train with FastCOVER, the algorithm
-upstream also defaults to. `--train-cover` and `--train-legacy` name algorithms
-this build does not have, so they are refused rather than quietly served by
-FastCOVER. `-D` takes either a dictionary produced by `--train` or any file at
+`--train` and `--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#]` train
+with FastCOVER, the algorithm upstream also defaults to, and `--train-cover`
+with the COVER trainer (whose reference-side tuning does not apply here, so it
+is refused rather than misread). `--train-legacy` names an algorithm this build
+does not have and is refused. `-D` takes either a dictionary produced by `--train` or any file at
 all, which is then used as raw content the way upstream does — such a
 dictionary has no ID, so the same bytes must be supplied when decoding.
 
@@ -90,7 +109,8 @@ familiar names works:
 
 ```bash
 ln -s "$(command -v structured-zstd)" ~/.local/bin/unzstd    # defaults to -d
-ln -s "$(command -v structured-zstd)" ~/.local/bin/zstdcat   # defaults to -d -c
+ln -s "$(command -v structured-zstd)" ~/.local/bin/zstdcat   # -d -c -f, pass-through, quiet
+ln -s "$(command -v structured-zstd)" ~/.local/bin/zstdmt    # compresses like zstd
 ```
 
 Distributions should register it with their alternatives mechanism rather than

zstd/src/bin/structured-zstd/display.rs

@@ -0,0 +1,186 @@
+//! What the tool says on stderr, and how loudly.
+//!
+//! Mirrors the reference command's display levels: `0` says nothing, `1`
+//! reports errors, `2` (the default) adds the result summary, warnings and
+//! interactive prompts, `3` adds progress, `4` adds information. `-v` raises
+//! the level and `-q` lowers it, so `-qq` silences errors as well.
+
+use std::fmt;
+use std::io::{BufRead, Write};
+
+/// The level a run starts at, before `-q` / `-v` move it.
+pub const DEFAULT_LEVEL: i32 = 2;
+
+/// Whether the progress counter is drawn (`--progress` / `--no-progress`).
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum Progress {
+    /// Drawn when stderr is a terminal and the level allows it.
+    Auto,
+    /// Drawn regardless of where stderr goes.
+    Always,
+    /// Never drawn; every other message is unaffected.
+    Never,
+}
+
+impl Progress {
+    /// Whether a run at `verbosity` draws the counter, given where stderr goes.
+    pub fn shown(self, verbosity: i32, stderr_is_terminal: bool) -> bool {
+        match self {
+            Self::Always => true,
+            Self::Never => false,
+            Self::Auto => verbosity >= DEFAULT_LEVEL && stderr_is_terminal,
+        }
+    }
+}
+
+/// A byte count scaled to the unit the reference command prints it in.
+///
+/// Scaled in powers of two, with the precision chosen from the magnitude of
+/// the scaled value: three decimals below one, two below ten, one below a
+/// hundred, none above that or when the value is a whole number of units.
+/// `verbose` keeps the raw byte count instead, as `-vv` does.
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct HumanSize {
+    value: f64,
+    precision: usize,
+    suffix: &'static str,
+}
+
+impl HumanSize {
+    /// Scale `bytes` for display.
+    pub fn new(bytes: u64, verbose: bool) -> Self {
+        if verbose {
+            // Past the integral precision of a double the count itself is not
+            // representable, so it is scaled once and rounded well up.
+            return if bytes >= 1 << 53 {
+                Self {
+                    value: bytes as f64 / (1u64 << 20) as f64,
+                    precision: 2,
+                    suffix: " MiB",
+                }
+            } else {
+                Self {
+                    value: bytes as f64,
+                    precision: 0,
+                    suffix: " B",
+                }
+            };
+        }
+        const UNITS: [(u32, &str); 6] = [
+            (60, " EiB"),
+            (50, " PiB"),
+            (40, " TiB"),
+            (30, " GiB"),
+            (20, " MiB"),
+            (10, " KiB"),
+        ];
+        let (value, suffix) = UNITS
+            .iter()
+            .find(|(shift, _)| bytes >= 1u64 << shift)
+            .map_or((bytes as f64, " B"), |(shift, suffix)| {
+                (bytes as f64 / (1u64 << shift) as f64, *suffix)
+            });
+        let precision = if value >= 100.0 || value as u64 as f64 == value {
+            0
+        } else if value >= 10.0 {
+            1
+        } else if value > 1.0 {
+            2
+        } else {
+            3
+        };
+        Self {
+            value,
+            precision,
+            suffix,
+        }
+    }
+
+    /// The scaled value.
+    #[cfg(test)]
+    pub fn value(&self) -> f64 {
+        self.value
+    }
+
+    /// Decimals the scaled value is shown with.
+    #[cfg(test)]
+    pub fn precision(&self) -> usize {
+        self.precision
+    }
+
+    /// The unit, with its leading space (`" KiB"`).
+    #[cfg(test)]
+    pub fn suffix(&self) -> &'static str {
+        self.suffix
+    }
+
+    /// The value in a column `width` wide and the unit in one four wide: the
+    /// reference command's `%6.*f%4s` layout for `-l` rows and multi-file
+    /// summaries, where `" B"` is padded out to the width of `" KiB"`.
+    pub fn columns(&self, width: usize) -> String {
+        format!(
+            "{:>width$.prec$}{:>4}",
+            self.value,
+            self.suffix,
+            width = width,
+            prec = self.precision
+        )
+    }
+}
+
+impl fmt::Display for HumanSize {
+    /// The value at its precision, then the unit; a field width applies to the
+    /// value alone, the way the reference command's `%6.*f%s` lays it out.
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match f.width() {
+            Some(width) => write!(
+                f,
+                "{:>width$.prec$}{}",
+                self.value,
+                self.suffix,
+                width = width,
+                prec = self.precision
+            ),
+            None => write!(
+                f,
+                "{:.prec$}{}",
+                self.value,
+                self.suffix,
+                prec = self.precision
+            ),
+        }
+    }
+}
+
+/// Ask the user a yes/no question on stderr and read the answer from `input`.
+///
+/// Returns `true` only for an answer starting with `y` or `Y`. The reference
+/// command refuses to ask at all when stdin is one of the inputs: the answer
+/// would be read out of the data. `abort_message` is printed on a refusal so
+/// the caller can say what did not happen.
+pub fn confirm(
+    prompt: &str,
+    abort_message: &str,
+    stdin_is_input: bool,
+    input: &mut impl BufRead,
+) -> bool {
+    let mut stderr = std::io::stderr().lock();
+    if stdin_is_input {
+        let _ = writeln!(stderr, "stdin is an input - not proceeding.");
+        return false;
+    }
+    let _ = write!(stderr, "{prompt}");
+    let _ = stderr.flush();
+    let mut answer = String::new();
+    // The first character decides, as the reference command's `getchar` does:
+    // a space before the `y` is not a yes.
+    let accepted =
+        input.read_line(&mut answer).is_ok() && matches!(answer.chars().next(), Some('y' | 'Y'));
+    if !accepted {
+        let _ = writeln!(stderr, "{abort_message}");
+    }
+    accepted
+}
+
+#[cfg(test)]
+mod tests;

zstd/src/bin/structured-zstd/display/tests.rs

@@ -0,0 +1,93 @@
+use super::{DEFAULT_LEVEL, HumanSize, Progress, confirm};
+
+/// The reference command scales sizes in powers of two and picks the number
+/// of decimals from the scaled value, so `-l` columns and result summaries
+/// line up with what its own output shows. A whole number of units drops the
+/// decimals entirely; anything else keeps three significant figures.
+#[test]
+fn sizes_scale_and_round_the_way_the_reference_prints_them() {
+    assert_eq!(HumanSize::new(0, false).to_string(), "0 B");
+    assert_eq!(HumanSize::new(100, false).to_string(), "100 B");
+    assert_eq!(HumanSize::new(1023, false).to_string(), "1023 B");
+    assert_eq!(HumanSize::new(1024, false).to_string(), "1 KiB");
+    assert_eq!(HumanSize::new(1025, false).to_string(), "1.00 KiB");
+    assert_eq!(HumanSize::new(1536, false).to_string(), "1.50 KiB");
+    assert_eq!(
+        HumanSize::new(10 * 1024 + 512, false).to_string(),
+        "10.5 KiB"
+    );
+    assert_eq!(HumanSize::new(12 * 1024, false).to_string(), "12 KiB");
+    assert_eq!(HumanSize::new(7 << 20, false).to_string(), "7 MiB");
+    assert_eq!(HumanSize::new(123 << 30, false).to_string(), "123 GiB");
+    assert_eq!(HumanSize::new(150 << 30, false).to_string(), "150 GiB");
+    assert_eq!(HumanSize::new(1 << 40, false).to_string(), "1 TiB");
+}
+
+/// Below one unit the value keeps three decimals: a file just under a
+/// kibibyte scaled to KiB would otherwise print as `1 KiB`, which it is not.
+#[test]
+fn a_fraction_of_a_unit_keeps_three_decimals() {
+    let size = HumanSize::new((1 << 20) + 1, false);
+    assert_eq!(size.precision(), 2);
+    let just_over_a_unit = HumanSize::new(1024 + 1, false);
+    assert_eq!(just_over_a_unit.to_string(), "1.00 KiB");
+    // Only values strictly above one unit reach the two-decimal branch; a
+    // value scaled to exactly one unit is whole and prints without decimals.
+    assert_eq!(HumanSize::new(1 << 30, false).precision(), 0);
+}
+
+/// `-vv` keeps the raw byte count: a summary being pasted into a report wants
+/// the exact figure, not a rounded one.
+#[test]
+fn verbose_sizes_are_raw_byte_counts() {
+    assert_eq!(HumanSize::new(1536, true).to_string(), "1536 B");
+    assert_eq!(HumanSize::new(0, true).to_string(), "0 B");
+    // Past the integral precision of a double the count itself cannot be
+    // shown exactly, so it is scaled once.
+    assert_eq!(HumanSize::new(1 << 53, true).suffix(), " MiB");
+}
+
+/// A field width lays out the number and leaves the unit attached: that is how
+/// the reference command's `%6.*f%4s` keeps its columns aligned.
+#[test]
+fn a_field_width_pads_the_number_not_the_unit() {
+    assert_eq!(format!("{:>6}", HumanSize::new(100, false)), "   100 B");
+    assert_eq!(format!("{:>6}", HumanSize::new(1536, false)), "  1.50 KiB");
+    assert_eq!(HumanSize::new(1536, false).value(), 1.5);
+}
+
+/// The progress counter is drawn for a person watching a terminal, and only
+/// then unless asked for outright: piped stderr must stay clean of carriage
+/// returns, and `-q` says not to decorate.
+#[test]
+fn progress_is_drawn_for_a_terminal_at_the_default_level_unless_forced() {
+    assert!(Progress::Auto.shown(DEFAULT_LEVEL, true));
+    assert!(!Progress::Auto.shown(DEFAULT_LEVEL, false));
+    assert!(!Progress::Auto.shown(DEFAULT_LEVEL - 1, true));
+    assert!(Progress::Always.shown(0, false));
+    assert!(!Progress::Never.shown(4, true));
+}
+
+/// A prompt is answered by the first character of the line, `y` or `Y`, and
+/// nothing else: an empty answer, a refusal, or the end of input all decline.
+#[test]
+fn only_a_yes_confirms() {
+    assert!(confirm("go? ", "no", false, &mut &b"y\n"[..]));
+    assert!(confirm("go? ", "no", false, &mut &b"Y\n"[..]));
+    assert!(confirm("go? ", "no", false, &mut &b"yes please\n"[..]));
+    assert!(!confirm("go? ", "no", false, &mut &b"n\n"[..]));
+    assert!(!confirm("go? ", "no", false, &mut &b"\n"[..]));
+    assert!(!confirm("go? ", "no", false, &mut &b""[..]));
+    // The first character decides, as the reference command reads it.
+    assert!(!confirm("go? ", "no", false, &mut &b"  y\n"[..]));
+}
+
+/// When stdin is one of the inputs, the answer would be read out of the data
+/// being compressed. The reference command refuses to ask; so does this one,
+/// and it must not consume a byte of the input while declining.
+#[test]
+fn no_prompt_is_read_from_an_input_stream() {
+    let mut input = &b"y\n"[..];
+    assert!(!confirm("go? ", "no", true, &mut input));
+    assert_eq!(input, b"y\n", "the input must be left untouched");
+}