Add a pure-Rust Zstandard backend behind a zstd-rust feature

.github/workflows/ci.yml

@@ -29,6 +29,8 @@ jobs:
           - ""
           - "--no-default-features --features deflate"
           - "--no-default-features --features zstd"
+          - "--no-default-features --features zstd-rust"
+          - "--features zstd-rust"
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -52,7 +54,7 @@ jobs:
           default: true
       - name: assert deps don't exist with --no-default-features
         run: |
-          for DEP in libflate zstd; do
+          for DEP in libflate zstd structured-zstd; do
             if cargo tree --no-default-features --invert $DEP; then
               echo "Dependency $DEP should not exist with --no-default-features"
               exit 1
@@ -70,3 +72,11 @@ jobs:
             echo "Feature `zstd` should not create dep on `libflate`"
             exit 1
           fi
+      - name: assert `zstd-rust` feature doesn't create dep on the C zstd bindings
+        run: |
+          for DEP in zstd zstd-sys libflate; do
+            if cargo tree --no-default-features --features zstd-rust --invert $DEP; then
+              echo "Feature `zstd-rust` should not create dep on $DEP"
+              exit 1
+            fi
+          done

Cargo.toml

@@ -12,6 +12,7 @@ license = "Apache-2.0"
 [workspace.dependencies]
 libflate = "2.3.0"
 zstd = "0.14.0"
+structured-zstd = "0.0.56"
 
 [package]
 name = "include-flate"
@@ -34,4 +35,7 @@ include-flate-compress = { version = "0.3.4", path = "compress", default-feature
 default = ["deflate", "zstd"]
 deflate = ["include-flate-codegen/deflate", "include-flate-compress/deflate"]
 zstd = ["include-flate-codegen/zstd", "include-flate-compress/zstd"]
+# Zstandard through the pure-Rust `structured-zstd`, with no C toolchain. The
+# `with zstd` syntax and the compressed format are the same as with `zstd`.
+zstd-rust = ["include-flate-codegen/zstd-rust", "include-flate-compress/zstd-rust"]
 no-compression-warnings = ["include-flate-codegen/no-compression-warnings"]

codegen/Cargo.toml

@@ -22,4 +22,5 @@ proc-macro-error3 = "3.0.2"
 default = ["deflate", "zstd"]
 deflate = ["include-flate-compress/deflate"]
 zstd = ["include-flate-compress/zstd"]
+zstd-rust = ["include-flate-compress/zstd-rust"]
 no-compression-warnings = []

codegen/src/lib.rs

@@ -98,13 +98,16 @@ impl syn::parse::Parse for FlateArgs {
                     "Please enable the `deflate` feature",
                 ));
             } else if lookahead.peek(kw::zstd) {
-                #[cfg(feature = "zstd")]
+                #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
                 {
                     input.parse::<kw::zstd>()?;
                     Some(CompressionMethodTy(CompressionMethod::Zstd))
                 }
-                #[cfg(not(feature = "zstd"))]
-                return Err(Error::new(input.span(), "Please enable the `zstd` feature"));
+                #[cfg(not(any(feature = "zstd", feature = "zstd-rust")))]
+                return Err(Error::new(
+                    input.span(),
+                    "Please enable the `zstd` or `zstd-rust` feature",
+                ));
             } else {
                 return Err(lookahead.error());
             }

compress/Cargo.toml

@@ -11,8 +11,12 @@ description = "Compression algorithm provider"
 [dependencies]
 libflate = { workspace = true, optional = true }
 zstd = { workspace = true, optional = true }
+structured-zstd = { workspace = true, optional = true }
 
 [features]
 default = ["deflate", "zstd"]
 deflate = ["dep:libflate"]
 zstd = ["dep:zstd"]
+# Zstandard through the pure-Rust `structured-zstd` instead of the C library.
+# Same frame format; takes precedence when `zstd` is enabled as well.
+zstd-rust = ["dep:structured-zstd"]

compress/src/lib.rs

@@ -13,8 +13,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(not(any(feature = "zstd", feature = "deflate")))]
-compile_error!("You must enable either the `deflate` or `zstd` feature.");
+#[cfg(not(any(feature = "zstd", feature = "zstd-rust", feature = "deflate")))]
+compile_error!("You must enable either the `deflate`, `zstd` or `zstd-rust` feature.");
 
 use std::{
     fmt,
@@ -25,16 +25,55 @@ use std::{
 use libflate::deflate::Decoder as DeflateDecoder;
 #[cfg(feature = "deflate")]
 use libflate::deflate::Encoder as DeflateEncoder;
-#[cfg(feature = "zstd")]
-use zstd::Decoder as ZstdDecoder;
-#[cfg(feature = "zstd")]
-use zstd::Encoder as ZstdEncoder;
+
+// Zstandard has two interchangeable backends producing and reading the same
+// frame format: the `zstd` crate (bindings to the C library) and the pure-Rust
+// `structured-zstd`. When both features are enabled, `zstd-rust` is used: a
+// dependency that asks for it opts out of the C toolchain.
+//
+// The pure-Rust coder keeps its state inline (the C one holds a pointer to
+// it), so it is boxed to keep the enums below the size of their other
+// variants.
+#[cfg(feature = "zstd-rust")]
+type ZstdEncoder<W> = Box<structured_zstd::encoding::StreamingEncoder<W>>;
+#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))]
+type ZstdEncoder<W> = zstd::Encoder<'static, W>;
+#[cfg(feature = "zstd-rust")]
+type ZstdDecoder<R> =
+    Box<structured_zstd::decoding::StreamingDecoder<R, structured_zstd::decoding::FrameDecoder>>;
+#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))]
+type ZstdDecoder<R> = zstd::Decoder<'static, std::io::BufReader<R>>;
+
+#[cfg(feature = "zstd-rust")]
+fn zstd_encoder<W: Write>(write: W) -> io::Result<ZstdEncoder<W>> {
+    // `Default` is this backend's counterpart of zstd level 3, which the C
+    // backend selects through level 0.
+    Ok(Box::new(structured_zstd::encoding::StreamingEncoder::new(
+        write,
+        structured_zstd::encoding::CompressionLevel::Default,
+    )))
+}
+#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))]
+fn zstd_encoder<W: Write>(write: W) -> io::Result<ZstdEncoder<W>> {
+    ZstdEncoder::new(write, 0)
+}
+
+#[cfg(feature = "zstd-rust")]
+fn zstd_decoder<R: Read>(read: R) -> io::Result<ZstdDecoder<R>> {
+    structured_zstd::decoding::StreamingDecoder::new(read)
+        .map(Box::new)
+        .map_err(io::Error::other)
+}
+#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))]
+fn zstd_decoder<R: Read>(read: R) -> io::Result<ZstdDecoder<R>> {
+    ZstdDecoder::new(read)
+}
 
 #[derive(Debug)]
 pub enum CompressionError {
     #[cfg(feature = "deflate")]
     DeflateError(io::Error),
-    #[cfg(feature = "zstd")]
+    #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
     ZstdError(io::Error),
     IoError(io::Error),
 }
@@ -50,7 +89,7 @@ impl fmt::Display for CompressionError {
         match self {
             #[cfg(feature = "deflate")]
             CompressionError::DeflateError(err) => write!(f, "Deflate error: {}", err),
-            #[cfg(feature = "zstd")]
+            #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
             CompressionError::ZstdError(err) => write!(f, "Zstd error: {}", err),
             CompressionError::IoError(err) => write!(f, "I/O error: {}", err),
         }
@@ -61,7 +100,7 @@ impl fmt::Display for CompressionError {
 pub enum CompressionMethod {
     #[cfg(feature = "deflate")]
     Deflate,
-    #[cfg(feature = "zstd")]
+    #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
     Zstd,
 }
 
@@ -79,14 +118,14 @@ impl CompressionMethod {
     clippy::derivable_impls,
     reason = "cfg_attr on defaults could be confusing"
 )]
-#[cfg(any(feature = "deflate", feature = "zstd"))]
+#[cfg(any(feature = "deflate", feature = "zstd", feature = "zstd-rust"))]
 impl Default for CompressionMethod {
     fn default() -> Self {
         #[cfg(feature = "deflate")]
         {
             Self::Deflate
         }
-        #[cfg(all(not(feature = "deflate"), feature = "zstd"))]
+        #[cfg(all(not(feature = "deflate"), any(feature = "zstd", feature = "zstd-rust")))]
         {
             Self::Zstd
         }
@@ -98,7 +137,7 @@ impl fmt::Display for CompressionMethod {
         f.write_str(match self {
             #[cfg(feature = "deflate")]
             Self::Deflate => "deflate",
-            #[cfg(feature = "zstd")]
+            #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
             Self::Zstd => "zstd",
         })
     }
@@ -107,17 +146,17 @@ impl fmt::Display for CompressionMethod {
 pub enum FlateEncoder<W: Write> {
     #[cfg(feature = "deflate")]
     Deflate(DeflateEncoder<W>),
-    #[cfg(feature = "zstd")]
-    Zstd(ZstdEncoder<'static, W>),
+    #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
+    Zstd(ZstdEncoder<W>),
 }
 
 impl<W: Write> FlateEncoder<W> {
     pub fn new(method: CompressionMethod, write: W) -> Result<FlateEncoder<W>, CompressionError> {
         match method {
             #[cfg(feature = "deflate")]
             CompressionMethod::Deflate => Ok(FlateEncoder::Deflate(DeflateEncoder::new(write))),
-            #[cfg(feature = "zstd")]
-            CompressionMethod::Zstd => ZstdEncoder::new(write, 0)
+            #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
+            CompressionMethod::Zstd => zstd_encoder(write)
                 .map(FlateEncoder::Zstd)
                 .map_err(CompressionError::ZstdError),
         }
@@ -129,7 +168,7 @@ impl<W: Write> Write for FlateEncoder<W> {
         match self {
             #[cfg(feature = "deflate")]
             FlateEncoder::Deflate(encoder) => encoder.write(buf),
-            #[cfg(feature = "zstd")]
+            #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
             FlateEncoder::Zstd(encoder) => encoder.write(buf),
         }
     }
@@ -138,7 +177,7 @@ impl<W: Write> Write for FlateEncoder<W> {
         match self {
             #[cfg(feature = "deflate")]
             FlateEncoder::Deflate(encoder) => encoder.flush(),
-            #[cfg(feature = "zstd")]
+            #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
             FlateEncoder::Zstd(encoder) => encoder.flush(),
         }
     }
@@ -152,27 +191,29 @@ impl<W: Write> FlateEncoder<W> {
                 .finish()
                 .into_result()
                 .map_err(CompressionError::DeflateError),
-            #[cfg(feature = "zstd")]
+            #[cfg(feature = "zstd-rust")]
+            FlateEncoder::Zstd(encoder) => (*encoder).finish().map_err(CompressionError::ZstdError),
+            #[cfg(all(feature = "zstd", not(feature = "zstd-rust")))]
             FlateEncoder::Zstd(encoder) => encoder.finish().map_err(CompressionError::ZstdError),
         }
     }
 }
 
-pub enum FlateDecoder<R> {
+pub enum FlateDecoder<R: Read> {
     #[cfg(feature = "deflate")]
     Deflate(DeflateDecoder<R>),
-    #[cfg(feature = "zstd")]
-    Zstd(ZstdDecoder<'static, std::io::BufReader<R>>),
+    #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
+    Zstd(ZstdDecoder<R>),
 }
 
 impl<R: Read> FlateDecoder<R> {
     pub fn new(method: CompressionMethod, read: R) -> Result<FlateDecoder<R>, CompressionError> {
         match method {
             #[cfg(feature = "deflate")]
             CompressionMethod::Deflate => Ok(FlateDecoder::Deflate(DeflateDecoder::new(read))),
-            #[cfg(feature = "zstd")]
+            #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
             CompressionMethod::Zstd => {
-                let decoder = ZstdDecoder::new(read)?;
+                let decoder = zstd_decoder(read)?;
                 Ok(FlateDecoder::Zstd(decoder))
             }
         }
@@ -184,7 +225,7 @@ impl<R: Read> Read for FlateDecoder<R> {
         match self {
             #[cfg(feature = "deflate")]
             FlateDecoder::Deflate(decoder) => decoder.read(buf),
-            #[cfg(feature = "zstd")]
+            #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
             FlateDecoder::Zstd(decoder) => decoder.read(buf),
         }
     }

compress/tests/zstd_interop.rs

@@ -0,0 +1,53 @@
+// include-flate
+// Copyright (C) SOFe
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! The two Zstandard backends read each other's output: a crate built with
+//! `zstd-rust` decodes what a `zstd` build embedded, and the reverse.
+
+#![cfg(all(feature = "zstd", feature = "zstd-rust"))]
+
+use std::io::Read;
+
+use include_flate_compress::{CompressionMethod, apply_compression, apply_decompression};
+
+fn sample() -> Vec<u8> {
+    (0..200_000u32)
+        .flat_map(|i| format!("{} {} ", i % 977, i % 13).into_bytes())
+        .collect()
+}
+
+#[test]
+fn a_c_zstd_frame_decodes_with_the_rust_backend() {
+    let data = sample();
+    let frame = zstd::encode_all(&data[..], 0).unwrap();
+
+    let mut decoded = Vec::new();
+    apply_decompression(&frame[..], &mut decoded, CompressionMethod::Zstd).unwrap();
+    assert_eq!(decoded, data);
+}
+
+#[test]
+fn a_rust_backend_frame_decodes_with_c_zstd() {
+    let data = sample();
+    let mut frame = Vec::new();
+    apply_compression(&mut &data[..], &mut frame, CompressionMethod::Zstd).unwrap();
+
+    let mut decoded = Vec::new();
+    zstd::Decoder::new(&frame[..])
+        .unwrap()
+        .read_to_end(&mut decoded)
+        .unwrap();
+    assert_eq!(decoded, data);
+}

src/lib.rs

@@ -61,7 +61,11 @@ pub use include_flate_compress::CompressionMethod;
 /// - If `$type` is `str` but the file is not fully valid UTF-8
 ///
 /// # Algorithm
-/// Compression and decompression use the DEFLATE algorithm from [`libflate`][5].
+/// Compression and decompression use the DEFLATE algorithm from [`libflate`][5] by default,
+/// or Zstandard with `with zstd`. Zstandard is provided by the C library through the `zstd`
+/// feature, or by the pure-Rust [`structured-zstd`][7] through the `zstd-rust` feature, which
+/// needs no C toolchain and takes precedence when both are enabled. The two produce and read
+/// the same format.
 ///
 /// # Examples
 /// Below are some basic examples. For actual compiled examples, see the [`tests`][6] directory.
@@ -88,6 +92,7 @@ pub use include_flate_compress::CompressionMethod;
 ///   [4]: https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates
 ///   [5]: https://docs.rs/libflate/0.1.26/libflate/
 ///   [6]: https://github.com/SOF3/include-flate/tree/master/tests
+///   [7]: https://docs.rs/structured-zstd
 #[macro_export]
 macro_rules! flate {
     ($(#[$meta:meta])*

test_util.rs

@@ -60,7 +60,7 @@ pub fn verify_compression<P: AsRef<Path>>(name: P, data: &[u8], method: Compress
 pub fn verify<P: AsRef<Path>>(name: P, data: &[u8]) {
     #[cfg(feature = "deflate")]
     verify_compression(&name, data, CompressionMethod::Deflate);
-    #[cfg(feature = "zstd")]
+    #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
     verify_compression(&name, data, CompressionMethod::Zstd);
     assert_eq!(read_file(&name), data);
 }

tests/009f.rs

@@ -20,22 +20,22 @@ use include_flate::flate;
 flate!(pub static DATA1: [u8] from "assets/009f.dat");
 #[cfg(feature = "deflate")]
 flate!(pub static DATA2: [u8] from "assets/009f.dat" with deflate);
-#[cfg(feature = "zstd")]
+#[cfg(any(feature = "zstd", feature = "zstd-rust"))]
 flate!(pub static DATA3: [u8] from "assets/009f.dat" with zstd);
 #[cfg(feature = "deflate")]
 flate!(pub static DATA4: IFlate from "assets/009f.dat" with deflate);
-#[cfg(feature = "zstd")]
+#[cfg(any(feature = "zstd", feature = "zstd-rust"))]
 flate!(pub static DATA5: IFlate from "assets/009f.dat" with zstd);
 
 #[test]
 fn test() {
     verify("009f.dat", &DATA1);
     #[cfg(feature = "deflate")]
     verify("009f.dat", &DATA2);
-    #[cfg(feature = "zstd")]
+    #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
     verify("009f.dat", &DATA3);
     #[cfg(feature = "deflate")]
     verify_iflate("009f.dat", CompressionMethod::Deflate, &DATA4);
-    #[cfg(feature = "zstd")]
+    #[cfg(any(feature = "zstd", feature = "zstd-rust"))]
     verify_iflate("009f.dat", CompressionMethod::Zstd, &DATA5);
 }