From e259c9d9df8006ed59ce4c3aae6c8f1238232e26 Mon Sep 17 00:00:00 2001 From: Jack Taylor Date: Thu, 4 Sep 2025 14:39:40 -0400 Subject: [PATCH 1/2] Support xz-compressed tarballs --- Cargo.lock | 26 ++++++++++++++++++++++++++ Cargo.toml | 1 + lib/result.rs | 9 +++++++++ lib/sources/artifact/format.rs | 23 +++++++++++++++++++++++ lib/sources/artifact/mod.rs | 6 +++++- lib/sources/artifact/util.rs | 22 +++++++++++++++++++++- lib/sources/decompression.rs | 21 +++++++++++++++++++++ 7 files changed, 106 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc45f1a..13ea02c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -523,6 +523,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1541,6 +1556,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "matchers" version = "0.2.0" @@ -2243,6 +2268,7 @@ dependencies = [ "futures", "goblin", "indicatif", + "lzma-rs", "postcard", "pulldown-cmark", "pulldown-cmark-mdcat", diff --git a/Cargo.toml b/Cargo.toml index dbd1fa2..5c6384c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ dunce = "1.0" filepath = "0.2" flate2 = "1.1" goblin = "0.10" +lzma-rs = "0.3" postcard = { version = "1.1", features = ["alloc"] } semver = { version = "1.0", features = ["serde"] } tar = "0.4" diff --git a/lib/result.rs b/lib/result.rs index a92b5cf..e290a2e 100644 --- a/lib/result.rs +++ b/lib/result.rs @@ -1,6 +1,7 @@ use std::io::Error as IoError; use std::path::PathBuf; +use lzma_rs::error::Error as LzmaError; use postcard::Error as PostcardError; use serde_json::Error as JsonError; use thiserror::Error; @@ -32,6 +33,8 @@ pub enum RokitError { Postcard(Box), #[error("Zip file error: {0}")] Zip(Box), + #[error("LZMA error: {0}")] + Lzma(Box), #[error("GitHub error: {0}")] GitHub(Box), } @@ -82,6 +85,12 @@ impl From for RokitError { } } +impl From for RokitError { + fn from(err: LzmaError) -> Self { + RokitError::Lzma(err.into()) + } +} + impl From for RokitError { fn from(err: GithubError) -> Self { RokitError::GitHub(err.into()) diff --git a/lib/sources/artifact/format.rs b/lib/sources/artifact/format.rs index f300b40..b08ec44 100644 --- a/lib/sources/artifact/format.rs +++ b/lib/sources/artifact/format.rs @@ -8,6 +8,7 @@ use super::util::split_filename_and_extensions; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum ArtifactFormat { TarGz, + TarXz, Tar, Zip, Gz, @@ -20,6 +21,7 @@ impl ArtifactFormat { Self::Zip => "zip", Self::Tar => "tar", Self::TarGz => "tar.gz", + Self::TarXz => "tar.xz", Self::Gz => "gz", } } @@ -30,11 +32,17 @@ impl ArtifactFormat { [.., ext] if ext.eq_ignore_ascii_case("zip") => Some(Self::Zip), [.., ext] if ext.eq_ignore_ascii_case("tar") => Some(Self::Tar), [.., ext] if ext.eq_ignore_ascii_case("tgz") => Some(Self::TarGz), + [.., ext] if ext.eq_ignore_ascii_case("txz") => Some(Self::TarXz), [.., ext1, ext2] if ext1.eq_ignore_ascii_case("tar") && ext2.eq_ignore_ascii_case("gz") => { Some(Self::TarGz) } + [.., ext1, ext2] + if ext1.eq_ignore_ascii_case("tar") && ext2.eq_ignore_ascii_case("xz") => + { + Some(Self::TarXz) + } [.., ext] if ext.eq_ignore_ascii_case("gz") => Some(Self::Gz), _ => None, } @@ -56,6 +64,7 @@ impl FromStr for ArtifactFormat { "zip" => Ok(Self::Zip), "tar" => Ok(Self::Tar), "tar.gz" | "tgz" => Ok(Self::TarGz), + "tar.xz" | "txz" => Ok(Self::TarXz), _ => Err(format!("unknown artifact format '{l}'")), } } @@ -82,6 +91,8 @@ mod tests { assert_eq!(format_from_str("file.tar"), Some(ArtifactFormat::Tar)); assert_eq!(format_from_str("file.tar.gz"), Some(ArtifactFormat::TarGz)); assert_eq!(format_from_str("file.tgz"), Some(ArtifactFormat::TarGz)); + assert_eq!(format_from_str("file.tar.xz"), Some(ArtifactFormat::TarXz)); + assert_eq!(format_from_str("file.txz"), Some(ArtifactFormat::TarXz)); assert_eq!(format_from_str("file.gz"), Some(ArtifactFormat::Gz)); assert_eq!( format_from_str("file.with.many.extensions.tar.gz.zip"), @@ -95,6 +106,10 @@ mod tests { format_from_str("file.with.many.extensions.tar.gz"), Some(ArtifactFormat::TarGz) ); + assert_eq!( + format_from_str("file.with.many.extensions.tar.xz"), + Some(ArtifactFormat::TarXz) + ); } #[test] @@ -145,6 +160,12 @@ mod tests { assert_eq!(format_from_str("file.gz"), Some(ArtifactFormat::Gz)); assert_eq!(format_from_str("file.GZ"), Some(ArtifactFormat::Gz)); assert_eq!(format_from_str("file.Gz"), Some(ArtifactFormat::Gz)); + assert_eq!(format_from_str("file.tar.xz"), Some(ArtifactFormat::TarXz)); + assert_eq!(format_from_str("file.TAR.XZ"), Some(ArtifactFormat::TarXz)); + assert_eq!(format_from_str("file.Tar.Xz"), Some(ArtifactFormat::TarXz)); + assert_eq!(format_from_str("file.txz"), Some(ArtifactFormat::TarXz)); + assert_eq!(format_from_str("file.TXZ"), Some(ArtifactFormat::TarXz)); + assert_eq!(format_from_str("file.Txz"), Some(ArtifactFormat::TarXz)); } #[test] @@ -153,6 +174,7 @@ mod tests { "tool-v1.0.0-x86_64-linux.zip", "tool-v1.0.0-x86_64-linux.tar", "tool-v1.0.0-x86_64-linux.tar.gz", + "tool-v1.0.0-x86_64-linux.tar.xz", "tool-v1.0.0-x86_64-linux.gz", "tool-v1.0.0-x86_64-linux", "tool-v1.0.0-x86_64-linux.elf", @@ -171,6 +193,7 @@ mod tests { None, None, Some(ArtifactFormat::TarGz), + Some(ArtifactFormat::TarXz), Some(ArtifactFormat::Tar), Some(ArtifactFormat::Zip), Some(ArtifactFormat::Gz), diff --git a/lib/sources/artifact/mod.rs b/lib/sources/artifact/mod.rs index dc1f5b3..dc94ef2 100644 --- a/lib/sources/artifact/mod.rs +++ b/lib/sources/artifact/mod.rs @@ -8,7 +8,7 @@ use crate::{ }; use super::{ - decompression::decompress_gzip, + decompression::{decompress_gzip, decompress_xz}, extraction::{extract_tar_file, extract_zip_file}, github::models::GithubAsset, ExtractError, @@ -85,6 +85,10 @@ impl Artifact { let tar = decompress_gzip(&contents).await?; extract_tar_file(&tar, &file_name).await } + ArtifactFormat::TarXz => { + let tar = decompress_xz(&contents).await?; + extract_tar_file(&tar, &file_name).await + } ArtifactFormat::Gz => decompress_gzip(&contents).await.map(Some), }; diff --git a/lib/sources/artifact/util.rs b/lib/sources/artifact/util.rs index 591922a..3a51dc3 100644 --- a/lib/sources/artifact/util.rs +++ b/lib/sources/artifact/util.rs @@ -1,6 +1,6 @@ use std::path::Path; -const ALLOWED_EXTENSION_NAMES: [&str; 4] = ["zip", "tar", "gz", "tgz"]; +const ALLOWED_EXTENSION_NAMES: [&str; 6] = ["zip", "tar", "gz", "tgz", "xz", "txz"]; const ALLOWED_EXTENSION_COUNT: usize = 2; pub(super) fn split_filename_and_extensions(name: &str) -> (&str, Vec<&str>) { @@ -52,6 +52,14 @@ mod tests { split_filename_and_extensions("file.tar.gz"), ("file", vec!["tar", "gz"]) ); + assert_eq!( + split_filename_and_extensions("file.tar.xz"), + ("file", vec!["tar", "xz"]) + ); + assert_eq!( + split_filename_and_extensions("file.txz"), + ("file", vec!["txz"]) + ); assert_eq!( split_filename_and_extensions("file.with.many.extensions.tar.gz.zip"), ("file.with.many.extensions.tar", vec!["gz", "zip"]) @@ -64,6 +72,10 @@ mod tests { split_filename_and_extensions("file.with.many.extensions.tar.gz"), ("file.with.many.extensions", vec!["tar", "gz"]) ); + assert_eq!( + split_filename_and_extensions("file.with.many.extensions.tar.xz"), + ("file.with.many.extensions", vec!["tar", "xz"]) + ); } #[test] @@ -84,5 +96,13 @@ mod tests { split_filename_and_extensions("sentry-cli-linux-i686-2.32.1.tgz"), ("sentry-cli-linux-i686-2.32.1", vec!["tgz"]) ); + assert_eq!( + split_filename_and_extensions("tool-1.0.0-linux-x86_64.tar.xz"), + ("tool-1.0.0-linux-x86_64", vec!["tar", "xz"]) + ); + assert_eq!( + split_filename_and_extensions("tool-1.0.0-linux-x86_64.txz"), + ("tool-1.0.0-linux-x86_64", vec!["txz"]) + ); } } diff --git a/lib/sources/decompression.rs b/lib/sources/decompression.rs index aca9461..73aa350 100644 --- a/lib/sources/decompression.rs +++ b/lib/sources/decompression.rs @@ -1,6 +1,7 @@ use std::io::Read; use flate2::read::GzDecoder; +use lzma_rs::xz_decompress; use tokio::{task::spawn_blocking, time::Instant}; use crate::result::RokitResult; @@ -26,3 +27,23 @@ pub async fn decompress_gzip(gz_contents: impl AsRef<[u8]>) -> RokitResult) -> RokitResult> { + let xz_contents = xz_contents.as_ref().to_vec(); + let num_kilobytes = xz_contents.len() / 1024; + let start = Instant::now(); + + // using spawn_blocking for the same reason as gzip + spawn_blocking(move || { + let mut contents = Vec::new(); + xz_decompress(&mut xz_contents.as_slice(), &mut contents)?; + + tracing::trace!( + num_kilobytes, + elapsed = ?start.elapsed(), + "decompressed xz" + ); + Ok(contents) + }) + .await? +} From a0056fcbbab2f1de721570fb6a3eb77066de89b3 Mon Sep 17 00:00:00 2001 From: Jack Taylor Date: Thu, 4 Sep 2025 14:49:40 -0400 Subject: [PATCH 2/2] Use asphalt as a real tool example in test --- lib/sources/artifact/util.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/sources/artifact/util.rs b/lib/sources/artifact/util.rs index 3a51dc3..531dcc7 100644 --- a/lib/sources/artifact/util.rs +++ b/lib/sources/artifact/util.rs @@ -97,12 +97,8 @@ mod tests { ("sentry-cli-linux-i686-2.32.1", vec!["tgz"]) ); assert_eq!( - split_filename_and_extensions("tool-1.0.0-linux-x86_64.tar.xz"), - ("tool-1.0.0-linux-x86_64", vec!["tar", "xz"]) - ); - assert_eq!( - split_filename_and_extensions("tool-1.0.0-linux-x86_64.txz"), - ("tool-1.0.0-linux-x86_64", vec!["txz"]) + split_filename_and_extensions("asphalt-x86_64-apple-darwin.tar.xz"), + ("asphalt-x86_64-apple-darwin", vec!["tar", "xz"]) ); } }