diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index c7f9179..76012aa 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wasm-pkg-client" description = "Wasm package client" -repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wasm-pkg-client" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/" edition.workspace = true version.workspace = true authors.workspace = true @@ -44,5 +44,5 @@ tempfile = { workspace = true } [dev-dependencies] rcgen = { workspace = true } -rstest = "0.23" +rstest = { workspace = true } testcontainers = { workspace = true } diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index a97b7b8..5808600 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -131,6 +131,7 @@ impl Client { package: &PackageRef, version: &Version, ) -> Result { + // FIXME: the `None` below means we ignore workspace overrides for this call let source = self.resolve_source(package, None).await?; source.get_release(package, version).await } @@ -191,7 +192,12 @@ impl Client { fetch_semver_series(source.as_ref().as_ref(), &package, &version).await? { match version.cmp(&version_info.version) { - Ordering::Equal => return Err(Error::VersionAlreadyExists(version.to_owned())), + Ordering::Equal => { + return Err(Error::VersionAlreadyExists( + package.clone(), + version.to_owned(), + )); + } Ordering::Greater => { // incoming version is greater than neighbor neighbors[0] = Some(version_info); diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 1062366..f6d87e3 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -1,6 +1,6 @@ //! Local filesystem-based package backend. //! -//! Each package release is a file: `///.wasm` +//! Each package release is a file: `///.wasm` use std::{ io, diff --git a/crates/wasm-pkg-client/tests/publish_semver_check.rs b/crates/wasm-pkg-client/tests/publish_semver_check.rs index 5dee7c9..ec5cd47 100644 --- a/crates/wasm-pkg-client/tests/publish_semver_check.rs +++ b/crates/wasm-pkg-client/tests/publish_semver_check.rs @@ -38,8 +38,11 @@ fn semver_incompatible(previous: &str, new: &str) -> Error { } } -fn version_already_exists(version: &str) -> Error { - Error::VersionAlreadyExists(version.parse().unwrap()) +fn version_already_exists(package: &str, version: &str) -> Error { + Error::VersionAlreadyExists( + format!("{NAMESPACE}:{package}").parse().unwrap(), + version.parse().unwrap(), + ) } fn make_client(root: &Path) -> Client { @@ -141,7 +144,7 @@ async fn publish(client: &Client, bytes: Vec, opts: PublishOpts) -> Result<( Some(("0.1.0", WorldDiff::AddBase)), ("0.1.0", WorldDiff::AddBase), false, - Err(version_already_exists("0.1.0")) + Err(version_already_exists("dup-version", "0.1.0")) )] #[case::duplicate_version_with_skip_semver_check( "dup-version-opt-out", @@ -209,8 +212,12 @@ async fn publish_semver_check( assert_eq!(previous, exp_prev, "previous version mismatch"); assert_eq!(new, exp_new, "new version mismatch"); } - (Err(Error::VersionAlreadyExists(exp)), Err(Error::VersionAlreadyExists(actual))) => { - assert_eq!(actual, exp, "duplicate version mismatch"); + ( + Err(Error::VersionAlreadyExists(exp_pkg, exp_ver)), + Err(Error::VersionAlreadyExists(pkg, ver)), + ) => { + assert_eq!(pkg, exp_pkg, "duplicate package mismatch"); + assert_eq!(ver, exp_ver, "duplicate version mismatch"); } _ => panic!("expectation mismatch\n expected: {expected:?}\n actual: {result:?}",), } diff --git a/crates/wasm-pkg-common/Cargo.toml b/crates/wasm-pkg-common/Cargo.toml index d246758..e118235 100644 --- a/crates/wasm-pkg-common/Cargo.toml +++ b/crates/wasm-pkg-common/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wasm-pkg-common" description = "Wasm Package common types and configuration" -repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wasm-pkg-common" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/" edition.workspace = true version.workspace = true authors.workspace = true diff --git a/crates/wasm-pkg-common/src/lib.rs b/crates/wasm-pkg-common/src/lib.rs index a4053c8..9c49102 100644 --- a/crates/wasm-pkg-common/src/lib.rs +++ b/crates/wasm-pkg-common/src/lib.rs @@ -1,6 +1,8 @@ use http::uri::InvalidUri; use label::Label; +use crate::package::PackageRef; + #[cfg(feature = "registry-config")] pub mod config; pub mod digest; @@ -53,8 +55,8 @@ pub enum Error { RegistryMetadataError(#[source] anyhow::Error), #[error("version not found: {0}")] VersionNotFound(semver::Version), - #[error("version {0} is already published for this package")] - VersionAlreadyExists(semver::Version), + #[error("{0}@{1} already exists in the registry")] + VersionAlreadyExists(PackageRef, semver::Version), #[error( "new version {new} is not semver-compatible with existing version {previous}: {source:#}" )] diff --git a/crates/wasm-pkg-core/Cargo.toml b/crates/wasm-pkg-core/Cargo.toml index ebe8cb1..b7f085e 100644 --- a/crates/wasm-pkg-core/Cargo.toml +++ b/crates/wasm-pkg-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wasm-pkg-core" description = "Wasm Package Tools core libraries for wkg" -repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wasm-pkg-core" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/" edition.workspace = true version.workspace = true authors.workspace = true diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 9ca5125..5aba92c 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -28,6 +28,9 @@ use crate::{ }, }; +/// Directory holding WIT dependencies for one or more packages +pub const WIT_DEPS_DIR: &str = "deps"; + /// The supported output types for WIT deps #[derive(Debug, Clone, Copy, Default)] pub enum OutputType { diff --git a/crates/wasm-pkg-core/tests/build.rs b/crates/wasm-pkg-core/tests/build.rs index a1b40e4..99a9add 100644 --- a/crates/wasm-pkg-core/tests/build.rs +++ b/crates/wasm-pkg-core/tests/build.rs @@ -94,7 +94,7 @@ async fn test_bad_dep_failure() { .expect("Should be able to create a new lock file"); let (_temp_cache, client) = common::get_client().await.unwrap(); - let world_file = fixture_path.join("wit").join("proxy.wit"); + let world_file = fixture_path.join("wit/proxy.wit"); let str_world = tokio::fs::read_to_string(&world_file) .await .expect("Should be able to read the world file"); diff --git a/crates/wasm-pkg-core/tests/common.rs b/crates/wasm-pkg-core/tests/common.rs index f1adcbf..45bc81b 100644 --- a/crates/wasm-pkg-core/tests/common.rs +++ b/crates/wasm-pkg-core/tests/common.rs @@ -2,9 +2,10 @@ use std::path::{Path, PathBuf}; use tempfile::TempDir; use wasm_pkg_client::{ - Client, + Client, Config, caching::{CachingClient, FileCache}, }; +use wasm_pkg_core::wit::WIT_DEPS_DIR; pub fn fixture_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -13,7 +14,8 @@ pub fn fixture_dir() -> PathBuf { } pub async fn get_client() -> anyhow::Result<(TempDir, CachingClient)> { - let client = Client::with_global_defaults().await?; + // NOTE: `Client::with_global_defaults()` may pick up a user's redirect of `wasi` to a private registry + let client = Client::new(Config::default()); let cache_temp_dir = tempfile::tempdir()?; let cache = FileCache::new(cache_temp_dir.path()).await?; @@ -38,7 +40,7 @@ async fn copy_dir(source: impl AsRef, destination: impl AsRef) -> an let filetype = entry.file_type().await?; if filetype.is_dir() { // Skip the deps directory in case it is there from debugging - if entry.path().file_name().unwrap_or_default() == "deps" { + if entry.path().file_name().unwrap_or_default() == WIT_DEPS_DIR { continue; } Box::pin(copy_dir( diff --git a/crates/wasm-pkg-core/tests/fetch.rs b/crates/wasm-pkg-core/tests/fetch.rs index 11f9d2f..a56685c 100644 --- a/crates/wasm-pkg-core/tests/fetch.rs +++ b/crates/wasm-pkg-core/tests/fetch.rs @@ -60,7 +60,7 @@ async fn test_nested_local(#[values(OutputType::Wasm, OutputType::Wit)] output: overrides.insert( "my:local".to_string(), Override { - path: Some(fixture_path.join("local-dep").join("wit")), + path: Some(fixture_path.join("local-dep/wit")), ..Default::default() }, ); @@ -103,21 +103,21 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp ( "example-b:bar".to_string(), Override { - path: Some(fixture_path.join("example-b").join("wit")), + path: Some(fixture_path.join("example-b/wit")), version: None, }, ), ( "example-c:baz".to_string(), Override { - path: Some(fixture_path.join("example-c").join("wit")), + path: Some(fixture_path.join("example-c/wit")), version: None, }, ), ( "example-c:nested".to_string(), Override { - path: Some(fixture_path.join("example-c").join("wit/nested")), + path: Some(fixture_path.join("example-c/wit/nested")), version: None, }, ), @@ -138,7 +138,7 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp .unwrap_or_else(|e| panic!("Should be able to fetch the dependencies: {e:#}")); // Ensure that the deps directory contains the correct dependencies - let mut deps_dir = tokio::fs::read_dir(project_path.join("wit").join("deps")) + let mut deps_dir = tokio::fs::read_dir(project_path.join("wit/deps")) .await .expect("Should be able to read the deps directory"); let mut deps = Vec::new(); diff --git a/crates/wkg/Cargo.toml b/crates/wkg/Cargo.toml index 3cfdc84..dc01ac1 100644 --- a/crates/wkg/Cargo.toml +++ b/crates/wkg/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wkg" description = "Wasm Package Tools CLI" -repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wkg" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/" edition.workspace = true version.workspace = true authors.workspace = true diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 3e61343..887eb03 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -1,5 +1,4 @@ use std::{ - collections::HashMap, io::{Cursor, Seek}, path::PathBuf, }; @@ -8,30 +7,29 @@ use anstream::eprintln; use anyhow::{Context, anyhow, ensure}; use clap::{Args, Parser, Subcommand, ValueEnum}; use futures_util::TryStreamExt; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::io::AsyncWriteExt; use tracing::level_filters::LevelFilter; use wasm_pkg_client::{ - Client, PublishOpts, + Client, PackageRef, PublishOpts, Version, caching::{CachingClient, FileCache}, - local::LocalConfig, }; use wasm_pkg_common::{ self, - config::{Config, RegistryConfig, RegistryMapping}, - metadata::LOCAL_PROTOCOL, + config::{Config, RegistryMapping}, package::PackageSpec, registry::Registry, }; -use wasm_pkg_core::{lock::LockFile, resolver::PublishPlan}; +use wasm_pkg_core::lock::LockFile; use wit_component::DecodedWasm; mod oci; +mod overlay; mod wit; use oci::OciCommands; use wit::{BuildArgs, FetchArgs, UpdateArgs, WitCommands}; -use crate::wit::temp_wit_file; +use crate::{overlay::PublishVerifier, wit::temp_wit_file}; #[macro_export] macro_rules! warnln { @@ -290,6 +288,12 @@ struct PublishArgs { #[arg(long)] skip_semver_check: bool, + /// Skip publishing any package whose `(name, version)` already exists on + /// the target registry. A successful `get_release` probe short-circuits the + /// publish; probe failures fall through to a normal publish attempt. + #[arg(long)] + skip_dupes: bool, + #[command(flatten)] common: Common, } @@ -300,105 +304,42 @@ impl PublishArgs { let path = match &self.paths[..] { [path] => path, paths => { - let mut overlay_config = self.common.load_config().await?; - let cache = self.common.load_cache().await?; - - let local_config = LocalConfig::temp_dir()?; - let reg_config = RegistryConfig::default() - .with_default_backend(LOCAL_PROTOCOL, &local_config)?; - // Route every package in the plan to the local overlay registry - // backed by `reg_config`, so the client used in `build_wit_dir` - // resolves these packages against the local overlay instead of - // an upstream remote. - let local_registry: Registry = "tmp_local_publish".parse()?; - overlay_config - .get_or_insert_registry_config_mut(&local_registry) - .merge(reg_config); - - let mut plan = PublishPlan::from_paths(paths)?; - anstream::eprintln!("{plan}"); - - // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle - // querying on a per package, namespace, and registry level - // to handle cargo style overlays. - // see this reference of `cargo::core::Dependency` usage for local overlays in Cargo: - // https://github.com/rust-lang/cargo/blob/d6900d00af2644ea1c0068c5694d9dbe11a3ab39/src/cargo/sources/overlay.rs#L47 - // this is still needed for `wit::build_wit_dir` - for spec in plan.iter() { - overlay_config.set_package_registry_override( - spec.package.clone(), - RegistryMapping::Registry(local_registry.clone()), - ); - } - let client = CachingClient::new(Some(Client::new(overlay_config)), cache); - let mut lock_file = LockFile::load(false).await?; - // these are packages that have been successfully pushed to our "tmp_local_publish" - let mut validated_packages = HashMap::new(); - - // 1. Publish our packages to "tmp_local_publish" ensuring all dependencies are - // resolved by the local backend - for spec in plan.iter() { - let path = plan.get_path(&spec.package).unwrap(); - let data = if path.is_dir() { - let _prev_lock_ref = (lock_file.version, lock_file.packages.clone()); - let (_pkg_ref, _version, bytes) = - wit::build_wit_dir(&path, client.clone(), &mut lock_file).await?; - bytes - } else { - let mut file = tokio::fs::OpenOptions::new().read(true).open(path).await?; - let mut buf = Vec::new(); - file.read_exact(&mut buf).await?; - buf - }; - - let source = Box::pin(Cursor::new(data.clone())); - client - .client()? - .publish_release_data( - source, - PublishOpts { - package: None, - registry: Some(local_registry.clone()), - // we want to publish to "tmp_local_publish" regardless of flags passed in - dry_run: false, - skip_semver_check: publish_opts.skip_semver_check, - }, - ) - .await?; - - let id = plan - .get_node_index(&spec.package) - .expect("missing node index"); - validated_packages.insert(id, data); - } + let verifier = PublishVerifier::try_new( + paths, + "tmp_local_publish", + self.common.load_config().await?, + self.common.load_cache().await?, + &mut lock_file, + true, + ) + .await?; + let mut plan = verifier.plan; + anstream::eprintln!("{plan}"); let client = self.common.get_client().await?; - // 2. Publish our packages in "waves" to the actual registries ensuring all - // possible dependency free packages are published in the same group + // Publish our packages in "waves" to the actual registries ensuring all + // possible dependency free packages are published in the same group while !plan.is_empty() { - // `ready_for_publish` is guaranteed to be nonempty IF `plan.is_empty() == false + // `ready_for_publish` is guaranteed to be nonempty IF `plan.is_empty() == false` // // A `DependencyGraph` (`petgraph::Acyclic`) should always hold valid edges. // Any insertions to the `DependencyGraph` that would produce invalid edges should // result in an error when calling `try_update_edge` inside `wasm_pkg_core::wit::get_local_dependencies` let ready_for_publish = plan.take_ready(); for spec in &ready_for_publish { - let id = plan - .get_node_index(&spec.package) - .expect("missing node index"); - let source = Box::pin(Cursor::new(validated_packages[&id].clone())); + let data = verifier + .data + .get(&spec.package) + .expect("missing package ref"); + let source = Box::pin(Cursor::new(data.clone())); // we do not have guarantees that the underlying `PackagePublisher::publish` // will terminate - let (package, version) = client + let res = client .client()? .publish_release_data(source, publish_opts.clone()) - .await?; - if self.dry_run { - warnln!("Aborting publish due to dry run: {}@{}", package, version); - } else { - eprintln!("Published {}@{}", package, version); - } + .await; + self.handle_publish_result(res).context(spec.clone())?; } plan.mark_confirmed(ready_for_publish); } @@ -437,14 +378,31 @@ impl PublishArgs { (path.clone(), None) }; - let (package, version) = client + let res = client .client()? .publish_release_file(&publish_path, publish_opts) - .await?; - if self.dry_run { - eprintln!("Aborting publish due to dry run: {}@{}", package, version); - } else { - eprintln!("Published {}@{}", package, version); + .await; + self.handle_publish_result(res)?; + + Ok(()) + } + + fn handle_publish_result( + &self, + res: Result<(PackageRef, Version), wasm_pkg_common::Error>, + ) -> Result<(), anyhow::Error> { + match res { + Err(e @ wasm_pkg_common::Error::VersionAlreadyExists(..)) if self.skip_dupes => { + warnln!("Skipping publish: {e}"); + } + Ok((package, version)) => { + if self.dry_run { + warnln!("Aborting publish due to dry run: {}@{}", package, version); + } else { + eprintln!("Published {}@{}", package, version); + } + } + Err(e) => return Err(e.into()), } Ok(()) } diff --git a/crates/wkg/src/overlay.rs b/crates/wkg/src/overlay.rs new file mode 100644 index 0000000..4a3e8b0 --- /dev/null +++ b/crates/wkg/src/overlay.rs @@ -0,0 +1,107 @@ +use std::collections::{BTreeSet, HashMap}; +use std::io::Cursor; +use std::path::PathBuf; + +use anyhow::Context; +use wasm_pkg_client::{ + Client, PublishOpts, + caching::{CachingClient, FileCache}, + local::LocalConfig, +}; +use wasm_pkg_common::{ + config::{Config, RegistryConfig, RegistryMapping}, + metadata::LOCAL_PROTOCOL, + package::PackageRef, + registry::Registry, +}; +use wasm_pkg_core::{lock::LockFile, resolver::PublishPlan}; + +use crate::wit::build_wit_dir; + +/// A [`CachingClient`] and [`PublishPlan`] wired to a temporary local backend +pub(crate) struct PublishVerifier { + #[expect(dead_code, reason = "workspaces")] + pub(crate) client: CachingClient, + pub(crate) plan: PublishPlan, + #[expect(dead_code, reason = "workspaces")] + pub(crate) packages: BTreeSet, + pub(crate) data: HashMap>, + /// Held so the temp local backend outlives the returned client. + _local_config: LocalConfig, +} + +impl PublishVerifier { + pub(crate) async fn try_new( + paths: &[PathBuf], + registry_name: &str, + mut base_config: Config, + cache: FileCache, + lock_file: &mut LockFile, + capture_bytes: bool, + ) -> anyhow::Result { + let local_config = LocalConfig::temp_dir()?; + let reg_config = + RegistryConfig::default().with_default_backend(LOCAL_PROTOCOL, &local_config)?; + let registry: Registry = registry_name.parse()?; + base_config + .get_or_insert_registry_config_mut(®istry) + .merge(reg_config); + + let plan = PublishPlan::from_paths(paths).context("failed to build publish plan")?; + let packages: BTreeSet = plan.iter().map(|spec| spec.package.clone()).collect(); + + // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle + // querying on a per package, namespace, and registry level + // to handle cargo style overlays. + // see this reference of `cargo::core::Dependency` usage for local overlays in Cargo: + // https://github.com/rust-lang/cargo/blob/d6900d00af2644ea1c0068c5694d9dbe11a3ab39/src/cargo/sources/overlay.rs#L47 + for pkg in &packages { + base_config.set_package_registry_override( + pkg.clone(), + RegistryMapping::Registry(registry.clone()), + ); + } + + let client = CachingClient::new(Some(Client::new(base_config)), cache); + + let mut bytes_by_package = HashMap::new(); + for spec in plan.iter() { + let path = plan + .get_path(&spec.package) + .expect("PublishPlan guarantees a path for each iterated spec"); + let bytes = if path.is_dir() { + let (_pkg_ref, _version, bytes) = + build_wit_dir(path, client.clone(), lock_file).await?; + bytes + } else { + tokio::fs::read(path).await.with_context(|| { + format!("failed to read workspace member at {}", path.display()) + })? + }; + client + .client()? + .publish_release_data( + Box::pin(Cursor::new(bytes.clone())), + PublishOpts { + package: None, + registry: Some(registry.clone()), + dry_run: false, + skip_semver_check: false, + }, + ) + .await + .with_context(|| format!("verifier failed to publish: {}", spec.package))?; + if capture_bytes { + bytes_by_package.insert(spec.package.clone(), bytes); + } + } + + Ok(PublishVerifier { + client, + plan, + packages, + data: bytes_by_package, + _local_config: local_config, + }) + } +} diff --git a/crates/wkg/tests/common.rs b/crates/wkg/tests/common.rs index 7315ad8..0aca4de 100644 --- a/crates/wkg/tests/common.rs +++ b/crates/wkg/tests/common.rs @@ -12,6 +12,7 @@ use testcontainers::{ }; use tokio::{net::TcpListener, process::Command}; use wasm_pkg_client::{Config, CustomConfig, Registry, RegistryMetadata, oci::OciRegistryConfig}; +use wasm_pkg_core::wit::WIT_DEPS_DIR; /// Returns an open port on localhost pub async fn find_open_port() -> u16 { @@ -145,7 +146,7 @@ pub async fn copy_dir( let filetype = entry.file_type().await?; if filetype.is_dir() { // Skip the deps directory in case it is there from debugging - if entry.path().file_name().unwrap_or_default() == "deps" { + if entry.path().file_name().unwrap_or_default() == WIT_DEPS_DIR { continue; } Box::pin(copy_dir( diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index fd17e51..92a9899 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -158,11 +158,19 @@ async fn publish_multiple_transitive_local_packages() { #[tokio::test] pub async fn check() { + // Use an explicit config that maps `wasi` to `wasi.dev`. + let mut config = wasm_pkg_client::Config::empty(); + config.set_namespace_registry( + "wasi".parse().unwrap(), + wasm_pkg_client::RegistryMapping::Registry("wasi.dev".parse().unwrap()), + ); + let fixture = common::load_fixture("wasi-http").await; let output = fixture.temp_dir.path().join("out"); let get = fixture - .command() + .command_with_config(&config) + .await .arg("get") .arg("wasi:http") .arg("--output") @@ -173,7 +181,8 @@ pub async fn check() { assert!(get.success()); let check_same = fixture - .command() + .command_with_config(&config) + .await .arg("get") .arg("--check") .arg("wasi:http") @@ -187,7 +196,8 @@ pub async fn check() { std::fs::write(&output, vec![1, 2, 3, 4]).expect("overwrite output with bogus contents"); let check_diff = fixture - .command() + .command_with_config(&config) + .await .arg("get") .arg("--check") .arg("wasi:http")