From 16c8e3a2564237002d7fed7567cc79611ac5f8e4 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 10 Jul 2026 11:33:06 -0500 Subject: [PATCH 01/11] feat(wkg,core,client): intial wkg workspace approach --- Cargo.lock | 8 +- Cargo.toml | 5 + crates/wasm-pkg-client/Cargo.toml | 4 +- crates/wasm-pkg-client/src/lib.rs | 8 +- crates/wasm-pkg-client/src/local.rs | 2 +- .../tests/publish_semver_check.rs | 17 +- crates/wasm-pkg-common/Cargo.toml | 2 +- crates/wasm-pkg-common/src/lib.rs | 6 +- crates/wasm-pkg-core/Cargo.toml | 3 +- crates/wasm-pkg-core/src/lock.rs | 1 - crates/wasm-pkg-core/src/manifest.rs | 106 +++++- crates/wasm-pkg-core/src/manifest/paths.rs | 102 ++++++ .../wasm-pkg-core/src/manifest/workspace.rs | 311 ++++++++++++++++++ crates/wasm-pkg-core/src/wit.rs | 233 ++++++++----- crates/wasm-pkg-core/tests/build.rs | 2 +- crates/wasm-pkg-core/tests/common.rs | 8 +- crates/wasm-pkg-core/tests/fetch.rs | 10 +- .../tests/fixtures/transitive-local/wkg.toml | 6 + crates/wkg/Cargo.toml | 2 +- crates/wkg/src/main.rs | 199 ++++++----- crates/wkg/src/overlay.rs | 105 ++++++ crates/wkg/src/wit.rs | 195 +++++++++-- crates/wkg/tests/common.rs | 3 +- crates/wkg/tests/e2e.rs | 80 ++++- 24 files changed, 1180 insertions(+), 238 deletions(-) create mode 100644 crates/wasm-pkg-core/src/manifest/paths.rs create mode 100644 crates/wasm-pkg-core/src/manifest/workspace.rs create mode 100644 crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml create mode 100644 crates/wkg/src/overlay.rs diff --git a/Cargo.lock b/Cargo.lock index 2f647de..18f58b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3643,7 +3643,7 @@ dependencies = [ [[package]] name = "wasm-pkg-client" -version = "0.16.0" +version = "0.16.1" dependencies = [ "anyhow", "async-trait", @@ -3679,7 +3679,7 @@ dependencies = [ [[package]] name = "wasm-pkg-common" -version = "0.16.0" +version = "0.16.1" dependencies = [ "anyhow", "bytes", @@ -3699,7 +3699,7 @@ dependencies = [ [[package]] name = "wasm-pkg-core" -version = "0.16.0" +version = "0.16.1" dependencies = [ "anyhow", "futures-util", @@ -4190,7 +4190,7 @@ dependencies = [ [[package]] name = "wkg" -version = "0.16.0" +version = "0.16.1" dependencies = [ "anstream 0.6.21", "anstyle", diff --git a/Cargo.toml b/Cargo.toml index e50fa51..23a0aff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,3 +69,8 @@ extend-ignore-words-re = [ [workspace.metadata.typos.default.extend-words] # strategy shorthand strat = "strat" + +# The profile that 'dist' will build with +[profile.dist] +inherits = "release" +lto = "thin" diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index c7f9179..76012aa 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wasm-pkg-client" description = "Wasm package client" -repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wasm-pkg-client" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/" edition.workspace = true version.workspace = true authors.workspace = true @@ -44,5 +44,5 @@ tempfile = { workspace = true } [dev-dependencies] rcgen = { workspace = true } -rstest = "0.23" +rstest = { workspace = true } testcontainers = { workspace = true } diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index a97b7b8..5808600 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -131,6 +131,7 @@ impl Client { package: &PackageRef, version: &Version, ) -> Result { + // FIXME: the `None` below means we ignore workspace overrides for this call let source = self.resolve_source(package, None).await?; source.get_release(package, version).await } @@ -191,7 +192,12 @@ impl Client { fetch_semver_series(source.as_ref().as_ref(), &package, &version).await? { match version.cmp(&version_info.version) { - Ordering::Equal => return Err(Error::VersionAlreadyExists(version.to_owned())), + Ordering::Equal => { + return Err(Error::VersionAlreadyExists( + package.clone(), + version.to_owned(), + )); + } Ordering::Greater => { // incoming version is greater than neighbor neighbors[0] = Some(version_info); diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 1062366..f6d87e3 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -1,6 +1,6 @@ //! Local filesystem-based package backend. //! -//! Each package release is a file: `///.wasm` +//! Each package release is a file: `///.wasm` use std::{ io, diff --git a/crates/wasm-pkg-client/tests/publish_semver_check.rs b/crates/wasm-pkg-client/tests/publish_semver_check.rs index 5dee7c9..ec5cd47 100644 --- a/crates/wasm-pkg-client/tests/publish_semver_check.rs +++ b/crates/wasm-pkg-client/tests/publish_semver_check.rs @@ -38,8 +38,11 @@ fn semver_incompatible(previous: &str, new: &str) -> Error { } } -fn version_already_exists(version: &str) -> Error { - Error::VersionAlreadyExists(version.parse().unwrap()) +fn version_already_exists(package: &str, version: &str) -> Error { + Error::VersionAlreadyExists( + format!("{NAMESPACE}:{package}").parse().unwrap(), + version.parse().unwrap(), + ) } fn make_client(root: &Path) -> Client { @@ -141,7 +144,7 @@ async fn publish(client: &Client, bytes: Vec, opts: PublishOpts) -> Result<( Some(("0.1.0", WorldDiff::AddBase)), ("0.1.0", WorldDiff::AddBase), false, - Err(version_already_exists("0.1.0")) + Err(version_already_exists("dup-version", "0.1.0")) )] #[case::duplicate_version_with_skip_semver_check( "dup-version-opt-out", @@ -209,8 +212,12 @@ async fn publish_semver_check( assert_eq!(previous, exp_prev, "previous version mismatch"); assert_eq!(new, exp_new, "new version mismatch"); } - (Err(Error::VersionAlreadyExists(exp)), Err(Error::VersionAlreadyExists(actual))) => { - assert_eq!(actual, exp, "duplicate version mismatch"); + ( + Err(Error::VersionAlreadyExists(exp_pkg, exp_ver)), + Err(Error::VersionAlreadyExists(pkg, ver)), + ) => { + assert_eq!(pkg, exp_pkg, "duplicate package mismatch"); + assert_eq!(ver, exp_ver, "duplicate version mismatch"); } _ => panic!("expectation mismatch\n expected: {expected:?}\n actual: {result:?}",), } diff --git a/crates/wasm-pkg-common/Cargo.toml b/crates/wasm-pkg-common/Cargo.toml index d246758..e118235 100644 --- a/crates/wasm-pkg-common/Cargo.toml +++ b/crates/wasm-pkg-common/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wasm-pkg-common" description = "Wasm Package common types and configuration" -repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wasm-pkg-common" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/" edition.workspace = true version.workspace = true authors.workspace = true diff --git a/crates/wasm-pkg-common/src/lib.rs b/crates/wasm-pkg-common/src/lib.rs index a4053c8..9c49102 100644 --- a/crates/wasm-pkg-common/src/lib.rs +++ b/crates/wasm-pkg-common/src/lib.rs @@ -1,6 +1,8 @@ use http::uri::InvalidUri; use label::Label; +use crate::package::PackageRef; + #[cfg(feature = "registry-config")] pub mod config; pub mod digest; @@ -53,8 +55,8 @@ pub enum Error { RegistryMetadataError(#[source] anyhow::Error), #[error("version not found: {0}")] VersionNotFound(semver::Version), - #[error("version {0} is already published for this package")] - VersionAlreadyExists(semver::Version), + #[error("{0}@{1} already exists in the registry")] + VersionAlreadyExists(PackageRef, semver::Version), #[error( "new version {new} is not semver-compatible with existing version {previous}: {source:#}" )] diff --git a/crates/wasm-pkg-core/Cargo.toml b/crates/wasm-pkg-core/Cargo.toml index ebe8cb1..5e03473 100644 --- a/crates/wasm-pkg-core/Cargo.toml +++ b/crates/wasm-pkg-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wasm-pkg-core" description = "Wasm Package Tools core libraries for wkg" -repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wasm-pkg-core" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/" edition.workspace = true version.workspace = true authors.workspace = true @@ -23,6 +23,7 @@ wasm-pkg-client = { workspace = true } wit-component = { workspace = true } wit-parser = { workspace = true } petgraph = { workspace = true } +glob = "0.3.3" [target.'cfg(unix)'.dependencies.libc] version = "0.2" diff --git a/crates/wasm-pkg-core/src/lock.rs b/crates/wasm-pkg-core/src/lock.rs index 441bbcd..143bef6 100644 --- a/crates/wasm-pkg-core/src/lock.rs +++ b/crates/wasm-pkg-core/src/lock.rs @@ -373,7 +373,6 @@ impl AsRef for Locker { // NOTE(thomastaylor312): These lock file primitives from here on down are mostly copied wholesale // from the lock file implementation of cargo-component with some minor modifications to make them // work with tokio - impl Locker { // NOTE(thomastaylor312): I am keeping around these try methods for possible later use. Right // now we're ignoring the dead code diff --git a/crates/wasm-pkg-core/src/manifest.rs b/crates/wasm-pkg-core/src/manifest.rs index 3acd335..8ed7456 100644 --- a/crates/wasm-pkg-core/src/manifest.rs +++ b/crates/wasm-pkg-core/src/manifest.rs @@ -10,14 +10,28 @@ use semver::VersionReq; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; +mod paths; +pub mod workspace; + +use workspace::*; + +use crate::manifest::paths::{find_root_iter, find_root_manifest_for_wd}; + /// The default name of the manifest file. pub const MANIFEST_FILE_NAME: &str = "wkg.toml"; +/// Directory next to the root [`MANIFEST_FILE_NAME`] that holds multi-package `deps` and `config.toml`. +pub const WORKSPACE_OUT_DIR: &str = "wkg"; /// The structure for a wkg.toml manifest file. This file is entirely optional and is used for /// overriding and annotating wasm packages. +/// `workspace` is mutually exclusive with `overrides` and top-level `metadata` #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct Manifest { + /// Workspace declaration. + // TODO: this should be a `TomlWorkspace` so that serialization is not coupled to the config structure + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, /// Overrides for various packages #[serde(default, skip_serializing_if = "Option::is_none")] pub overrides: Option>, @@ -28,17 +42,70 @@ pub struct Manifest { } impl Manifest { + fn from_toml(contents: &str) -> Result { + let manifest: Manifest = + toml::from_str(contents).context("unable to parse manifest file")?; + manifest.validate()?; + Ok(manifest) + } + /// Loads a manifest file from the given path. pub async fn load_from_path(path: impl AsRef) -> Result { - tracing::info!(path = %path.as_ref().display(), "loading wkg manifest file"); - let contents = tokio::fs::read_to_string(path) - .await - .context("unable to load manifest from file")?; - let manifest: Manifest = - toml::from_str(&contents).context("unable to parse manifest file")?; + let path = path.as_ref(); + tracing::info!(path = %path.display(), "loading wkg manifest file"); + let contents = std::fs::read_to_string(path) + .with_context(|| format!("unable to load manifest from {}", path.display()))?; + let mut manifest = Self::from_toml(&contents) + .with_context(|| format!("invalid manifest at {}", path.display()))?; + if let Some(WorkspaceConfig::Root(root)) = &mut manifest.workspace { + root.root_dir = path + .parent() + .with_context(|| { + format!("manifest path has no parent directory: {}", path.display()) + })? + .to_path_buf(); + // Resolve globs and relative paths eagerly + root.members = WorkspaceRootConfig::resolve_members(&root.members, &root.root_dir); + } Ok(manifest) } + fn root(&self) -> Option<&WorkspaceRootConfig> { + if let Some(WorkspaceConfig::Root(root)) = &self.workspace { + return Some(&root); + } + None + } + + // `Manifest` validations, mirrors cargo's `Workspace::validate` + fn validate(&self) -> Result<()> { + self.validate_workspace_exclusivity()?; + // Add new validation rules with `self.validate_*()?;` + Ok(()) + } + + // no overrides or top-level metadata when workspace is present + fn validate_workspace_exclusivity(&self) -> Result<()> { + if self.workspace.is_none() { + return Ok(()); + } + let mut conflicts = Vec::new(); + if self.overrides.is_some() { + conflicts.push("overrides"); + } + if self.metadata.is_some() { + conflicts.push("metadata"); + } + if conflicts.is_empty() { + return Ok(()); + } + anyhow::bail!( + "`[workspace]` cannot coexist with: `[{}]` - \ + use `[workspace.metadata]` for workspace level values", + conflicts.join("]`, `[") + ); + } + /// Attempts to load the manifest from the current directory. Most of the time, users of this /// crate should use this function. Right now it just checks for a `wkg.toml` file in the current /// directory, but we could add more resolution logic in the future. If the file is not found, a @@ -51,6 +118,32 @@ impl Manifest { Self::load_from_path(manifest_path).await } + /// Tries to find the root workspace config + /// Returns `Ok(None)` when there is no `wkg.toml` ancestor that can be [`WorkspaceRootConfig`] + pub async fn load_root_workspace(cwd: &Path) -> Result> { + let Some(manifest_file) = find_root_manifest_for_wd(cwd) else { + return Ok(None); + }; + let manifest_dir = manifest_file.parent().unwrap(); + let manifest = Self::load_from_path(&manifest_file).await?; + + if let Some(root) = manifest.root() { + return Ok(Some(root.clone())); + } + + // keep walking up if we have not found root + for file in find_root_iter(&manifest_file) { + let manifest = Self::load_from_path(&file).await?; + if let Some(WorkspaceConfig::Root(root)) = manifest.workspace { + if root.is_explicitly_listed_member(&manifest_dir) { + return Ok(Some(root)); + } + } + } + + Ok(None) + } + /// Serializes and writes the manifest to the given path. pub async fn write(&self, path: impl AsRef) -> Result<()> { let contents = toml::to_string_pretty(self)?; @@ -117,6 +210,7 @@ mod tests { let tempdir = tempfile::tempdir().unwrap(); let manifest_path = tempdir.path().join(MANIFEST_FILE_NAME); let manifest = Manifest { + workspace: None, overrides: Some(HashMap::from([( "foo:bar".to_string(), Override { diff --git a/crates/wasm-pkg-core/src/manifest/paths.rs b/crates/wasm-pkg-core/src/manifest/paths.rs new file mode 100644 index 0000000..6bcf8d0 --- /dev/null +++ b/crates/wasm-pkg-core/src/manifest/paths.rs @@ -0,0 +1,102 @@ +use anyhow::Result; +use std::path::{Component, Path, PathBuf}; + +use crate::manifest::MANIFEST_FILE_NAME; + +/// Find the first ancestor [`super::Manifest`] path for current working directory. +pub(crate) fn find_root_manifest_for_wd(cwd: impl AsRef) -> Option { + for current in cwd.as_ref().ancestors() { + let manifest = current.join(MANIFEST_FILE_NAME); + if manifest.exists() { + return Some(manifest); + } + } + None +} + +// Returns first ancestor [`super::Manifest`] path for a given existing manifest +pub(crate) fn find_root_iter<'a>(manifest_path: &'a Path) -> impl Iterator + 'a { + manifest_path + .ancestors() + .skip(2) // skip `manifest_path` and the parent dir + .map(|dir| dir.join(MANIFEST_FILE_NAME)) + .filter(|path| path.exists()) +} + +// copied from: +// https://github.com/rust-lang/cargo/blob/a595d0da21f228b7fdae64d3d5c0e527ea66bb59/crates/cargo-util/src/paths.rs#L84-L84 +/// Normalize a path, removing things like `.` and `..`. +/// +/// CAUTION: This does not resolve symlinks (unlike +/// [`std::fs::canonicalize`]). This may cause incorrect or surprising +/// behavior at times. This should be used carefully. Unfortunately, +/// [`std::fs::canonicalize`] can be hard to use correctly, since it can often +/// fail, or on Windows returns annoying device paths. This is a problem Cargo +/// needs to improve on. +pub(crate) fn normalize_path(path: &Path) -> PathBuf { + let mut components = path.components().peekable(); + let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() { + components.next(); + PathBuf::from(c.as_os_str()) + } else { + PathBuf::new() + }; + + for component in components { + match component { + Component::Prefix(..) => unreachable!(), + Component::RootDir => { + ret.push(Component::RootDir); + } + Component::CurDir => {} + Component::ParentDir => { + if ret.ends_with(Component::ParentDir) { + ret.push(Component::ParentDir); + } else { + let popped = ret.pop(); + if !popped && !ret.has_root() { + ret.push(Component::ParentDir); + } + } + } + Component::Normal(c) => { + ret.push(c); + } + } + } + ret +} + +// Check if path holds at least one `.wit` file. +fn dir_contains_wit(dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(dir) else { + return false; + }; + entries.filter_map(Result::ok).any(|entry| { + entry.path().extension().is_some_and(|ext| ext == "wit") + && entry.file_type().is_ok_and(|ft| ft.is_file()) + }) +} + +// Expand a string if is determined to be a glob pattern +pub(crate) fn expand_globs(entry: &Path, root_dir: &Path) -> Result> { + let joined = root_dir.join(entry); + // TODO(mkatychev): handle escaping glob chars + let is_glob = entry.to_str().is_some_and(|s| s.contains(['*', '?', '['])); + let pattern = joined.to_str().filter(|_| is_glob); + let Some(pattern) = pattern else { + return Ok(vec![joined]); + }; + let matches = glob::glob(pattern)?; + let mut expanded: Vec = matches + .filter_map(Result::ok) + .filter(|p| p.is_dir() && dir_contains_wit(p)) + .collect(); + + if expanded.is_empty() { + return Ok(vec![joined]); + } + + expanded.sort(); + Ok(expanded) +} diff --git a/crates/wasm-pkg-core/src/manifest/workspace.rs b/crates/wasm-pkg-core/src/manifest/workspace.rs new file mode 100644 index 0000000..ba29d39 --- /dev/null +++ b/crates/wasm-pkg-core/src/manifest/workspace.rs @@ -0,0 +1,311 @@ +use crate::manifest::paths::{expand_globs, normalize_path}; + +use super::*; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// `[workspace]` table inside a `wkg.toml` manifest. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum WorkspaceConfig { + // `[workspace]` + Root(WorkspaceRootConfig), + // - `workspace = "path/to/workspace/root"` + // - TODO(mkatychev): implement workspace package level field + // - https://doc.rust-lang.org/cargo/reference/manifest.html#the-workspace-field + // Member { + // root: Option, + // }, +} + +impl Default for WorkspaceConfig { + fn default() -> Self { + // TODO(mkatychev): use member variant instead + Self::Root(WorkspaceRootConfig::default()) + } +} + +impl WorkspaceConfig { + /// Resolves the [`WorkspaceRootConfig`] for a given [`Self`] + pub fn as_root(&self) -> Option<&WorkspaceRootConfig> { + match self { + WorkspaceConfig::Root(r) => Some(r), + } + } +} + +/// Intermediate configuration of a workspace root in a manifest. +/// +/// Knows the Workspace Root path, as well as `members` and `metadata`, which +/// together tell if some path is recognized as a member by this root or not. +/// +/// `members` is stored as resolved, normalized paths: glob patterns from the +/// raw TOML are expanded, joined to `root_dir`, and normalized during +/// [`super::Manifest::load_from_path`]. Downstream code can read +/// `self.members` directly without re-resolving. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct WorkspaceRootConfig { + #[serde(default, skip)] + pub(super) root_dir: PathBuf, + pub members: Vec, + pub metadata: Option, +} + +impl WorkspaceRootConfig { + /// Directory containing the workspace-root `wkg.toml`. + pub fn root_dir(&self) -> &Path { + &self.root_dir + } + + /// Directory containing workspace level `deps` `and config.toml` + pub fn out_dir(&self) -> PathBuf { + self.root_dir.join(PathBuf::from(WORKSPACE_OUT_DIR)) + } + + /// Checks if the path is explicitly listed as a workspace member. + /// + /// Returns `true` ONLY if: + /// - The path is the workspace root manifest itself, or + /// - The path matches one of the explicit `members` patterns + /// + /// NOTE: This does NOT check for implicit path dependency membership. + /// A `false` return does NOT mean the package is definitely not a member - + /// it could still be a member via path dependencies. Callers should fallback + /// to full workspace loading when this returns `false`. + // FIXME: implement cargo WorkspaceRootConfig::{member_paths, expand_member_path} + // this should not be eagerly evaluated + pub(super) fn is_explicitly_listed_member(&self, manifest_path: &Path) -> bool { + let root_manifest = self.root_dir.join("Cargo.toml"); + if manifest_path == root_manifest { + return true; + } + + let manifest_path = normalize_path(manifest_path); + self.members.iter().any(|member| member == &manifest_path) + } + // Expand globs and relative paths + pub(super) fn resolve_members(members: &[PathBuf], root_dir: &Path) -> Vec { + members + .iter() + .flat_map(|entry| expand_globs(entry, root_dir)) + .flatten() + .map(|p| normalize_path(&p)) + .collect() + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + + // NOTE: tree dir comments generated using `eza -T dir/` + fn touch(path: impl AsRef) { + let path = path.as_ref(); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "").unwrap(); + } + + #[test] + fn find_root_manifest_for_wd_first_ancestor() { + // dir + // ├── wkg.toml + // └── pkg + // └── wkg.toml + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + touch(root_dir.join("wkg.toml")); + touch(root_dir.join("pkg/wkg.toml")); + fs::create_dir_all(root_dir.join("pkg/wit")).unwrap(); + + let found = find_root_manifest_for_wd(root_dir.join("pkg/wit")).unwrap(); + assert_eq!(found, root_dir.join("pkg/wkg.toml")); + } + + #[test] + fn find_root_manifest_for_wd_none() { + let tempdir = tempfile::tempdir().unwrap(); + assert!(find_root_manifest_for_wd(tempdir.path()).is_none()); + } + + #[test] + fn find_root_iter_ancestors() { + // dir + // ├── wkg.toml + // └── pkg + // └── wkg.toml + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + touch(root_dir.join("wkg.toml")); + let member = root_dir.join("pkg/wkg.toml"); + touch(&member); + + let roots: Vec = find_root_iter(&member).collect(); + assert_eq!(roots, vec![root_dir.join("wkg.toml")]); + } + + #[test] + fn resolve_members_skip_non_dirs() { + // dir + // ├── example-a + // │ └── wit + // │ └── pkg.wit + // ├── example-b + // │ └── wit + // │ └── pkg.wit + // ├── example-c + // │ └── wit + // │ └── pkg.wit + // └── example-d + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + let member_wit = ["example-a/wit", "example-b/wit", "example-c/wit"]; + for dir in member_wit { + fs::create_dir_all(root_dir.join(dir)).unwrap(); + fs::write(root_dir.join(dir).join("pkg.wit"), "").unwrap(); + } + fs::write(root_dir.join("example-d"), "").unwrap(); + + let members = vec![PathBuf::from("example-*/wit")]; + let mut result = WorkspaceRootConfig::resolve_members(&members, root_dir); + result.sort(); + let mut expected: Vec<_> = member_wit.iter().map(|p| root_dir.join(p)).collect(); + expected.sort(); + assert_eq!(result, expected, "glob should expand to matching dirs only"); + } + + // skip directories without `.wit` files + #[test] + fn resolve_members_skip_non_wit_dirs() { + // dir + // └── pkg + // ├── core + // │ └── types.wit + // └── filesystem + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + let core = root_dir.join("pkg/core"); + let filesystem = root_dir.join("pkg/filesystem"); + fs::create_dir_all(&core).unwrap(); + fs::write(core.join("types.wit"), "package foo:core;\n").unwrap(); + fs::create_dir_all(filesystem).unwrap(); + + let members = vec![PathBuf::from("pkg/*")]; + let result = WorkspaceRootConfig::resolve_members(&members, root_dir); + assert_eq!(result, vec![core]); + } + + #[test] + fn resolved_members_keeps_literal_even_when_empty() { + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + // dir + // └── empty-pkg + fs::create_dir_all(root_dir.join("empty-pkg")).unwrap(); + + let members = vec![PathBuf::from("empty-pkg")]; + let result = WorkspaceRootConfig::resolve_members(&members, root_dir); + assert_eq!(result, vec![root_dir.join("empty-pkg")]); + } + + #[test] + fn workspace_root() { + let toml = r#" +[workspace] +members = ["a/wit", "b/wit"] +"#; + let cfg = Manifest::from_toml(toml).unwrap(); + let root = cfg.workspace.unwrap(); + assert_eq!( + root.as_root().unwrap().members, + vec![PathBuf::from("a/wit"), PathBuf::from("b/wit")], + ); + assert!(cfg.overrides.is_none() && cfg.metadata.is_none()); + } + + #[test] + fn workspace_metadata() { + let toml = r#" +[workspace] +members = ["a/wit"] + +[workspace.metadata] +authors = "Webster Assembler" +"#; + let cfg = Manifest::from_toml(toml).unwrap(); + let root = cfg.root().unwrap(); + let meta = root.metadata.as_ref().unwrap(); + assert_eq!(meta.authors.as_ref().unwrap(), "Webster Assembler"); + } + + #[test] + fn workspace_overrides_incompatible() { + let toml = r#" +[workspace] +members = ["a/wit"] + +[overrides."foo:bar"] +path = "oo-bar" +"#; + Manifest::from_toml(toml).unwrap_err(); + } + + fn write(path: impl AsRef, contents: &str) { + let path = path.as_ref(); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, contents).unwrap(); + } + + #[tokio::test] + async fn load_workspace_from_root_and_member() { + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + // dir + // ├── wkg.toml + // └── pkg-a + // └── wit + write( + root_dir.join("wkg.toml"), + r#" +[workspace] +members = ["pkg-a"] +"#, + ); + fs::create_dir_all(root_dir.join("pkg-a/wit")).unwrap(); + + let expected = root_dir.canonicalize().unwrap(); + let from_root = Manifest::load_root_workspace(root_dir) + .await + .unwrap() + .unwrap(); + assert_eq!(from_root.root_dir.canonicalize().unwrap(), expected); + + let from_member = Manifest::load_root_workspace(&root_dir.join("pkg-a/wit")) + .await + .unwrap() + .unwrap(); + assert_eq!(from_member.root_dir.canonicalize().unwrap(), expected); + } + + #[tokio::test] + async fn load_workspace_no_root() { + // Standalone: wkg.toml exists but has no [workspace] table -> None. + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + write( + root_dir.join("wkg.toml"), + r#" +[metadata] +authors = "Webster Assembler" +"#, + ); + assert!( + Manifest::load_root_workspace(root_dir) + .await + .unwrap() + .is_none() + ); + } +} diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 9ca5125..7012e22 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -2,11 +2,12 @@ use std::{ collections::{HashMap, HashSet}, - path::Path, + path::{Path, PathBuf}, str::FromStr, }; use anyhow::{Context as _, Result, bail}; +use indexmap::IndexMap; use petgraph::{Direction, data::Build}; use semver::{Version, VersionReq}; use wasm_metadata::{AddMetadata, AddMetadataField}; @@ -28,6 +29,9 @@ use crate::{ }, }; +/// Directory holding WIT dependencies for one or more packages +pub const WIT_DEPS_DIR: &str = "deps"; + /// The supported output types for WIT deps #[derive(Debug, Clone, Copy, Default)] pub enum OutputType { @@ -318,91 +322,22 @@ pub async fn resolve_dependencies( /// will be created. Any existing files in the directory will be deleted. The dependencies will be /// put into the `deps` subdirectory within the directory in the format specified by the output /// type. Please note that if a local dep is encountered when using [`OutputType::Wasm`] and it -/// isn't a wasm binary, it will be copied directly to the directory and not packaged into a wit +/// isn't a wasm binary, it will be copied directly to the directory and not packaged into a WIT /// package first pub async fn populate_dependencies( path: impl AsRef, deps: &DependencyResolutionMap, output: OutputType, ) -> Result<()> { - // Canonicalizing will error if the path doesn't exist, so we don't need to check for that - let path = tokio::fs::canonicalize(path).await?; - let metadata = tokio::fs::metadata(&path).await?; - if !metadata.is_dir() { - anyhow::bail!("Path is not a directory"); - } - let deps_path = path.join("deps"); - // Remove the whole directory if it already exists and then recreate - if let Err(e) = tokio::fs::remove_dir_all(&deps_path).await { - // If the directory doesn't exist, ignore the error - if e.kind() != std::io::ErrorKind::NotFound { - return Err(anyhow::anyhow!("Unable to remove deps directory: {e}")); - } - } - tokio::fs::create_dir_all(&deps_path).await?; + let deps_path = prepare_deps_dir(path.as_ref()).await?; // For wit output, generate the resolve and then output each package in the resolve if let OutputType::Wit = output { - let (resolve, pkg_id) = deps.generate_resolve(&path).await?; + let (resolve, pkg_id) = deps.generate_resolve(path.as_ref()).await?; return print_wit_from_resolve(&resolve, pkg_id, &deps_path).await; } - // If we got binary output, write them instead of the wit - let decoded_deps = deps.decode_dependencies().await?; - - for (name, dep) in decoded_deps.iter() { - let mut output_path = deps_path.join(name_from_package_name(name)); - - match dep { - DecodedDependency::Wit { - resolution: DependencyResolution::Local(local), - .. - } => { - // Local deps always need to be written to a subdirectory of deps so create that here - tokio::fs::create_dir_all(&output_path).await?; - write_local_dep(local, output_path).await?; - } - // This case shouldn't happen because registries only support wit packages. We can't get - // a resolve from the unresolved group, so error out here. Ideally we could print the - // unresolved group, but WitPrinter doesn't support that yet - DecodedDependency::Wit { - resolution: DependencyResolution::Registry(_), - .. - } => { - anyhow::bail!("Unable to resolve dependency, this is a programmer error"); - } - // Right now WIT packages include all of their dependencies, so we don't need to fetch - // those too. In the future, we'll need to look for unsatisfied dependencies and fetch - // them - DecodedDependency::Wasm { resolution, .. } => { - // This is going to be written to a single file, so we don't create a directory here - // NOTE(thomastaylor312): This janky looking thing is to avoid chopping off the - // patch number from the release. Once `add_extension` is stabilized, we can use - // that instead - let mut file_name = output_path.file_name().unwrap().to_owned(); - file_name.push(".wasm"); - output_path.set_file_name(file_name); - match resolution { - DependencyResolution::Local(local) => { - let meta = tokio::fs::metadata(&local.path).await?; - if !meta.is_file() { - anyhow::bail!("Local dependency is not single wit package file"); - } - tokio::fs::copy(&local.path, output_path) - .await - .context("Unable to copy local dependency")?; - } - DependencyResolution::Registry(registry) => { - let mut reader = registry.fetch().await?; - let mut output_file = tokio::fs::File::create(output_path).await?; - tokio::io::copy(&mut reader, &mut output_file).await?; - output_file.sync_all().await?; - } - } - } - } - } - Ok(()) + write_wasm_deps(&deps_path, &deps.decode_dependencies().await?).await } fn packages_from_foreign_deps( @@ -455,11 +390,23 @@ async fn print_wit_from_resolve( top_level_id: PackageId, root_deps_dir: &Path, ) -> Result<()> { - for (id, pkg) in resolve - .packages - .iter() - .filter(|(id, _)| *id != top_level_id) - { + print_wit_packages( + resolve, + root_deps_dir, + resolve + .packages + .iter() + .filter(|(id, _)| *id != top_level_id), + ) + .await +} + +async fn print_wit_packages<'a>( + resolve: &Resolve, + root_deps_dir: &Path, + packages: impl IntoIterator, +) -> Result<()> { + for (id, pkg) in packages { let dep_path = root_deps_dir.join(name_from_package_name(&pkg.name)); tokio::fs::create_dir_all(&dep_path).await?; let mut printer = WitPrinter::default(); @@ -471,6 +418,134 @@ async fn print_wit_from_resolve( Ok(()) } +/// Populate dependency list to a given directory, aggregating subdirectory dependencies. +/// Behaves like [`populate_dependencies`], but is intended for workspace-scoped aggregation +/// +/// [`OutputType::Wit`] will merge all decoded dependencies into a single [`Resolve`], printing +/// each package. +/// [`OutputType::Wasm`] behaves identically to [`populate_dependencies`]. +pub async fn populate_dependencies_workspace( + path: impl AsRef, + deps: &DependencyResolutionMap, + output: OutputType, +) -> Result<()> { + let deps_path = prepare_deps_dir(path.as_ref()).await?; + let deps = deps.decode_dependencies().await?; + + if let OutputType::Wit = output { + let mut merged = Resolve { + all_features: true, + ..Resolve::default() + }; + for decoded in deps.into_values() { + match decoded { + DecodedDependency::Wit { + resolution, + package, + } => { + let name = resolution.name().to_string(); + merged + .push_group(package) + .with_context(|| format!("failed to merge `{name}`"))?; + } + DecodedDependency::Wasm { + resolution, + decoded, + } => { + let name = resolution.name().to_string(); + let resolve = match decoded { + wit_component::DecodedWasm::WitPackage(resolve, _) => resolve, + wit_component::DecodedWasm::Component(resolve, _) => resolve, + }; + merged + .merge(resolve) + .with_context(|| format!("failed to merge world for `{name}`"))?; + } + } + } + return print_wit_packages(&merged, &deps_path, merged.packages.iter()).await; + } + + write_wasm_deps(&deps_path, &deps).await +} + +async fn prepare_deps_dir(path: &Path) -> Result { + // Canonicalizing will error if the path doesn't exist, so we don't need to check for that + let path = tokio::fs::canonicalize(path).await?; + if !tokio::fs::metadata(&path).await?.is_dir() { + anyhow::bail!("Path is not a directory"); + } + let deps_path = path.join(WIT_DEPS_DIR); + // Remove the whole directory if it already exists and then recreate + if let Err(e) = tokio::fs::remove_dir_all(&deps_path).await + && e.kind() != std::io::ErrorKind::NotFound + { + return Err(anyhow::anyhow!("Unable to remove deps directory: {e}") + .context(format!("dir: {}", deps_path.display()))); + } + tokio::fs::create_dir_all(&deps_path).await?; + Ok(deps_path) +} + +async fn write_wasm_deps( + deps_path: &Path, + decoded_deps: &IndexMap>, +) -> Result<()> { + for (name, dep) in decoded_deps.iter() { + let mut output_path = deps_path.join(name_from_package_name(name)); + + match dep { + DecodedDependency::Wit { + resolution: DependencyResolution::Local(local), + .. + } => { + // Local deps always need to be written to a subdirectory of deps so create that here + tokio::fs::create_dir_all(&output_path).await?; + write_local_dep(local, output_path).await?; + } + // This case shouldn't happen because registries only support wit packages. We can't get + // a resolve from the unresolved group, so error out here. Ideally we could print the + // unresolved group, but WitPrinter doesn't support that yet + DecodedDependency::Wit { + resolution: DependencyResolution::Registry(_), + .. + } => { + anyhow::bail!("Unable to resolve dependency, this is a programmer error"); + } + // Right now WIT packages include all of their dependencies, so we don't need to fetch + // those too. In the future, we'll need to look for unsatisfied dependencies and fetch + // them + DecodedDependency::Wasm { resolution, .. } => { + // This is going to be written to a single file, so we don't create a directory here + // NOTE(thomastaylor312): This janky looking thing is to avoid chopping off the + // patch number from the release. Once `add_extension` is stabilized, we can use + // that instead + let mut file_name = output_path.file_name().unwrap().to_owned(); + file_name.push(".wasm"); + output_path.set_file_name(file_name); + match resolution { + DependencyResolution::Local(local) => { + let meta = tokio::fs::metadata(&local.path).await?; + if !meta.is_file() { + anyhow::bail!("Local dependency is not single wit package file"); + } + tokio::fs::copy(&local.path, output_path) + .await + .context("Unable to copy local dependency")?; + } + DependencyResolution::Registry(registry) => { + let mut reader = registry.fetch().await?; + let mut output_file = tokio::fs::File::create(output_path).await?; + tokio::io::copy(&mut reader, &mut output_file).await?; + output_file.sync_all().await?; + } + } + } + } + } + Ok(()) +} + /// Given a package name, returns a valid directory/file name for it (thanks windows!) fn name_from_package_name(package_name: &PackageName) -> String { let package_name_str = package_name.to_string(); diff --git a/crates/wasm-pkg-core/tests/build.rs b/crates/wasm-pkg-core/tests/build.rs index a1b40e4..99a9add 100644 --- a/crates/wasm-pkg-core/tests/build.rs +++ b/crates/wasm-pkg-core/tests/build.rs @@ -94,7 +94,7 @@ async fn test_bad_dep_failure() { .expect("Should be able to create a new lock file"); let (_temp_cache, client) = common::get_client().await.unwrap(); - let world_file = fixture_path.join("wit").join("proxy.wit"); + let world_file = fixture_path.join("wit/proxy.wit"); let str_world = tokio::fs::read_to_string(&world_file) .await .expect("Should be able to read the world file"); diff --git a/crates/wasm-pkg-core/tests/common.rs b/crates/wasm-pkg-core/tests/common.rs index f1adcbf..45bc81b 100644 --- a/crates/wasm-pkg-core/tests/common.rs +++ b/crates/wasm-pkg-core/tests/common.rs @@ -2,9 +2,10 @@ use std::path::{Path, PathBuf}; use tempfile::TempDir; use wasm_pkg_client::{ - Client, + Client, Config, caching::{CachingClient, FileCache}, }; +use wasm_pkg_core::wit::WIT_DEPS_DIR; pub fn fixture_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -13,7 +14,8 @@ pub fn fixture_dir() -> PathBuf { } pub async fn get_client() -> anyhow::Result<(TempDir, CachingClient)> { - let client = Client::with_global_defaults().await?; + // NOTE: `Client::with_global_defaults()` may pick up a user's redirect of `wasi` to a private registry + let client = Client::new(Config::default()); let cache_temp_dir = tempfile::tempdir()?; let cache = FileCache::new(cache_temp_dir.path()).await?; @@ -38,7 +40,7 @@ async fn copy_dir(source: impl AsRef, destination: impl AsRef) -> an let filetype = entry.file_type().await?; if filetype.is_dir() { // Skip the deps directory in case it is there from debugging - if entry.path().file_name().unwrap_or_default() == "deps" { + if entry.path().file_name().unwrap_or_default() == WIT_DEPS_DIR { continue; } Box::pin(copy_dir( diff --git a/crates/wasm-pkg-core/tests/fetch.rs b/crates/wasm-pkg-core/tests/fetch.rs index 11f9d2f..a56685c 100644 --- a/crates/wasm-pkg-core/tests/fetch.rs +++ b/crates/wasm-pkg-core/tests/fetch.rs @@ -60,7 +60,7 @@ async fn test_nested_local(#[values(OutputType::Wasm, OutputType::Wit)] output: overrides.insert( "my:local".to_string(), Override { - path: Some(fixture_path.join("local-dep").join("wit")), + path: Some(fixture_path.join("local-dep/wit")), ..Default::default() }, ); @@ -103,21 +103,21 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp ( "example-b:bar".to_string(), Override { - path: Some(fixture_path.join("example-b").join("wit")), + path: Some(fixture_path.join("example-b/wit")), version: None, }, ), ( "example-c:baz".to_string(), Override { - path: Some(fixture_path.join("example-c").join("wit")), + path: Some(fixture_path.join("example-c/wit")), version: None, }, ), ( "example-c:nested".to_string(), Override { - path: Some(fixture_path.join("example-c").join("wit/nested")), + path: Some(fixture_path.join("example-c/wit/nested")), version: None, }, ), @@ -138,7 +138,7 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp .unwrap_or_else(|e| panic!("Should be able to fetch the dependencies: {e:#}")); // Ensure that the deps directory contains the correct dependencies - let mut deps_dir = tokio::fs::read_dir(project_path.join("wit").join("deps")) + let mut deps_dir = tokio::fs::read_dir(project_path.join("wit/deps")) .await .expect("Should be able to read the deps directory"); let mut deps = Vec::new(); diff --git a/crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml b/crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml new file mode 100644 index 0000000..acf504d --- /dev/null +++ b/crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml @@ -0,0 +1,6 @@ +[workspace] +# Mix glob with literal entries +members = [ + "example-*/wit", + "example-c/wit/nested", +] diff --git a/crates/wkg/Cargo.toml b/crates/wkg/Cargo.toml index 3cfdc84..dc01ac1 100644 --- a/crates/wkg/Cargo.toml +++ b/crates/wkg/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "wkg" description = "Wasm Package Tools CLI" -repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wkg" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/" edition.workspace = true version.workspace = true authors.workspace = true diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 3e61343..4b5c8da 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -1,37 +1,38 @@ use std::{ - collections::HashMap, io::{Cursor, Seek}, path::PathBuf, }; use anstream::eprintln; -use anyhow::{Context, anyhow, ensure}; +use anyhow::{Context, anyhow, bail, ensure}; use clap::{Args, Parser, Subcommand, ValueEnum}; use futures_util::TryStreamExt; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::io::AsyncWriteExt; use tracing::level_filters::LevelFilter; use wasm_pkg_client::{ - Client, PublishOpts, + Client, PackageRef, PublishOpts, Version, caching::{CachingClient, FileCache}, - local::LocalConfig, }; use wasm_pkg_common::{ self, - config::{Config, RegistryConfig, RegistryMapping}, - metadata::LOCAL_PROTOCOL, + config::{Config, RegistryMapping}, package::PackageSpec, registry::Registry, }; -use wasm_pkg_core::{lock::LockFile, resolver::PublishPlan}; +use wasm_pkg_core::{ + lock::LockFile, + manifest::{Manifest, workspace::WorkspaceRootConfig}, +}; use wit_component::DecodedWasm; mod oci; +mod overlay; mod wit; use oci::OciCommands; use wit::{BuildArgs, FetchArgs, UpdateArgs, WitCommands}; -use crate::wit::temp_wit_file; +use crate::{overlay::PublishVerifier, wit::temp_wit_file}; #[macro_export] macro_rules! warnln { @@ -286,119 +287,77 @@ struct PublishArgs { #[arg(long)] dry_run: bool, + /// Publish all packages in the workspace + #[arg(long)] + workspace: bool, + /// Disable semver compatibility checks. #[arg(long)] skip_semver_check: bool, + /// Skip publishing any package whose `(name, version)` already exists on + /// the target registry. A successful `get_release` probe short-circuits the + /// publish; probe failures fall through to a normal publish attempt. + #[arg(long)] + skip_dupes: bool, + #[command(flatten)] common: Common, } impl PublishArgs { - pub async fn run(self) -> anyhow::Result<()> { + pub async fn run(mut self) -> anyhow::Result<()> { let publish_opts = self.publish_opts()?; + let _root = self.workspace_root().await?; let path = match &self.paths[..] { + [] => { + anyhow::bail!( + "no publish targets: pass one or more paths, or run from a workspace \ + (see `[workspace] members` in `wkg.toml`)" + ); + } [path] => path, paths => { - let mut overlay_config = self.common.load_config().await?; - let cache = self.common.load_cache().await?; - - let local_config = LocalConfig::temp_dir()?; - let reg_config = RegistryConfig::default() - .with_default_backend(LOCAL_PROTOCOL, &local_config)?; // Route every package in the plan to the local overlay registry // backed by `reg_config`, so the client used in `build_wit_dir` // resolves these packages against the local overlay instead of // an upstream remote. - let local_registry: Registry = "tmp_local_publish".parse()?; - overlay_config - .get_or_insert_registry_config_mut(&local_registry) - .merge(reg_config); - - let mut plan = PublishPlan::from_paths(paths)?; - anstream::eprintln!("{plan}"); - - // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle - // querying on a per package, namespace, and registry level - // to handle cargo style overlays. - // see this reference of `cargo::core::Dependency` usage for local overlays in Cargo: - // https://github.com/rust-lang/cargo/blob/d6900d00af2644ea1c0068c5694d9dbe11a3ab39/src/cargo/sources/overlay.rs#L47 - // this is still needed for `wit::build_wit_dir` - for spec in plan.iter() { - overlay_config.set_package_registry_override( - spec.package.clone(), - RegistryMapping::Registry(local_registry.clone()), - ); - } - let client = CachingClient::new(Some(Client::new(overlay_config)), cache); - let mut lock_file = LockFile::load(false).await?; - // these are packages that have been successfully pushed to our "tmp_local_publish" - let mut validated_packages = HashMap::new(); - - // 1. Publish our packages to "tmp_local_publish" ensuring all dependencies are - // resolved by the local backend - for spec in plan.iter() { - let path = plan.get_path(&spec.package).unwrap(); - let data = if path.is_dir() { - let _prev_lock_ref = (lock_file.version, lock_file.packages.clone()); - let (_pkg_ref, _version, bytes) = - wit::build_wit_dir(&path, client.clone(), &mut lock_file).await?; - bytes - } else { - let mut file = tokio::fs::OpenOptions::new().read(true).open(path).await?; - let mut buf = Vec::new(); - file.read_exact(&mut buf).await?; - buf - }; - - let source = Box::pin(Cursor::new(data.clone())); - client - .client()? - .publish_release_data( - source, - PublishOpts { - package: None, - registry: Some(local_registry.clone()), - // we want to publish to "tmp_local_publish" regardless of flags passed in - dry_run: false, - skip_semver_check: publish_opts.skip_semver_check, - }, - ) - .await?; - - let id = plan - .get_node_index(&spec.package) - .expect("missing node index"); - validated_packages.insert(id, data); - } + let verifier = PublishVerifier::try_new( + paths, + "tmp_local_publish", + self.common.load_config().await?, + self.common.load_cache().await?, + &mut lock_file, + true, + ) + .await?; + let mut plan = verifier.plan; + anstream::eprintln!("{plan}"); let client = self.common.get_client().await?; - // 2. Publish our packages in "waves" to the actual registries ensuring all - // possible dependency free packages are published in the same group + // Publish our packages in "waves" to the actual registries ensuring all + // possible dependency free packages are published in the same group while !plan.is_empty() { - // `ready_for_publish` is guaranteed to be nonempty IF `plan.is_empty() == false + // `ready_for_publish` is guaranteed to be nonempty IF `plan.is_empty() == false` // // A `DependencyGraph` (`petgraph::Acyclic`) should always hold valid edges. // Any insertions to the `DependencyGraph` that would produce invalid edges should // result in an error when calling `try_update_edge` inside `wasm_pkg_core::wit::get_local_dependencies` let ready_for_publish = plan.take_ready(); for spec in &ready_for_publish { - let id = plan - .get_node_index(&spec.package) - .expect("missing node index"); - let source = Box::pin(Cursor::new(validated_packages[&id].clone())); + let data = verifier + .data + .get(&spec.package) + .expect("missing package ref"); + let source = Box::pin(Cursor::new(data.clone())); // we do not have guarantees that the underlying `PackagePublisher::publish` // will terminate - let (package, version) = client + let res = client .client()? .publish_release_data(source, publish_opts.clone()) - .await?; - if self.dry_run { - warnln!("Aborting publish due to dry run: {}@{}", package, version); - } else { - eprintln!("Published {}@{}", package, version); - } + .await; + self.handle_publish_result(res).context(spec.clone())?; } plan.mark_confirmed(ready_for_publish); } @@ -437,23 +396,20 @@ impl PublishArgs { (path.clone(), None) }; - let (package, version) = client + let res = client .client()? .publish_release_file(&publish_path, publish_opts) - .await?; - if self.dry_run { - eprintln!("Aborting publish due to dry run: {}@{}", package, version); - } else { - eprintln!("Published {}@{}", package, version); - } + .await; + self.handle_publish_result(res)?; + Ok(()) } - fn publish_opts(&self) -> anyhow::Result { + fn publish_opts(&mut self) -> anyhow::Result { let package = match self.package.clone() { - Some(_) if self.paths.len() > 2 => { + Some(_) if self.paths.len() != 1 => { anyhow::bail!( - "`--package` is currently unsupported when providing more than one path argument" + "`--package` is currently only supported when providing one path argument" ); } Some(PackageSpec { @@ -465,6 +421,7 @@ impl PublishArgs { } None => None, }; + Ok(PublishOpts { package, registry: self.registry_args.registry.clone(), @@ -472,6 +429,46 @@ impl PublishArgs { skip_semver_check: self.skip_semver_check, }) } + async fn workspace_root(&mut self) -> anyhow::Result> { + match self.workspace { + true if !self.paths.is_empty() => anyhow::bail!( + "`--workspace` selects every workspace member; do not also pass explicit \ + path arguments" + ), + + true => {} + false => return Ok(None), + } + let cwd = std::env::current_dir()?; + let Some(root) = Manifest::load_root_workspace(&cwd).await? else { + bail!( + "`--workspace` called but unable to find workspace root from {}", + cwd.display(), + ) + }; + self.paths = root.members.clone(); + Ok(Some(root)) + } + + fn handle_publish_result( + &self, + res: Result<(PackageRef, Version), wasm_pkg_common::Error>, + ) -> Result<(), anyhow::Error> { + match res { + Err(e @ wasm_pkg_common::Error::VersionAlreadyExists(..)) if self.skip_dupes => { + warnln!("Skipping publish: {e}"); + } + Ok((package, version)) => { + if self.dry_run { + warnln!("Aborting publish due to dry run: {}@{}", package, version); + } else { + eprintln!("Published {}@{}", package, version); + } + } + Err(e) => return Err(e.into()), + } + Ok(()) + } } #[derive(ValueEnum, Clone, Debug, PartialEq)] diff --git a/crates/wkg/src/overlay.rs b/crates/wkg/src/overlay.rs new file mode 100644 index 0000000..0f40085 --- /dev/null +++ b/crates/wkg/src/overlay.rs @@ -0,0 +1,105 @@ +use std::collections::{BTreeSet, HashMap}; +use std::io::Cursor; +use std::path::PathBuf; + +use anyhow::Context; +use wasm_pkg_client::{ + Client, PublishOpts, + caching::{CachingClient, FileCache}, + local::LocalConfig, +}; +use wasm_pkg_common::{ + config::{Config, RegistryConfig, RegistryMapping}, + metadata::LOCAL_PROTOCOL, + package::PackageRef, + registry::Registry, +}; +use wasm_pkg_core::{lock::LockFile, resolver::PublishPlan}; + +use crate::wit::build_wit_dir; + +/// A [`CachingClient`] and [`PublishPlan`] wired to a temporary local backend +pub(crate) struct PublishVerifier { + pub(crate) client: CachingClient, + pub(crate) plan: PublishPlan, + pub(crate) packages: BTreeSet, + pub(crate) data: HashMap>, + /// Held so the temp local backend outlives the returned client. + _local_config: LocalConfig, +} + +impl PublishVerifier { + pub(crate) async fn try_new( + paths: &[PathBuf], + registry_name: &str, + mut base_config: Config, + cache: FileCache, + lock_file: &mut LockFile, + capture_bytes: bool, + ) -> anyhow::Result { + let local_config = LocalConfig::temp_dir()?; + let reg_config = + RegistryConfig::default().with_default_backend(LOCAL_PROTOCOL, &local_config)?; + let registry: Registry = registry_name.parse()?; + base_config + .get_or_insert_registry_config_mut(®istry) + .merge(reg_config); + + let plan = PublishPlan::from_paths(paths).context("failed to build publish plan")?; + let packages: BTreeSet = plan.iter().map(|spec| spec.package.clone()).collect(); + + // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle + // querying on a per package, namespace, and registry level + // to handle cargo style overlays. + // see this reference of `cargo::core::Dependency` usage for local overlays in Cargo: + // https://github.com/rust-lang/cargo/blob/d6900d00af2644ea1c0068c5694d9dbe11a3ab39/src/cargo/sources/overlay.rs#L47 + for pkg in &packages { + base_config.set_package_registry_override( + pkg.clone(), + RegistryMapping::Registry(registry.clone()), + ); + } + + let client = CachingClient::new(Some(Client::new(base_config)), cache); + + let mut bytes_by_package = HashMap::new(); + for spec in plan.iter() { + let path = plan + .get_path(&spec.package) + .expect("PublishPlan guarantees a path for each iterated spec"); + let bytes = if path.is_dir() { + let (_pkg_ref, _version, bytes) = + build_wit_dir(path, client.clone(), lock_file).await?; + bytes + } else { + tokio::fs::read(path).await.with_context(|| { + format!("failed to read workspace member at {}", path.display()) + })? + }; + client + .client()? + .publish_release_data( + Box::pin(Cursor::new(bytes.clone())), + PublishOpts { + package: None, + registry: Some(registry.clone()), + dry_run: false, + skip_semver_check: false, + }, + ) + .await + .with_context(|| format!("verifier failed to publish: {}", spec.package))?; + if capture_bytes { + bytes_by_package.insert(spec.package.clone(), bytes); + } + } + + Ok(PublishVerifier { + client, + plan, + packages, + data: bytes_by_package, + _local_config: local_config, + }) + } +} diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 052596d..196ba93 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -1,4 +1,5 @@ //! Args and commands for interacting with WIT files and dependencies +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use anstream::eprintln; @@ -7,13 +8,21 @@ use clap::{Args, Subcommand}; use tempfile::NamedTempFile; use wasm_pkg_client::caching::{CachingClient, FileCache}; use wasm_pkg_common::package::{PackageRef, Version}; +use wasm_pkg_core::wit::WIT_DEPS_DIR; use wasm_pkg_core::{ - lock::LockFile, - manifest::Manifest, + lock::{LOCK_FILE_NAME, LockFile, LockedPackage}, + manifest::{MANIFEST_FILE_NAME, Manifest, workspace::WorkspaceRootConfig}, + resolver::DependencyResolutionMap, wit::{self, OutputType}, }; use crate::Common; +use crate::overlay::PublishVerifier; + +/// Registry name used as the overlay during workspace fetches. The same backend the publish +/// command uses for its dry-run staging - we shadow workspace-member packages here so peer +/// members can resolve each other locally without ever touching an upstream registry. +const WORKSPACE_OVERLAY_REGISTRY: &str = "tmp_local_fetch"; /// Commands for interacting with wit #[derive(Debug, Subcommand)] @@ -43,7 +52,7 @@ pub struct BuildArgs { pub dir: PathBuf, /// The name of the file that should be written. This can also be a full path. Defaults to the - /// current directory with the name of the package + /// current directory with the name of the package. #[clap(short = 'o', long = "output")] pub output: Option, @@ -60,8 +69,8 @@ pub struct BuildArgs { #[derive(Debug, Args)] pub struct FetchArgs { /// The directory containing the WIT files to fetch dependencies for. - #[clap(short = 'd', long = "wit-dir", default_value = "wit")] - pub dir: PathBuf, + /// Falls back to workspace manifest if empty. + pub dir: Option, /// The desired output type of the dependencies. Valid options are "wit" or "wasm" (wasm is the /// WIT package binary format). @@ -150,21 +159,167 @@ pub async fn temp_wit_file(package: &PackageRef, bytes: &[u8]) -> anyhow::Result impl FetchArgs { pub async fn run(self) -> anyhow::Result<()> { - check_dir(&self.dir).await?; - let client = self.common.get_client().await?; - let manifest = Manifest::load().await?; - let mut lock_file = LockFile::load(false).await?; - wit::fetch_dependencies( - &manifest, - self.dir, - &mut lock_file, - client, - self.output_type.unwrap_or_default(), - ) - .await?; - // Now write out the lock file since everything else succeeded - lock_file.write().await?; - Ok(()) + let cwd = std::env::current_dir()?; + let root = Manifest::load_root_workspace(&cwd).await?; + + let dirs = if let Some(dir) = self.dir.clone() { + vec![dir] + } else { + match root.as_ref() { + Some(root) => root.members.clone(), + None => vec![PathBuf::from("wit")], + } + }; + let config = match root.as_ref() { + Some(root) => { + let manifest_path = root.root_dir().join(MANIFEST_FILE_NAME); + Manifest::load_from_path(manifest_path).await? + } + None => Manifest::load().await?, + }; + let output = self.output_type.unwrap_or_default(); + + for dir in &dirs { + check_dir(dir).await?; + } + + match root { + Some(root) => run_workspace_fetch(&dirs, output, &config, root, &self.common).await, + None => run_simple_fetch(&dirs, output, &config, &self.common).await, + } + } +} + +async fn run_simple_fetch( + dirs: &[PathBuf], + output: OutputType, + config: &Manifest, + common: &Common, +) -> anyhow::Result<()> { + let client = common.get_client().await?; + let mut lock_file = LockFile::load(false).await?; + fetch_into_lock(dirs, config, client, output, &mut lock_file).await?; + lock_file.write().await?; + Ok(()) +} + +// fetch dependneces for a given workspace root, merging dependencies trees for included packages +async fn run_workspace_fetch( + dirs: &[PathBuf], + output: OutputType, + config: &Manifest, + root: WorkspaceRootConfig, + common: &Common, +) -> anyhow::Result<()> { + // Load/create the root lock file. This is the file that will be committed back to disk + let lock_path = root.root_dir().join(LOCK_FILE_NAME); + let mut lock_file = load_or_create_lock(&lock_path).await?; + + let verifier = PublishVerifier::try_new( + root.members.as_ref(), + WORKSPACE_OVERLAY_REGISTRY, + common.load_config().await?, + common.load_cache().await?, + &mut lock_file, + false, + ) + .await?; + + // Resolve dependencies for every requested member through the publish verifier + let mut merged: DependencyResolutionMap = DependencyResolutionMap::default(); + for dir in dirs { + let resolved = + wit::resolve_dependencies(config, dir, Some(&lock_file), verifier.client.clone()) + .await + .with_context(|| format!("failed to resolve dependencies for {}", dir.display()))?; + for (pkg, resolution) in resolved.as_ref() { + if verifier.packages.contains(pkg) { + continue; + } + merged.insert(pkg.clone(), resolution.clone()); + } + } + + lock_file.update_dependencies(&merged); + lock_file + .write() + .await + .with_context(|| format!("failed to commit lock file at {}", lock_path.display()))?; + + // 5. Ensure `/wkg/` exists and drop the aggregated deps into `/wkg/deps`. + // `populate_dependencies` canonicalizes its argument, so the parent must exist. + let out_dir = root.out_dir(); + tokio::fs::create_dir_all(&out_dir) + .await + .with_context(|| format!("failed to create {}", out_dir.display()))?; + wit::populate_dependencies_workspace(&out_dir, &merged, output) + .await + .with_context(|| { + format!( + "failed to populate workspace deps at {}", + out_dir.join(WIT_DEPS_DIR).display(), + ) + }) +} + +/// Iterate `dirs` and run `fetch_dependencies` for each, unioning each call's resolved +/// lock entries into a single set before assigning back. `fetch_dependencies` replaces +/// `lock_file.packages` on every call (it calls `update_dependencies` internally), so +/// we snapshot between calls to avoid losing earlier entries. +async fn fetch_into_lock( + dirs: &[PathBuf], + config: &Manifest, + client: CachingClient, + output: OutputType, + lock_file: &mut LockFile, +) -> anyhow::Result<()> { + let mut union: BTreeSet = BTreeSet::new(); + merge_locked_packages(&mut union, std::mem::take(&mut lock_file.packages)); + for dir in dirs { + wit::fetch_dependencies(config, dir, lock_file, client.clone(), output) + .await + .with_context(|| format!("failed to fetch dependencies for {}", dir.display()))?; + merge_locked_packages(&mut union, std::mem::take(&mut lock_file.packages)); + } + lock_file.packages = union; + Ok(()) +} + +/// Open `/wkg.lock` for read-write, creating it as an empty lockfile if missing. +/// Mirrors `LockFile::load`'s create-if-absent semantics but at an explicit path. +async fn load_or_create_lock(path: &Path) -> anyhow::Result { + if !tokio::fs::try_exists(path).await? { + let mut empty = LockFile::new_with_path([], path).await?; + empty.write().await?; + drop(empty); + } + LockFile::load_from_path(path, false) + .await + .with_context(|| format!("failed to load lock file at {}", path.display())) +} + +/// Merge `incoming` locked packages into `acc`, unioning per-package version +/// entries when the same `(name, registry)` key appears in both sets. Newer +/// version entries win on conflicting requirements within a package. +fn merge_locked_packages(acc: &mut BTreeSet, incoming: BTreeSet) { + for pkg in incoming { + if let Some(existing) = acc.take(&pkg) { + let mut merged = existing; + for v in pkg.versions { + if let Some(slot) = merged + .versions + .iter_mut() + .find(|e| e.requirement == v.requirement) + { + *slot = v; + } else { + merged.versions.push(v); + } + } + acc.insert(merged); + } else { + acc.insert(pkg); + } } } diff --git a/crates/wkg/tests/common.rs b/crates/wkg/tests/common.rs index 7315ad8..0aca4de 100644 --- a/crates/wkg/tests/common.rs +++ b/crates/wkg/tests/common.rs @@ -12,6 +12,7 @@ use testcontainers::{ }; use tokio::{net::TcpListener, process::Command}; use wasm_pkg_client::{Config, CustomConfig, Registry, RegistryMetadata, oci::OciRegistryConfig}; +use wasm_pkg_core::wit::WIT_DEPS_DIR; /// Returns an open port on localhost pub async fn find_open_port() -> u16 { @@ -145,7 +146,7 @@ pub async fn copy_dir( let filetype = entry.file_type().await?; if filetype.is_dir() { // Skip the deps directory in case it is there from debugging - if entry.path().file_name().unwrap_or_default() == "deps" { + if entry.path().file_name().unwrap_or_default() == WIT_DEPS_DIR { continue; } Box::pin(copy_dir( diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index fd17e51..d9ba89f 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -156,13 +156,85 @@ async fn publish_multiple_transitive_local_packages() { } } +#[cfg(feature = "docker-tests")] +#[tokio::test] +async fn publish_workspace_packages() { + use std::path::PathBuf; + + let (config, registry, _container) = common::start_registry().await; + let namespaces = ["example-a", "example-b", "example-c", "example-d"]; + + let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); + let src_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../wasm-pkg-core/tests/fixtures/transitive-local"); + let fixture_root = temp_dir.path().join("transitive-local"); + copy_dir(&src_root, &fixture_root).await.unwrap(); + + // The wkg.toml at fixture_root must have shipped over (it lives next to the example-* dirs). + assert!( + fixture_root.join("wkg.toml").exists(), + "fixture must include the workspace manifest copied from \ + crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml", + ); + + let mut mapped = config.clone(); + for ns in namespaces { + mapped = common::map_namespace(&mapped, ns, ®istry); + } + let config_path = temp_dir.path().join("config.toml"); + mapped.to_file(&config_path).await.expect("write config"); + + // `--workspace` expands to every `[workspace] members` entry from the root's `wkg.toml`. + let status = tokio::process::Command::new(env!("CARGO_BIN_EXE_wkg")) + .current_dir(&fixture_root) + .env("WKG_CACHE_DIR", temp_dir.path().join("cache")) + .env("WKG_CONFIG_FILE", &config_path) + .args(["publish", "--workspace"]) + .status() + .await + .expect("spawn wkg publish"); + assert!( + status.success(), + "wkg publish --workspace at workspace root should expand to all members and succeed", + ); + + let client = wasm_pkg_client::Client::new(mapped); + let expected_version = "0.1.0".parse::().unwrap(); + for name in [ + "example-a:foo", + "example-b:bar", + "example-c:baz", + "example-c:nested", + "example-d:foo", + ] { + let pkg = name.parse().unwrap(); + let versions = client + .list_all_versions(&pkg) + .await + .unwrap_or_else(|e| panic!("list versions for {name}: {e:#}")); + std::assert_matches!( + &versions[..], + [VersionInfo { version, .. }] if version == &expected_version, + "{name} should have exactly one published version", + ); + } +} + #[tokio::test] pub async fn check() { + // Use an explicit config that maps `wasi` to `wasi.dev`. + let mut config = wasm_pkg_client::Config::empty(); + config.set_namespace_registry( + "wasi".parse().unwrap(), + wasm_pkg_client::RegistryMapping::Registry("wasi.dev".parse().unwrap()), + ); + let fixture = common::load_fixture("wasi-http").await; let output = fixture.temp_dir.path().join("out"); let get = fixture - .command() + .command_with_config(&config) + .await .arg("get") .arg("wasi:http") .arg("--output") @@ -173,7 +245,8 @@ pub async fn check() { assert!(get.success()); let check_same = fixture - .command() + .command_with_config(&config) + .await .arg("get") .arg("--check") .arg("wasi:http") @@ -187,7 +260,8 @@ pub async fn check() { std::fs::write(&output, vec![1, 2, 3, 4]).expect("overwrite output with bogus contents"); let check_diff = fixture - .command() + .command_with_config(&config) + .await .arg("get") .arg("--check") .arg("wasi:http") From 02fcac39e90ec46d51ac0a4f7dd0be4bd50e6665 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 10 Jul 2026 14:29:04 -0500 Subject: [PATCH 02/11] fix: remove dist --- Cargo.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 23a0aff..e50fa51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,8 +69,3 @@ extend-ignore-words-re = [ [workspace.metadata.typos.default.extend-words] # strategy shorthand strat = "strat" - -# The profile that 'dist' will build with -[profile.dist] -inherits = "release" -lto = "thin" From 8ddfee379a4a50c0944420ac26d9f266e49279ff Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 08:38:58 -0500 Subject: [PATCH 03/11] revert: semver_check removal --- .../tests/publish_semver_check.rs | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/crates/wasm-pkg-client/tests/publish_semver_check.rs b/crates/wasm-pkg-client/tests/publish_semver_check.rs index ec5cd47..a1f3efe 100644 --- a/crates/wasm-pkg-client/tests/publish_semver_check.rs +++ b/crates/wasm-pkg-client/tests/publish_semver_check.rs @@ -222,3 +222,112 @@ async fn publish_semver_check( _ => panic!("expectation mismatch\n expected: {expected:?}\n actual: {result:?}",), } } + +// --------------------------------------------------------------------------- +// Multi-world packages (issue: semver_check assumed at most one world). +// +// A WIT package may declare more than one world. Semver checking must compare +// worlds pairwise by name: shared worlds must stay compatible, adding a world +// is additive (OK), and removing a world within the same compat range is +// breaking (strict policy). +// --------------------------------------------------------------------------- + +/// Which worlds a multi-world fixture declares, and the body of each. +struct MultiWorld { + /// `alpha` world body, if present. + alpha: Option, + /// `beta` world body, if present. + beta: Option, +} + +fn multiworld_wit(package: &str, version: &str, worlds: &MultiWorld) -> String { + let mut out = format!("\npackage {NAMESPACE}:{package}@{version};\n"); + if let Some(diff) = worlds.alpha { + out.push_str(&format!("\nworld alpha {{\n {diff}\n}}\n")); + } + if let Some(diff) = worlds.beta { + out.push_str(&format!("\nworld beta {{\n {diff}\n}}\n")); + } + out +} + +#[rstest] +// Both worlds compatible across a minor bump within a major -> OK. +#[case::multiworld_all_compatible( + "mw-all-compat", + ("1.2.0", MultiWorld { alpha: Some(WorldDiff::AddBase), beta: Some(WorldDiff::AddBase) }), + ("1.3.0", MultiWorld { alpha: Some(WorldDiff::AddExtra), beta: Some(WorldDiff::AddExtra) }), + Ok(()) +)] +// One shared world (beta) breaks across a minor bump -> incompatible. +#[case::multiworld_one_incompatible( + "mw-one-incompat", + ("1.2.0", MultiWorld { alpha: Some(WorldDiff::AddBase), beta: Some(WorldDiff::AddBase) }), + ("1.3.0", MultiWorld { alpha: Some(WorldDiff::AddExtra), beta: Some(WorldDiff::ChangeBase) }), + Err(semver_incompatible("1.2.0", "1.3.0")) +)] +// Adding a world across a minor bump is additive -> OK. +#[case::multiworld_added_world( + "mw-added", + ("1.2.0", MultiWorld { alpha: Some(WorldDiff::AddBase), beta: None }), + ("1.3.0", MultiWorld { alpha: Some(WorldDiff::AddExtra), beta: Some(WorldDiff::AddBase) }), + Ok(()) +)] +// Removing a world within the same major is breaking (strict policy) -> incompatible. +#[case::multiworld_removed_world_same_major( + "mw-removed", + ("1.2.0", MultiWorld { alpha: Some(WorldDiff::AddBase), beta: Some(WorldDiff::AddBase) }), + ("1.3.0", MultiWorld { alpha: Some(WorldDiff::AddExtra), beta: None }), + Err(semver_incompatible("1.2.0", "1.3.0")) +)] +// Removing a world across a major boundary is allowed: the versions fall in +// different compatibility ranges, so `semver_check` never compares them. +#[case::multiworld_removed_world_across_major( + "mw-removed-cross-major", + ("1.2.0", MultiWorld { alpha: Some(WorldDiff::AddBase), beta: Some(WorldDiff::AddBase) }), + ("2.0.0", MultiWorld { alpha: Some(WorldDiff::AddBase), beta: None }), + Ok(()) +)] +#[tokio::test] +async fn publish_semver_check_multiworld( + #[case] package: &str, + #[case] initial: (&str, MultiWorld), + #[case] candidate: (&str, MultiWorld), + #[case] expected: Result<(), Error>, +) { + let tmp = TempDir::new().unwrap(); + let client = make_client(tmp.path()); + + let (init_version, init_worlds) = initial; + publish( + &client, + wit_to_wasm(&multiworld_wit(package, init_version, &init_worlds)), + Default::default(), + ) + .await + .unwrap_or_else(|e| panic!("seeding {init_version} failed: {e:?}")); + + let (cand_version, cand_worlds) = candidate; + let result = publish( + &client, + wit_to_wasm(&multiworld_wit(package, cand_version, &cand_worlds)), + Default::default(), + ) + .await; + + match (&expected, &result) { + (Ok(()), Ok(())) => {} + ( + Err(Error::SemverIncompatible { + previous: exp_prev, + new: exp_new, + .. + }), + Err(Error::SemverIncompatible { previous, new, .. }), + ) => { + assert_eq!(previous, exp_prev, "previous version mismatch"); + assert_eq!(new, exp_new, "new version mismatch"); + } + _ => panic!("expectation mismatch\n expected: {expected:?}\n actual: {result:?}",), + } +} From 1367cc3135d34e0ce2b706bac14d19d8ac665815 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 09:00:55 -0500 Subject: [PATCH 04/11] refactor: inline fetch const --- .../wasm-pkg-core/src/manifest/workspace.rs | 5 --- crates/wkg/src/main.rs | 41 ++++++++++--------- crates/wkg/src/wit.rs | 15 +++---- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/crates/wasm-pkg-core/src/manifest/workspace.rs b/crates/wasm-pkg-core/src/manifest/workspace.rs index ba29d39..4f0b743 100644 --- a/crates/wasm-pkg-core/src/manifest/workspace.rs +++ b/crates/wasm-pkg-core/src/manifest/workspace.rs @@ -39,11 +39,6 @@ impl WorkspaceConfig { /// /// Knows the Workspace Root path, as well as `members` and `metadata`, which /// together tell if some path is recognized as a member by this root or not. -/// -/// `members` is stored as resolved, normalized paths: glob patterns from the -/// raw TOML are expanded, joined to `root_dir`, and normalized during -/// [`super::Manifest::load_from_path`]. Downstream code can read -/// `self.members` directly without re-resolving. #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] pub struct WorkspaceRootConfig { #[serde(default, skip)] diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 49370e4..c0ff439 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -425,26 +425,6 @@ impl PublishArgs { skip_semver_check: self.skip_semver_check, }) } - async fn workspace_root(&mut self) -> anyhow::Result> { - match self.workspace { - true if !self.paths.is_empty() => anyhow::bail!( - "`--workspace` selects every workspace member; do not also pass explicit \ - path arguments" - ), - - true => {} - false => return Ok(None), - } - let cwd = std::env::current_dir()?; - let Some(root) = Manifest::load_root_workspace(&cwd).await? else { - bail!( - "`--workspace` called but unable to find workspace root from {}", - cwd.display(), - ) - }; - self.paths = root.members.clone(); - Ok(Some(root)) - } fn handle_publish_result( &self, @@ -465,6 +445,27 @@ impl PublishArgs { } Ok(()) } + + async fn workspace_root(&mut self) -> anyhow::Result> { + match self.workspace { + true if !self.paths.is_empty() => anyhow::bail!( + "`--workspace` selects every workspace member; do not also pass explicit \ + path arguments" + ), + + true => {} + false => return Ok(None), + } + let cwd = std::env::current_dir()?; + let Some(root) = Manifest::load_root_workspace(&cwd).await? else { + bail!( + "`--workspace` called but unable to find workspace root from {}", + cwd.display(), + ) + }; + self.paths = root.members.clone(); + Ok(Some(root)) + } } #[derive(ValueEnum, Clone, Debug, PartialEq)] diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 196ba93..72b8c0f 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -19,11 +19,6 @@ use wasm_pkg_core::{ use crate::Common; use crate::overlay::PublishVerifier; -/// Registry name used as the overlay during workspace fetches. The same backend the publish -/// command uses for its dry-run staging - we shadow workspace-member packages here so peer -/// members can resolve each other locally without ever touching an upstream registry. -const WORKSPACE_OVERLAY_REGISTRY: &str = "tmp_local_fetch"; - /// Commands for interacting with wit #[derive(Debug, Subcommand)] pub enum WitCommands { @@ -217,7 +212,7 @@ async fn run_workspace_fetch( let verifier = PublishVerifier::try_new( root.members.as_ref(), - WORKSPACE_OVERLAY_REGISTRY, + "tmp_local_fetch", common.load_config().await?, common.load_cache().await?, &mut lock_file, @@ -262,10 +257,10 @@ async fn run_workspace_fetch( }) } -/// Iterate `dirs` and run `fetch_dependencies` for each, unioning each call's resolved -/// lock entries into a single set before assigning back. `fetch_dependencies` replaces -/// `lock_file.packages` on every call (it calls `update_dependencies` internally), so -/// we snapshot between calls to avoid losing earlier entries. +/// Iterate `dirs` and run [`wit::fetch_dependencies`] unioning each call's resolved lock entries into a +/// single set. +/// `fetch_dependencies` replaces [`LockFile`] packages on every call so we snapshot +/// between calls to avoid losing earlier entries. async fn fetch_into_lock( dirs: &[PathBuf], config: &Manifest, From b562b521fdf8374cc532434414bace3ca5d474ed Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 09:07:29 -0500 Subject: [PATCH 05/11] refactor(wkg,e2e): merge `publish_workspace_packages` into `publish_multiple_transitive_local_packages` --- crates/wkg/tests/e2e.rs | 70 +---------------------------------------- 1 file changed, 1 insertion(+), 69 deletions(-) diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index d9ba89f..136a481 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -89,73 +89,6 @@ async fn build_and_publish_with_metadata() { ); } -#[cfg(feature = "docker-tests")] -#[tokio::test] -async fn publish_multiple_transitive_local_packages() { - use std::path::PathBuf; - - let (config, registry, _container) = common::start_registry().await; - let namespaces = ["example-a", "example-b", "example-c", "example-d"]; - - // copy the transitive-local fixtures from wasm-pkg-core into a temp dir - let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); - let src_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../wasm-pkg-core/tests/fixtures/transitive-local"); - let fixture_root = temp_dir.path().join("transitive-local"); - copy_dir(&src_root, &fixture_root).await.unwrap(); - - let mut mapped = config.clone(); - for ns in namespaces { - mapped = common::map_namespace(&mapped, ns, ®istry); - } - let config_path = temp_dir.path().join("config.toml"); - mapped.to_file(&config_path).await.expect("write config"); - - // pass all fixture dirs to a single `wkg publish` invocation - let mut dirs: Vec = namespaces - .iter() - .map(|name| fixture_root.join(name).join("wit")) - .collect(); - // TODO use glob suchas in `wasm_pkgs_core::resolver::tests::transitive_local_paths` - dirs.push(fixture_root.join("example-c/wit/nested")); - - let mut publish = tokio::process::Command::new(env!("CARGO_BIN_EXE_wkg")); - publish - .current_dir(temp_dir.path()) - .env("WKG_CACHE_DIR", temp_dir.path().join("cache")) - .env("WKG_CONFIG_FILE", &config_path) - .arg("publish"); - for dir in &dirs { - publish.arg(dir); - } - let status = publish.status().await.expect("spawn wkg publish"); - assert!(status.success(), "wkg publish should succeed"); - - let client = wasm_pkg_client::Client::new(mapped); - - let expected_version = "0.1.0".parse::().unwrap(); - for name in [ - "example-a:foo", - "example-b:bar", - "example-c:baz", - "example-c:nested", - "example-d:foo", - ] { - let pkg = name.parse().unwrap(); - let versions = client - .list_all_versions(&pkg) - .await - .unwrap_or_else(|e| panic!("list versions for {name}: {e:#}")); - assert!( - matches!( - &versions[..], - [VersionInfo { version, .. }] if version == &expected_version, - ), - "{name} should have exactly one published version, got {versions:?}", - ); - } -} - #[cfg(feature = "docker-tests")] #[tokio::test] async fn publish_workspace_packages() { @@ -164,13 +97,13 @@ async fn publish_workspace_packages() { let (config, registry, _container) = common::start_registry().await; let namespaces = ["example-a", "example-b", "example-c", "example-d"]; + // copy the transitive-local fixtures from wasm-pkg-core into a temp dir let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); let src_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../wasm-pkg-core/tests/fixtures/transitive-local"); let fixture_root = temp_dir.path().join("transitive-local"); copy_dir(&src_root, &fixture_root).await.unwrap(); - // The wkg.toml at fixture_root must have shipped over (it lives next to the example-* dirs). assert!( fixture_root.join("wkg.toml").exists(), "fixture must include the workspace manifest copied from \ @@ -184,7 +117,6 @@ async fn publish_workspace_packages() { let config_path = temp_dir.path().join("config.toml"); mapped.to_file(&config_path).await.expect("write config"); - // `--workspace` expands to every `[workspace] members` entry from the root's `wkg.toml`. let status = tokio::process::Command::new(env!("CARGO_BIN_EXE_wkg")) .current_dir(&fixture_root) .env("WKG_CACHE_DIR", temp_dir.path().join("cache")) From 9410cb9ccc7953b9d479f0a52a5cc1f5f52d4893 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 09:44:25 -0500 Subject: [PATCH 06/11] feat: add fetch_workspace_packages e2e test --- crates/wasm-pkg-core/src/manifest.rs | 1 + crates/wkg/src/wit.rs | 181 +++++++++++++-------------- crates/wkg/tests/common.rs | 17 ++- crates/wkg/tests/e2e.rs | 62 ++++++--- 4 files changed, 148 insertions(+), 113 deletions(-) diff --git a/crates/wasm-pkg-core/src/manifest.rs b/crates/wasm-pkg-core/src/manifest.rs index 8ed7456..7b3a6ec 100644 --- a/crates/wasm-pkg-core/src/manifest.rs +++ b/crates/wasm-pkg-core/src/manifest.rs @@ -120,6 +120,7 @@ impl Manifest { /// Tries to find the root workspace config /// Returns `Ok(None)` when there is no `wkg.toml` ancestor that can be [`WorkspaceRootConfig`] + // TODO(maktychev): reconcile load_from_path and load_root_workspace pub async fn load_root_workspace(cwd: &Path) -> Result> { let Some(manifest_file) = find_root_manifest_for_wd(cwd) else { return Ok(None); diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 72b8c0f..38c7cdc 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -160,12 +160,11 @@ impl FetchArgs { let dirs = if let Some(dir) = self.dir.clone() { vec![dir] } else { - match root.as_ref() { - Some(root) => root.members.clone(), - None => vec![PathBuf::from("wit")], - } + root.as_ref() + .map(|r| r.members.clone()) + .unwrap_or_else(|| vec![PathBuf::from("wit")]) }; - let config = match root.as_ref() { + let manifest = match root.as_ref() { Some(root) => { let manifest_path = root.root_dir().join(MANIFEST_FILE_NAME); Manifest::load_from_path(manifest_path).await? @@ -179,105 +178,101 @@ impl FetchArgs { } match root { - Some(root) => run_workspace_fetch(&dirs, output, &config, root, &self.common).await, - None => run_simple_fetch(&dirs, output, &config, &self.common).await, + Some(root) => { + self.run_workspace_fetch(&dirs, output, &manifest, root) + .await + } + None => self.fetch_into_lock(&dirs, &manifest, output).await, } } -} -async fn run_simple_fetch( - dirs: &[PathBuf], - output: OutputType, - config: &Manifest, - common: &Common, -) -> anyhow::Result<()> { - let client = common.get_client().await?; - let mut lock_file = LockFile::load(false).await?; - fetch_into_lock(dirs, config, client, output, &mut lock_file).await?; - lock_file.write().await?; - Ok(()) -} + // fetch dependneces for a given workspace root, merging dependencies trees for included packages + async fn run_workspace_fetch( + &self, + dirs: &[PathBuf], + output: OutputType, + manifest: &Manifest, + root: WorkspaceRootConfig, + ) -> anyhow::Result<()> { + // Load/create the root lock file. This is the file that will be committed back to disk + let lock_path = root.root_dir().join(LOCK_FILE_NAME); + let mut lock_file = load_or_create_lock(&lock_path).await?; + + let verifier = PublishVerifier::try_new( + root.members.as_ref(), + "tmp_local_fetch", + self.common.load_config().await?, + self.common.load_cache().await?, + &mut lock_file, + false, + ) + .await?; -// fetch dependneces for a given workspace root, merging dependencies trees for included packages -async fn run_workspace_fetch( - dirs: &[PathBuf], - output: OutputType, - config: &Manifest, - root: WorkspaceRootConfig, - common: &Common, -) -> anyhow::Result<()> { - // Load/create the root lock file. This is the file that will be committed back to disk - let lock_path = root.root_dir().join(LOCK_FILE_NAME); - let mut lock_file = load_or_create_lock(&lock_path).await?; - - let verifier = PublishVerifier::try_new( - root.members.as_ref(), - "tmp_local_fetch", - common.load_config().await?, - common.load_cache().await?, - &mut lock_file, - false, - ) - .await?; - - // Resolve dependencies for every requested member through the publish verifier - let mut merged: DependencyResolutionMap = DependencyResolutionMap::default(); - for dir in dirs { - let resolved = - wit::resolve_dependencies(config, dir, Some(&lock_file), verifier.client.clone()) - .await - .with_context(|| format!("failed to resolve dependencies for {}", dir.display()))?; - for (pkg, resolution) in resolved.as_ref() { - if verifier.packages.contains(pkg) { - continue; + // Resolve dependencies for every requested member through the publish verifier + let mut merged = DependencyResolutionMap::default(); + for dir in dirs { + let resolved = + wit::resolve_dependencies(manifest, dir, Some(&lock_file), verifier.client.clone()) + .await + .with_context(|| { + format!("failed to resolve dependencies for {}", dir.display()) + })?; + for (pkg, resolution) in resolved.as_ref() { + if verifier.packages.contains(pkg) { + continue; + } + merged.insert(pkg.clone(), resolution.clone()); } - merged.insert(pkg.clone(), resolution.clone()); } - } - lock_file.update_dependencies(&merged); - lock_file - .write() - .await - .with_context(|| format!("failed to commit lock file at {}", lock_path.display()))?; - - // 5. Ensure `/wkg/` exists and drop the aggregated deps into `/wkg/deps`. - // `populate_dependencies` canonicalizes its argument, so the parent must exist. - let out_dir = root.out_dir(); - tokio::fs::create_dir_all(&out_dir) - .await - .with_context(|| format!("failed to create {}", out_dir.display()))?; - wit::populate_dependencies_workspace(&out_dir, &merged, output) - .await - .with_context(|| { - format!( - "failed to populate workspace deps at {}", - out_dir.join(WIT_DEPS_DIR).display(), - ) - }) -} + lock_file.update_dependencies(&merged); + lock_file + .write() + .await + .with_context(|| format!("failed to commit lock file at {}", lock_path.display()))?; -/// Iterate `dirs` and run [`wit::fetch_dependencies`] unioning each call's resolved lock entries into a -/// single set. -/// `fetch_dependencies` replaces [`LockFile`] packages on every call so we snapshot -/// between calls to avoid losing earlier entries. -async fn fetch_into_lock( - dirs: &[PathBuf], - config: &Manifest, - client: CachingClient, - output: OutputType, - lock_file: &mut LockFile, -) -> anyhow::Result<()> { - let mut union: BTreeSet = BTreeSet::new(); - merge_locked_packages(&mut union, std::mem::take(&mut lock_file.packages)); - for dir in dirs { - wit::fetch_dependencies(config, dir, lock_file, client.clone(), output) + // Ensure `/wkg/` exists and drop the aggregated deps into `/wkg/deps`. + // `populate_dependencies` canonicalizes its argument, so the parent must exist. + let out_dir = root.out_dir(); + tokio::fs::create_dir_all(&out_dir) .await - .with_context(|| format!("failed to fetch dependencies for {}", dir.display()))?; + .with_context(|| format!("failed to create {}", out_dir.display()))?; + wit::populate_dependencies_workspace(&out_dir, &merged, output) + .await + .with_context(|| { + format!( + "failed to populate workspace deps at {}", + out_dir.join(WIT_DEPS_DIR).display(), + ) + }) + } + + /// Iterate `dirs` and run [`wit::fetch_dependencies`] unioning each call's resolved lock entries into a + /// single set. + /// `fetch_dependencies` replaces [`LockFile`] packages on every call so we snapshot + /// between calls to avoid losing earlier entries. + async fn fetch_into_lock( + &self, + dirs: &[PathBuf], + manifest: &Manifest, + output: OutputType, + ) -> anyhow::Result<()> { + let client = self.common.get_client().await?; + let mut lock_file = LockFile::load(false).await?; + + let mut union: BTreeSet = BTreeSet::new(); merge_locked_packages(&mut union, std::mem::take(&mut lock_file.packages)); + for dir in dirs { + wit::fetch_dependencies(manifest, dir, &mut lock_file, client.clone(), output) + .await + .with_context(|| format!("failed to fetch dependencies for {}", dir.display()))?; + merge_locked_packages(&mut union, std::mem::take(&mut lock_file.packages)); + } + lock_file.packages = union; + + lock_file.write().await?; + Ok(()) } - lock_file.packages = union; - Ok(()) } /// Open `/wkg.lock` for read-write, creating it as an empty lockfile if missing. diff --git a/crates/wkg/tests/common.rs b/crates/wkg/tests/common.rs index 0aca4de..00694f1 100644 --- a/crates/wkg/tests/common.rs +++ b/crates/wkg/tests/common.rs @@ -116,18 +116,27 @@ pub fn fixture_dir() -> PathBuf { .join("fixtures") } +pub fn transitive_local_fixture() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../wasm-pkg-core/tests/fixtures/transitive-local") +} + /// Loads the fixture with the given name into a temporary directory. This will copy the fixture /// from the tests/fixtures directory into a temporary directory and return the tempdir containing /// that directory (and its path) pub async fn load_fixture(fixture: &str) -> Fixture { + load_fixture_from(fixture_dir().join(fixture)).await +} + +pub async fn load_fixture_from(src: impl AsRef) -> Fixture { + let src = src.as_ref(); let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); - let fixture_path = fixture_dir().join(fixture); // This will error if it doesn't exist, which is what we want - tokio::fs::metadata(&fixture_path) + tokio::fs::metadata(src) .await .expect("Fixture does not exist or couldn't be read"); - let copied_path = temp_dir.path().join(fixture_path.file_name().unwrap()); - copy_dir(&fixture_path, &copied_path) + let copied_path = temp_dir.path().join(src.file_name().unwrap()); + copy_dir(src, &copied_path) .await .expect("Failed to copy fixture"); Fixture { diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index 136a481..df675f5 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -1,6 +1,6 @@ use wasm_pkg_client::{Version, VersionInfo}; -use crate::common::copy_dir; +use crate::common::{load_fixture_from, transitive_local_fixture}; mod common; @@ -92,20 +92,13 @@ async fn build_and_publish_with_metadata() { #[cfg(feature = "docker-tests")] #[tokio::test] async fn publish_workspace_packages() { - use std::path::PathBuf; - let (config, registry, _container) = common::start_registry().await; let namespaces = ["example-a", "example-b", "example-c", "example-d"]; - // copy the transitive-local fixtures from wasm-pkg-core into a temp dir - let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); - let src_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../wasm-pkg-core/tests/fixtures/transitive-local"); - let fixture_root = temp_dir.path().join("transitive-local"); - copy_dir(&src_root, &fixture_root).await.unwrap(); + let fixture = load_fixture_from(transitive_local_fixture()).await; assert!( - fixture_root.join("wkg.toml").exists(), + fixture.fixture_path.join("wkg.toml").exists(), "fixture must include the workspace manifest copied from \ crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml", ); @@ -114,13 +107,10 @@ async fn publish_workspace_packages() { for ns in namespaces { mapped = common::map_namespace(&mapped, ns, ®istry); } - let config_path = temp_dir.path().join("config.toml"); - mapped.to_file(&config_path).await.expect("write config"); - let status = tokio::process::Command::new(env!("CARGO_BIN_EXE_wkg")) - .current_dir(&fixture_root) - .env("WKG_CACHE_DIR", temp_dir.path().join("cache")) - .env("WKG_CONFIG_FILE", &config_path) + let status = fixture + .command_with_config(&mapped) + .await .args(["publish", "--workspace"]) .status() .await @@ -152,6 +142,46 @@ async fn publish_workspace_packages() { } } +#[tokio::test] +async fn fetch_workspace_packages() { + let fixture = load_fixture_from(transitive_local_fixture()).await; + + // `transitive-local` is fully self-contained (every dep is a workspace member), + // so an empty config is enough — the PublishVerifier stands up a temp local + // backend to satisfy inter-member resolution. + let status = fixture + .command_with_config(&wasm_pkg_client::Config::empty()) + .await + .arg("fetch") + .status() + .await + .expect("spawn wkg fetch"); + assert!( + status.success(), + "wkg fetch at workspace root should expand to all members and succeed", + ); + + assert!( + fixture.fixture_path.join("wkg.lock").exists(), + "workspace fetch should create wkg.lock at the workspace root", + ); + let aggregated_deps = fixture.fixture_path.join("wkg/deps"); + assert!( + aggregated_deps.is_dir(), + "workspace fetch should create /wkg/deps/", + ); + + // All deps are workspace members and are filtered out of the aggregated + // map, so the deps directory should be created but empty. + let mut entries = tokio::fs::read_dir(&aggregated_deps) + .await + .expect("read aggregated deps dir"); + assert!( + entries.next_entry().await.unwrap().is_none(), + "aggregated deps should be empty when every dep is a workspace member", + ); +} + #[tokio::test] pub async fn check() { // Use an explicit config that maps `wasi` to `wasi.dev`. From 574197c82145fedccd3af051c4addcb7205dec06 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 10:01:31 -0500 Subject: [PATCH 07/11] chore: moved write_wasm_deps to be above populate_dependencies --- crates/wasm-pkg-core/src/wit.rs | 154 ++++++++++++++++---------------- crates/wkg/tests/e2e.rs | 21 ++--- 2 files changed, 82 insertions(+), 93 deletions(-) diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 7012e22..5fcf934 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -340,6 +340,83 @@ pub async fn populate_dependencies( write_wasm_deps(&deps_path, &deps.decode_dependencies().await?).await } +async fn prepare_deps_dir(path: &Path) -> Result { + // Canonicalizing will error if the path doesn't exist, so we don't need to check for that + let path = tokio::fs::canonicalize(path).await?; + if !tokio::fs::metadata(&path).await?.is_dir() { + anyhow::bail!("Path is not a directory"); + } + let deps_path = path.join(WIT_DEPS_DIR); + // Remove the whole directory if it already exists and then recreate + if let Err(e) = tokio::fs::remove_dir_all(&deps_path).await + && e.kind() != std::io::ErrorKind::NotFound + { + return Err(anyhow::anyhow!("Unable to remove deps directory: {e}") + .context(format!("dir: {}", deps_path.display()))); + } + tokio::fs::create_dir_all(&deps_path).await?; + Ok(deps_path) +} + +async fn write_wasm_deps( + deps_path: &Path, + decoded_deps: &IndexMap>, +) -> Result<()> { + for (name, dep) in decoded_deps.iter() { + let mut output_path = deps_path.join(name_from_package_name(name)); + + match dep { + DecodedDependency::Wit { + resolution: DependencyResolution::Local(local), + .. + } => { + // Local deps always need to be written to a subdirectory of deps so create that here + tokio::fs::create_dir_all(&output_path).await?; + write_local_dep(local, output_path).await?; + } + // This case shouldn't happen because registries only support wit packages. We can't get + // a resolve from the unresolved group, so error out here. Ideally we could print the + // unresolved group, but WitPrinter doesn't support that yet + DecodedDependency::Wit { + resolution: DependencyResolution::Registry(_), + .. + } => { + anyhow::bail!("Unable to resolve dependency, this is a programmer error"); + } + // Right now WIT packages include all of their dependencies, so we don't need to fetch + // those too. In the future, we'll need to look for unsatisfied dependencies and fetch + // them + DecodedDependency::Wasm { resolution, .. } => { + // This is going to be written to a single file, so we don't create a directory here + // NOTE(thomastaylor312): This janky looking thing is to avoid chopping off the + // patch number from the release. Once `add_extension` is stabilized, we can use + // that instead + let mut file_name = output_path.file_name().unwrap().to_owned(); + file_name.push(".wasm"); + output_path.set_file_name(file_name); + match resolution { + DependencyResolution::Local(local) => { + let meta = tokio::fs::metadata(&local.path).await?; + if !meta.is_file() { + anyhow::bail!("Local dependency is not single wit package file"); + } + tokio::fs::copy(&local.path, output_path) + .await + .context("Unable to copy local dependency")?; + } + DependencyResolution::Registry(registry) => { + let mut reader = registry.fetch().await?; + let mut output_file = tokio::fs::File::create(output_path).await?; + tokio::io::copy(&mut reader, &mut output_file).await?; + output_file.sync_all().await?; + } + } + } + } + } + Ok(()) +} + fn packages_from_foreign_deps( deps: impl IntoIterator, ) -> impl Iterator { @@ -469,83 +546,6 @@ pub async fn populate_dependencies_workspace( write_wasm_deps(&deps_path, &deps).await } -async fn prepare_deps_dir(path: &Path) -> Result { - // Canonicalizing will error if the path doesn't exist, so we don't need to check for that - let path = tokio::fs::canonicalize(path).await?; - if !tokio::fs::metadata(&path).await?.is_dir() { - anyhow::bail!("Path is not a directory"); - } - let deps_path = path.join(WIT_DEPS_DIR); - // Remove the whole directory if it already exists and then recreate - if let Err(e) = tokio::fs::remove_dir_all(&deps_path).await - && e.kind() != std::io::ErrorKind::NotFound - { - return Err(anyhow::anyhow!("Unable to remove deps directory: {e}") - .context(format!("dir: {}", deps_path.display()))); - } - tokio::fs::create_dir_all(&deps_path).await?; - Ok(deps_path) -} - -async fn write_wasm_deps( - deps_path: &Path, - decoded_deps: &IndexMap>, -) -> Result<()> { - for (name, dep) in decoded_deps.iter() { - let mut output_path = deps_path.join(name_from_package_name(name)); - - match dep { - DecodedDependency::Wit { - resolution: DependencyResolution::Local(local), - .. - } => { - // Local deps always need to be written to a subdirectory of deps so create that here - tokio::fs::create_dir_all(&output_path).await?; - write_local_dep(local, output_path).await?; - } - // This case shouldn't happen because registries only support wit packages. We can't get - // a resolve from the unresolved group, so error out here. Ideally we could print the - // unresolved group, but WitPrinter doesn't support that yet - DecodedDependency::Wit { - resolution: DependencyResolution::Registry(_), - .. - } => { - anyhow::bail!("Unable to resolve dependency, this is a programmer error"); - } - // Right now WIT packages include all of their dependencies, so we don't need to fetch - // those too. In the future, we'll need to look for unsatisfied dependencies and fetch - // them - DecodedDependency::Wasm { resolution, .. } => { - // This is going to be written to a single file, so we don't create a directory here - // NOTE(thomastaylor312): This janky looking thing is to avoid chopping off the - // patch number from the release. Once `add_extension` is stabilized, we can use - // that instead - let mut file_name = output_path.file_name().unwrap().to_owned(); - file_name.push(".wasm"); - output_path.set_file_name(file_name); - match resolution { - DependencyResolution::Local(local) => { - let meta = tokio::fs::metadata(&local.path).await?; - if !meta.is_file() { - anyhow::bail!("Local dependency is not single wit package file"); - } - tokio::fs::copy(&local.path, output_path) - .await - .context("Unable to copy local dependency")?; - } - DependencyResolution::Registry(registry) => { - let mut reader = registry.fetch().await?; - let mut output_file = tokio::fs::File::create(output_path).await?; - tokio::io::copy(&mut reader, &mut output_file).await?; - output_file.sync_all().await?; - } - } - } - } - } - Ok(()) -} - /// Given a package name, returns a valid directory/file name for it (thanks windows!) fn name_from_package_name(package_name: &PackageName) -> String { let package_name_str = package_name.to_string(); diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index df675f5..ca8a0f5 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -1,6 +1,8 @@ use wasm_pkg_client::{Version, VersionInfo}; use crate::common::{load_fixture_from, transitive_local_fixture}; +#[cfg(feature = "docker-tests")] +use crate::common::{map_transitive_local_namespaces, publish_transitive_local}; mod common; @@ -115,10 +117,7 @@ async fn publish_workspace_packages() { .status() .await .expect("spawn wkg publish"); - assert!( - status.success(), - "wkg publish --workspace at workspace root should expand to all members and succeed", - ); + assert!(status.success(), "`wkg publish --workspace` should succeed",); let client = wasm_pkg_client::Client::new(mapped); let expected_version = "0.1.0".parse::().unwrap(); @@ -146,9 +145,6 @@ async fn publish_workspace_packages() { async fn fetch_workspace_packages() { let fixture = load_fixture_from(transitive_local_fixture()).await; - // `transitive-local` is fully self-contained (every dep is a workspace member), - // so an empty config is enough — the PublishVerifier stands up a temp local - // backend to satisfy inter-member resolution. let status = fixture .command_with_config(&wasm_pkg_client::Config::empty()) .await @@ -156,10 +152,7 @@ async fn fetch_workspace_packages() { .status() .await .expect("spawn wkg fetch"); - assert!( - status.success(), - "wkg fetch at workspace root should expand to all members and succeed", - ); + assert!(status.success(), "`wkg fetch` in workspace should succeed",); assert!( fixture.fixture_path.join("wkg.lock").exists(), @@ -171,11 +164,7 @@ async fn fetch_workspace_packages() { "workspace fetch should create /wkg/deps/", ); - // All deps are workspace members and are filtered out of the aggregated - // map, so the deps directory should be created but empty. - let mut entries = tokio::fs::read_dir(&aggregated_deps) - .await - .expect("read aggregated deps dir"); + let mut entries = tokio::fs::read_dir(&aggregated_deps).await.unwrap(); assert!( entries.next_entry().await.unwrap().is_none(), "aggregated deps should be empty when every dep is a workspace member", From c00c592bff2b2407e799e81e5d525c1878e3575b Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 10:13:07 -0500 Subject: [PATCH 08/11] test(wkg): ensure e2e fetch lockfile contains union of dependencies --- crates/wkg/tests/common.rs | 26 +++++++ crates/wkg/tests/e2e.rs | 71 ++++++++++--------- .../fetch-workspace/fetch-a/wit/world.wit | 5 ++ .../fetch-workspace/fetch-d/wit/world.wit | 5 ++ .../tests/fixtures/fetch-workspace/wkg.toml | 2 + 5 files changed, 76 insertions(+), 33 deletions(-) create mode 100644 crates/wkg/tests/fixtures/fetch-workspace/fetch-a/wit/world.wit create mode 100644 crates/wkg/tests/fixtures/fetch-workspace/fetch-d/wit/world.wit create mode 100644 crates/wkg/tests/fixtures/fetch-workspace/wkg.toml diff --git a/crates/wkg/tests/common.rs b/crates/wkg/tests/common.rs index 00694f1..24d92dd 100644 --- a/crates/wkg/tests/common.rs +++ b/crates/wkg/tests/common.rs @@ -61,6 +61,32 @@ pub async fn start_registry() -> (Config, Registry, ContainerAsync (config, registry, container) } +pub const TRANSITIVE_LOCAL_NAMESPACES: &[&str] = + &["example-a", "example-b", "example-c", "example-d"]; + +/// Maps every namespace in [`TRANSITIVE_LOCAL_NAMESPACES`] to `registry`. +pub fn map_transitive_local_namespaces(config: &Config, registry: &Registry) -> Config { + let mut mapped = config.clone(); + for ns in TRANSITIVE_LOCAL_NAMESPACES { + mapped = map_namespace(&mapped, ns, registry); + } + mapped +} + +/// runs `wkg publish --workspace` for [`TRANSITIVE_LOCAL_NAMESPACES`] packages +pub async fn publish_transitive_local(config: &Config) -> Fixture { + let fixture = load_fixture_from(transitive_local_fixture()).await; + let status = fixture + .command_with_config(config) + .await + .args(["publish", "--workspace"]) + .status() + .await + .expect("spawn wkg publish"); + assert!(status.success(), "`wkg publish --workspace` should succeed",); + fixture +} + /// Clones the given config, mapping the namespace to the given registry at the top level pub fn map_namespace(config: &Config, namespace: &str, registry: &Registry) -> Config { let mut config = config.clone(); diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index ca8a0f5..6fbcfb6 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -1,6 +1,5 @@ use wasm_pkg_client::{Version, VersionInfo}; -use crate::common::{load_fixture_from, transitive_local_fixture}; #[cfg(feature = "docker-tests")] use crate::common::{map_transitive_local_namespaces, publish_transitive_local}; @@ -95,9 +94,9 @@ async fn build_and_publish_with_metadata() { #[tokio::test] async fn publish_workspace_packages() { let (config, registry, _container) = common::start_registry().await; - let namespaces = ["example-a", "example-b", "example-c", "example-d"]; - let fixture = load_fixture_from(transitive_local_fixture()).await; + let mapped = map_transitive_local_namespaces(&config, ®istry); + let fixture = publish_transitive_local(&mapped).await; assert!( fixture.fixture_path.join("wkg.toml").exists(), @@ -105,20 +104,6 @@ async fn publish_workspace_packages() { crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml", ); - let mut mapped = config.clone(); - for ns in namespaces { - mapped = common::map_namespace(&mapped, ns, ®istry); - } - - let status = fixture - .command_with_config(&mapped) - .await - .args(["publish", "--workspace"]) - .status() - .await - .expect("spawn wkg publish"); - assert!(status.success(), "`wkg publish --workspace` should succeed",); - let client = wasm_pkg_client::Client::new(mapped); let expected_version = "0.1.0".parse::().unwrap(); for name in [ @@ -141,34 +126,54 @@ async fn publish_workspace_packages() { } } +#[cfg(feature = "docker-tests")] #[tokio::test] async fn fetch_workspace_packages() { - let fixture = load_fixture_from(transitive_local_fixture()).await; + use wasm_pkg_core::lock::{LOCK_FILE_NAME, LockFile}; + + let (config, registry, _container) = common::start_registry().await; + let mapped = map_transitive_local_namespaces(&config, ®istry); + let _publisher = publish_transitive_local(&mapped).await; + let fixture = common::load_fixture("fetch-workspace").await; let status = fixture - .command_with_config(&wasm_pkg_client::Config::empty()) + .command_with_config(&mapped) .await .arg("fetch") .status() .await .expect("spawn wkg fetch"); - assert!(status.success(), "`wkg fetch` in workspace should succeed",); - assert!( - fixture.fixture_path.join("wkg.lock").exists(), - "workspace fetch should create wkg.lock at the workspace root", - ); - let aggregated_deps = fixture.fixture_path.join("wkg/deps"); - assert!( - aggregated_deps.is_dir(), - "workspace fetch should create /wkg/deps/", + status.success(), + "`wkg fetch` in fetch-workspace should succeed" ); - let mut entries = tokio::fs::read_dir(&aggregated_deps).await.unwrap(); - assert!( - entries.next_entry().await.unwrap().is_none(), - "aggregated deps should be empty when every dep is a workspace member", - ); + let lock_path = fixture.fixture_path.join(LOCK_FILE_NAME); + assert!(lock_path.exists(), "`wkg fetch` should create wkg.lock",); + let lock = LockFile::load_from_path(&lock_path, true) + .await + .expect("load fetch-workspace wkg.lock"); + let expected_version = "0.1.0".parse::().unwrap(); + for expected_pkgs in ["example-a:foo", "example-d:foo"] { + let pkg = expected_pkgs.parse().unwrap(); + let entry = lock + .packages + .iter() + .find(|p| p.name == pkg) + .unwrap_or_else(|| { + panic!( + "wkg.lock should contain {expected_pkgs}; had: {:?}", + lock.packages + .iter() + .map(|p| p.name.to_string()) + .collect::>() + ) + }); + assert!( + entry.versions.iter().any(|v| v.version == expected_version), + "{expected_pkgs} should be locked at {expected_version}; entry: {entry:?}", + ); + } } #[tokio::test] diff --git a/crates/wkg/tests/fixtures/fetch-workspace/fetch-a/wit/world.wit b/crates/wkg/tests/fixtures/fetch-workspace/fetch-a/wit/world.wit new file mode 100644 index 0000000..1c7b007 --- /dev/null +++ b/crates/wkg/tests/fixtures/fetch-workspace/fetch-a/wit/world.wit @@ -0,0 +1,5 @@ +package fetch:a@0.1.0; + +world app { + import example-a:foo/foo@0.1.0; +} diff --git a/crates/wkg/tests/fixtures/fetch-workspace/fetch-d/wit/world.wit b/crates/wkg/tests/fixtures/fetch-workspace/fetch-d/wit/world.wit new file mode 100644 index 0000000..9bb1cba --- /dev/null +++ b/crates/wkg/tests/fixtures/fetch-workspace/fetch-d/wit/world.wit @@ -0,0 +1,5 @@ +package fetch:d@0.1.0; + +world app { + import example-d:foo/bar@0.1.0; +} diff --git a/crates/wkg/tests/fixtures/fetch-workspace/wkg.toml b/crates/wkg/tests/fixtures/fetch-workspace/wkg.toml new file mode 100644 index 0000000..91c3cd9 --- /dev/null +++ b/crates/wkg/tests/fixtures/fetch-workspace/wkg.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["fetch-*/wit"] From 515d9f8703d0b8c8f2823a56190b4b74116a2c90 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 10:14:04 -0500 Subject: [PATCH 09/11] chore: clippy --- crates/wasm-pkg-core/src/manifest.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/wasm-pkg-core/src/manifest.rs b/crates/wasm-pkg-core/src/manifest.rs index 7b3a6ec..cb967e3 100644 --- a/crates/wasm-pkg-core/src/manifest.rs +++ b/crates/wasm-pkg-core/src/manifest.rs @@ -72,7 +72,7 @@ impl Manifest { fn root(&self) -> Option<&WorkspaceRootConfig> { if let Some(WorkspaceConfig::Root(root)) = &self.workspace { - return Some(&root); + return Some(root); } None } @@ -135,11 +135,10 @@ impl Manifest { // keep walking up if we have not found root for file in find_root_iter(&manifest_file) { let manifest = Self::load_from_path(&file).await?; - if let Some(WorkspaceConfig::Root(root)) = manifest.workspace { - if root.is_explicitly_listed_member(&manifest_dir) { + if let Some(WorkspaceConfig::Root(root)) = manifest.workspace + && root.is_explicitly_listed_member(manifest_dir) { return Ok(Some(root)); } - } } Ok(None) From befba6c0a3c3a765dbbf603a14c9bf7eb3bd4d80 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 10:20:47 -0500 Subject: [PATCH 10/11] fix(core): patch race condition in Manifest::write --- crates/wasm-pkg-core/src/manifest.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/wasm-pkg-core/src/manifest.rs b/crates/wasm-pkg-core/src/manifest.rs index cb967e3..4d09007 100644 --- a/crates/wasm-pkg-core/src/manifest.rs +++ b/crates/wasm-pkg-core/src/manifest.rs @@ -8,8 +8,6 @@ use std::{ use anyhow::{Context, Result}; use semver::VersionReq; use serde::{Deserialize, Serialize}; -use tokio::io::AsyncWriteExt; - mod paths; pub mod workspace; @@ -147,8 +145,7 @@ impl Manifest { /// Serializes and writes the manifest to the given path. pub async fn write(&self, path: impl AsRef) -> Result<()> { let contents = toml::to_string_pretty(self)?; - let mut file = tokio::fs::File::create(path).await?; - file.write_all(contents.as_bytes()) + tokio::fs::write(path, contents) .await .context("unable to write manifest to path") } From d671bac8db6a6c752e1ca59e32587138940b73d2 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Sat, 11 Jul 2026 10:26:06 -0500 Subject: [PATCH 11/11] chore: cargo-autoinherit --- crates/wasm-pkg-core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasm-pkg-core/Cargo.toml b/crates/wasm-pkg-core/Cargo.toml index 5e03473..e1b2fea 100644 --- a/crates/wasm-pkg-core/Cargo.toml +++ b/crates/wasm-pkg-core/Cargo.toml @@ -23,7 +23,7 @@ wasm-pkg-client = { workspace = true } wit-component = { workspace = true } wit-parser = { workspace = true } petgraph = { workspace = true } -glob = "0.3.3" +glob = { workspace = true } [target.'cfg(unix)'.dependencies.libc] version = "0.2"