diff --git a/crates/wasm-pkg-core/Cargo.toml b/crates/wasm-pkg-core/Cargo.toml index b7f085e..e1b2fea 100644 --- a/crates/wasm-pkg-core/Cargo.toml +++ b/crates/wasm-pkg-core/Cargo.toml @@ -23,6 +23,7 @@ wasm-pkg-client = { workspace = true } wit-component = { workspace = true } wit-parser = { workspace = true } petgraph = { workspace = true } +glob = { workspace = true } [target.'cfg(unix)'.dependencies.libc] version = "0.2" diff --git a/crates/wasm-pkg-core/src/lock.rs b/crates/wasm-pkg-core/src/lock.rs index 441bbcd..143bef6 100644 --- a/crates/wasm-pkg-core/src/lock.rs +++ b/crates/wasm-pkg-core/src/lock.rs @@ -373,7 +373,6 @@ impl AsRef for Locker { // NOTE(thomastaylor312): These lock file primitives from here on down are mostly copied wholesale // from the lock file implementation of cargo-component with some minor modifications to make them // work with tokio - impl Locker { // NOTE(thomastaylor312): I am keeping around these try methods for possible later use. Right // now we're ignoring the dead code diff --git a/crates/wasm-pkg-core/src/manifest.rs b/crates/wasm-pkg-core/src/manifest.rs index 3acd335..4d09007 100644 --- a/crates/wasm-pkg-core/src/manifest.rs +++ b/crates/wasm-pkg-core/src/manifest.rs @@ -8,16 +8,28 @@ use std::{ use anyhow::{Context, Result}; use semver::VersionReq; use serde::{Deserialize, Serialize}; -use tokio::io::AsyncWriteExt; +mod paths; +pub mod workspace; + +use workspace::*; + +use crate::manifest::paths::{find_root_iter, find_root_manifest_for_wd}; /// The default name of the manifest file. pub const MANIFEST_FILE_NAME: &str = "wkg.toml"; +/// Directory next to the root [`MANIFEST_FILE_NAME`] that holds multi-package `deps` and `config.toml`. +pub const WORKSPACE_OUT_DIR: &str = "wkg"; /// The structure for a wkg.toml manifest file. This file is entirely optional and is used for /// overriding and annotating wasm packages. +/// `workspace` is mutually exclusive with `overrides` and top-level `metadata` #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct Manifest { + /// Workspace declaration. + // TODO: this should be a `TomlWorkspace` so that serialization is not coupled to the config structure + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, /// Overrides for various packages #[serde(default, skip_serializing_if = "Option::is_none")] pub overrides: Option>, @@ -28,17 +40,70 @@ pub struct Manifest { } impl Manifest { + fn from_toml(contents: &str) -> Result { + let manifest: Manifest = + toml::from_str(contents).context("unable to parse manifest file")?; + manifest.validate()?; + Ok(manifest) + } + /// Loads a manifest file from the given path. pub async fn load_from_path(path: impl AsRef) -> Result { - tracing::info!(path = %path.as_ref().display(), "loading wkg manifest file"); - let contents = tokio::fs::read_to_string(path) - .await - .context("unable to load manifest from file")?; - let manifest: Manifest = - toml::from_str(&contents).context("unable to parse manifest file")?; + let path = path.as_ref(); + tracing::info!(path = %path.display(), "loading wkg manifest file"); + let contents = std::fs::read_to_string(path) + .with_context(|| format!("unable to load manifest from {}", path.display()))?; + let mut manifest = Self::from_toml(&contents) + .with_context(|| format!("invalid manifest at {}", path.display()))?; + if let Some(WorkspaceConfig::Root(root)) = &mut manifest.workspace { + root.root_dir = path + .parent() + .with_context(|| { + format!("manifest path has no parent directory: {}", path.display()) + })? + .to_path_buf(); + // Resolve globs and relative paths eagerly + root.members = WorkspaceRootConfig::resolve_members(&root.members, &root.root_dir); + } Ok(manifest) } + fn root(&self) -> Option<&WorkspaceRootConfig> { + if let Some(WorkspaceConfig::Root(root)) = &self.workspace { + return Some(root); + } + None + } + + // `Manifest` validations, mirrors cargo's `Workspace::validate` + fn validate(&self) -> Result<()> { + self.validate_workspace_exclusivity()?; + // Add new validation rules with `self.validate_*()?;` + Ok(()) + } + + // no overrides or top-level metadata when workspace is present + fn validate_workspace_exclusivity(&self) -> Result<()> { + if self.workspace.is_none() { + return Ok(()); + } + let mut conflicts = Vec::new(); + if self.overrides.is_some() { + conflicts.push("overrides"); + } + if self.metadata.is_some() { + conflicts.push("metadata"); + } + if conflicts.is_empty() { + return Ok(()); + } + anyhow::bail!( + "`[workspace]` cannot coexist with: `[{}]` - \ + use `[workspace.metadata]` for workspace level values", + conflicts.join("]`, `[") + ); + } + /// Attempts to load the manifest from the current directory. Most of the time, users of this /// crate should use this function. Right now it just checks for a `wkg.toml` file in the current /// directory, but we could add more resolution logic in the future. If the file is not found, a @@ -51,11 +116,36 @@ impl Manifest { Self::load_from_path(manifest_path).await } + /// Tries to find the root workspace config + /// Returns `Ok(None)` when there is no `wkg.toml` ancestor that can be [`WorkspaceRootConfig`] + // TODO(maktychev): reconcile load_from_path and load_root_workspace + pub async fn load_root_workspace(cwd: &Path) -> Result> { + let Some(manifest_file) = find_root_manifest_for_wd(cwd) else { + return Ok(None); + }; + let manifest_dir = manifest_file.parent().unwrap(); + let manifest = Self::load_from_path(&manifest_file).await?; + + if let Some(root) = manifest.root() { + return Ok(Some(root.clone())); + } + + // keep walking up if we have not found root + for file in find_root_iter(&manifest_file) { + let manifest = Self::load_from_path(&file).await?; + if let Some(WorkspaceConfig::Root(root)) = manifest.workspace + && root.is_explicitly_listed_member(manifest_dir) { + return Ok(Some(root)); + } + } + + Ok(None) + } + /// Serializes and writes the manifest to the given path. pub async fn write(&self, path: impl AsRef) -> Result<()> { let contents = toml::to_string_pretty(self)?; - let mut file = tokio::fs::File::create(path).await?; - file.write_all(contents.as_bytes()) + tokio::fs::write(path, contents) .await .context("unable to write manifest to path") } @@ -117,6 +207,7 @@ mod tests { let tempdir = tempfile::tempdir().unwrap(); let manifest_path = tempdir.path().join(MANIFEST_FILE_NAME); let manifest = Manifest { + workspace: None, overrides: Some(HashMap::from([( "foo:bar".to_string(), Override { diff --git a/crates/wasm-pkg-core/src/manifest/paths.rs b/crates/wasm-pkg-core/src/manifest/paths.rs new file mode 100644 index 0000000..6bcf8d0 --- /dev/null +++ b/crates/wasm-pkg-core/src/manifest/paths.rs @@ -0,0 +1,102 @@ +use anyhow::Result; +use std::path::{Component, Path, PathBuf}; + +use crate::manifest::MANIFEST_FILE_NAME; + +/// Find the first ancestor [`super::Manifest`] path for current working directory. +pub(crate) fn find_root_manifest_for_wd(cwd: impl AsRef) -> Option { + for current in cwd.as_ref().ancestors() { + let manifest = current.join(MANIFEST_FILE_NAME); + if manifest.exists() { + return Some(manifest); + } + } + None +} + +// Returns first ancestor [`super::Manifest`] path for a given existing manifest +pub(crate) fn find_root_iter<'a>(manifest_path: &'a Path) -> impl Iterator + 'a { + manifest_path + .ancestors() + .skip(2) // skip `manifest_path` and the parent dir + .map(|dir| dir.join(MANIFEST_FILE_NAME)) + .filter(|path| path.exists()) +} + +// copied from: +// https://github.com/rust-lang/cargo/blob/a595d0da21f228b7fdae64d3d5c0e527ea66bb59/crates/cargo-util/src/paths.rs#L84-L84 +/// Normalize a path, removing things like `.` and `..`. +/// +/// CAUTION: This does not resolve symlinks (unlike +/// [`std::fs::canonicalize`]). This may cause incorrect or surprising +/// behavior at times. This should be used carefully. Unfortunately, +/// [`std::fs::canonicalize`] can be hard to use correctly, since it can often +/// fail, or on Windows returns annoying device paths. This is a problem Cargo +/// needs to improve on. +pub(crate) fn normalize_path(path: &Path) -> PathBuf { + let mut components = path.components().peekable(); + let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() { + components.next(); + PathBuf::from(c.as_os_str()) + } else { + PathBuf::new() + }; + + for component in components { + match component { + Component::Prefix(..) => unreachable!(), + Component::RootDir => { + ret.push(Component::RootDir); + } + Component::CurDir => {} + Component::ParentDir => { + if ret.ends_with(Component::ParentDir) { + ret.push(Component::ParentDir); + } else { + let popped = ret.pop(); + if !popped && !ret.has_root() { + ret.push(Component::ParentDir); + } + } + } + Component::Normal(c) => { + ret.push(c); + } + } + } + ret +} + +// Check if path holds at least one `.wit` file. +fn dir_contains_wit(dir: &Path) -> bool { + let Ok(entries) = std::fs::read_dir(dir) else { + return false; + }; + entries.filter_map(Result::ok).any(|entry| { + entry.path().extension().is_some_and(|ext| ext == "wit") + && entry.file_type().is_ok_and(|ft| ft.is_file()) + }) +} + +// Expand a string if is determined to be a glob pattern +pub(crate) fn expand_globs(entry: &Path, root_dir: &Path) -> Result> { + let joined = root_dir.join(entry); + // TODO(mkatychev): handle escaping glob chars + let is_glob = entry.to_str().is_some_and(|s| s.contains(['*', '?', '['])); + let pattern = joined.to_str().filter(|_| is_glob); + let Some(pattern) = pattern else { + return Ok(vec![joined]); + }; + let matches = glob::glob(pattern)?; + let mut expanded: Vec = matches + .filter_map(Result::ok) + .filter(|p| p.is_dir() && dir_contains_wit(p)) + .collect(); + + if expanded.is_empty() { + return Ok(vec![joined]); + } + + expanded.sort(); + Ok(expanded) +} diff --git a/crates/wasm-pkg-core/src/manifest/workspace.rs b/crates/wasm-pkg-core/src/manifest/workspace.rs new file mode 100644 index 0000000..4f0b743 --- /dev/null +++ b/crates/wasm-pkg-core/src/manifest/workspace.rs @@ -0,0 +1,306 @@ +use crate::manifest::paths::{expand_globs, normalize_path}; + +use super::*; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// `[workspace]` table inside a `wkg.toml` manifest. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum WorkspaceConfig { + // `[workspace]` + Root(WorkspaceRootConfig), + // - `workspace = "path/to/workspace/root"` + // - TODO(mkatychev): implement workspace package level field + // - https://doc.rust-lang.org/cargo/reference/manifest.html#the-workspace-field + // Member { + // root: Option, + // }, +} + +impl Default for WorkspaceConfig { + fn default() -> Self { + // TODO(mkatychev): use member variant instead + Self::Root(WorkspaceRootConfig::default()) + } +} + +impl WorkspaceConfig { + /// Resolves the [`WorkspaceRootConfig`] for a given [`Self`] + pub fn as_root(&self) -> Option<&WorkspaceRootConfig> { + match self { + WorkspaceConfig::Root(r) => Some(r), + } + } +} + +/// Intermediate configuration of a workspace root in a manifest. +/// +/// Knows the Workspace Root path, as well as `members` and `metadata`, which +/// together tell if some path is recognized as a member by this root or not. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct WorkspaceRootConfig { + #[serde(default, skip)] + pub(super) root_dir: PathBuf, + pub members: Vec, + pub metadata: Option, +} + +impl WorkspaceRootConfig { + /// Directory containing the workspace-root `wkg.toml`. + pub fn root_dir(&self) -> &Path { + &self.root_dir + } + + /// Directory containing workspace level `deps` `and config.toml` + pub fn out_dir(&self) -> PathBuf { + self.root_dir.join(PathBuf::from(WORKSPACE_OUT_DIR)) + } + + /// Checks if the path is explicitly listed as a workspace member. + /// + /// Returns `true` ONLY if: + /// - The path is the workspace root manifest itself, or + /// - The path matches one of the explicit `members` patterns + /// + /// NOTE: This does NOT check for implicit path dependency membership. + /// A `false` return does NOT mean the package is definitely not a member - + /// it could still be a member via path dependencies. Callers should fallback + /// to full workspace loading when this returns `false`. + // FIXME: implement cargo WorkspaceRootConfig::{member_paths, expand_member_path} + // this should not be eagerly evaluated + pub(super) fn is_explicitly_listed_member(&self, manifest_path: &Path) -> bool { + let root_manifest = self.root_dir.join("Cargo.toml"); + if manifest_path == root_manifest { + return true; + } + + let manifest_path = normalize_path(manifest_path); + self.members.iter().any(|member| member == &manifest_path) + } + // Expand globs and relative paths + pub(super) fn resolve_members(members: &[PathBuf], root_dir: &Path) -> Vec { + members + .iter() + .flat_map(|entry| expand_globs(entry, root_dir)) + .flatten() + .map(|p| normalize_path(&p)) + .collect() + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use super::*; + + // NOTE: tree dir comments generated using `eza -T dir/` + fn touch(path: impl AsRef) { + let path = path.as_ref(); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "").unwrap(); + } + + #[test] + fn find_root_manifest_for_wd_first_ancestor() { + // dir + // ├── wkg.toml + // └── pkg + // └── wkg.toml + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + touch(root_dir.join("wkg.toml")); + touch(root_dir.join("pkg/wkg.toml")); + fs::create_dir_all(root_dir.join("pkg/wit")).unwrap(); + + let found = find_root_manifest_for_wd(root_dir.join("pkg/wit")).unwrap(); + assert_eq!(found, root_dir.join("pkg/wkg.toml")); + } + + #[test] + fn find_root_manifest_for_wd_none() { + let tempdir = tempfile::tempdir().unwrap(); + assert!(find_root_manifest_for_wd(tempdir.path()).is_none()); + } + + #[test] + fn find_root_iter_ancestors() { + // dir + // ├── wkg.toml + // └── pkg + // └── wkg.toml + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + touch(root_dir.join("wkg.toml")); + let member = root_dir.join("pkg/wkg.toml"); + touch(&member); + + let roots: Vec = find_root_iter(&member).collect(); + assert_eq!(roots, vec![root_dir.join("wkg.toml")]); + } + + #[test] + fn resolve_members_skip_non_dirs() { + // dir + // ├── example-a + // │ └── wit + // │ └── pkg.wit + // ├── example-b + // │ └── wit + // │ └── pkg.wit + // ├── example-c + // │ └── wit + // │ └── pkg.wit + // └── example-d + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + let member_wit = ["example-a/wit", "example-b/wit", "example-c/wit"]; + for dir in member_wit { + fs::create_dir_all(root_dir.join(dir)).unwrap(); + fs::write(root_dir.join(dir).join("pkg.wit"), "").unwrap(); + } + fs::write(root_dir.join("example-d"), "").unwrap(); + + let members = vec![PathBuf::from("example-*/wit")]; + let mut result = WorkspaceRootConfig::resolve_members(&members, root_dir); + result.sort(); + let mut expected: Vec<_> = member_wit.iter().map(|p| root_dir.join(p)).collect(); + expected.sort(); + assert_eq!(result, expected, "glob should expand to matching dirs only"); + } + + // skip directories without `.wit` files + #[test] + fn resolve_members_skip_non_wit_dirs() { + // dir + // └── pkg + // ├── core + // │ └── types.wit + // └── filesystem + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + let core = root_dir.join("pkg/core"); + let filesystem = root_dir.join("pkg/filesystem"); + fs::create_dir_all(&core).unwrap(); + fs::write(core.join("types.wit"), "package foo:core;\n").unwrap(); + fs::create_dir_all(filesystem).unwrap(); + + let members = vec![PathBuf::from("pkg/*")]; + let result = WorkspaceRootConfig::resolve_members(&members, root_dir); + assert_eq!(result, vec![core]); + } + + #[test] + fn resolved_members_keeps_literal_even_when_empty() { + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + // dir + // └── empty-pkg + fs::create_dir_all(root_dir.join("empty-pkg")).unwrap(); + + let members = vec![PathBuf::from("empty-pkg")]; + let result = WorkspaceRootConfig::resolve_members(&members, root_dir); + assert_eq!(result, vec![root_dir.join("empty-pkg")]); + } + + #[test] + fn workspace_root() { + let toml = r#" +[workspace] +members = ["a/wit", "b/wit"] +"#; + let cfg = Manifest::from_toml(toml).unwrap(); + let root = cfg.workspace.unwrap(); + assert_eq!( + root.as_root().unwrap().members, + vec![PathBuf::from("a/wit"), PathBuf::from("b/wit")], + ); + assert!(cfg.overrides.is_none() && cfg.metadata.is_none()); + } + + #[test] + fn workspace_metadata() { + let toml = r#" +[workspace] +members = ["a/wit"] + +[workspace.metadata] +authors = "Webster Assembler" +"#; + let cfg = Manifest::from_toml(toml).unwrap(); + let root = cfg.root().unwrap(); + let meta = root.metadata.as_ref().unwrap(); + assert_eq!(meta.authors.as_ref().unwrap(), "Webster Assembler"); + } + + #[test] + fn workspace_overrides_incompatible() { + let toml = r#" +[workspace] +members = ["a/wit"] + +[overrides."foo:bar"] +path = "oo-bar" +"#; + Manifest::from_toml(toml).unwrap_err(); + } + + fn write(path: impl AsRef, contents: &str) { + let path = path.as_ref(); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, contents).unwrap(); + } + + #[tokio::test] + async fn load_workspace_from_root_and_member() { + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + // dir + // ├── wkg.toml + // └── pkg-a + // └── wit + write( + root_dir.join("wkg.toml"), + r#" +[workspace] +members = ["pkg-a"] +"#, + ); + fs::create_dir_all(root_dir.join("pkg-a/wit")).unwrap(); + + let expected = root_dir.canonicalize().unwrap(); + let from_root = Manifest::load_root_workspace(root_dir) + .await + .unwrap() + .unwrap(); + assert_eq!(from_root.root_dir.canonicalize().unwrap(), expected); + + let from_member = Manifest::load_root_workspace(&root_dir.join("pkg-a/wit")) + .await + .unwrap() + .unwrap(); + assert_eq!(from_member.root_dir.canonicalize().unwrap(), expected); + } + + #[tokio::test] + async fn load_workspace_no_root() { + // Standalone: wkg.toml exists but has no [workspace] table -> None. + let tempdir = tempfile::tempdir().unwrap(); + let root_dir = tempdir.path(); + write( + root_dir.join("wkg.toml"), + r#" +[metadata] +authors = "Webster Assembler" +"#, + ); + assert!( + Manifest::load_root_workspace(root_dir) + .await + .unwrap() + .is_none() + ); + } +} diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 5aba92c..5fcf934 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -2,11 +2,12 @@ use std::{ collections::{HashMap, HashSet}, - path::Path, + path::{Path, PathBuf}, str::FromStr, }; use anyhow::{Context as _, Result, bail}; +use indexmap::IndexMap; use petgraph::{Direction, data::Build}; use semver::{Version, VersionReq}; use wasm_metadata::{AddMetadata, AddMetadataField}; @@ -321,38 +322,46 @@ pub async fn resolve_dependencies( /// will be created. Any existing files in the directory will be deleted. The dependencies will be /// put into the `deps` subdirectory within the directory in the format specified by the output /// type. Please note that if a local dep is encountered when using [`OutputType::Wasm`] and it -/// isn't a wasm binary, it will be copied directly to the directory and not packaged into a wit +/// isn't a wasm binary, it will be copied directly to the directory and not packaged into a WIT /// package first pub async fn populate_dependencies( path: impl AsRef, deps: &DependencyResolutionMap, output: OutputType, ) -> Result<()> { + let deps_path = prepare_deps_dir(path.as_ref()).await?; + + // For wit output, generate the resolve and then output each package in the resolve + if let OutputType::Wit = output { + let (resolve, pkg_id) = deps.generate_resolve(path.as_ref()).await?; + return print_wit_from_resolve(&resolve, pkg_id, &deps_path).await; + } + + write_wasm_deps(&deps_path, &deps.decode_dependencies().await?).await +} + +async fn prepare_deps_dir(path: &Path) -> Result { // Canonicalizing will error if the path doesn't exist, so we don't need to check for that let path = tokio::fs::canonicalize(path).await?; - let metadata = tokio::fs::metadata(&path).await?; - if !metadata.is_dir() { + if !tokio::fs::metadata(&path).await?.is_dir() { anyhow::bail!("Path is not a directory"); } - let deps_path = path.join("deps"); + let deps_path = path.join(WIT_DEPS_DIR); // Remove the whole directory if it already exists and then recreate - if let Err(e) = tokio::fs::remove_dir_all(&deps_path).await { - // If the directory doesn't exist, ignore the error - if e.kind() != std::io::ErrorKind::NotFound { - return Err(anyhow::anyhow!("Unable to remove deps directory: {e}")); - } + if let Err(e) = tokio::fs::remove_dir_all(&deps_path).await + && e.kind() != std::io::ErrorKind::NotFound + { + return Err(anyhow::anyhow!("Unable to remove deps directory: {e}") + .context(format!("dir: {}", deps_path.display()))); } tokio::fs::create_dir_all(&deps_path).await?; + Ok(deps_path) +} - // For wit output, generate the resolve and then output each package in the resolve - if let OutputType::Wit = output { - let (resolve, pkg_id) = deps.generate_resolve(&path).await?; - return print_wit_from_resolve(&resolve, pkg_id, &deps_path).await; - } - - // If we got binary output, write them instead of the wit - let decoded_deps = deps.decode_dependencies().await?; - +async fn write_wasm_deps( + deps_path: &Path, + decoded_deps: &IndexMap>, +) -> Result<()> { for (name, dep) in decoded_deps.iter() { let mut output_path = deps_path.join(name_from_package_name(name)); @@ -458,11 +467,23 @@ async fn print_wit_from_resolve( top_level_id: PackageId, root_deps_dir: &Path, ) -> Result<()> { - for (id, pkg) in resolve - .packages - .iter() - .filter(|(id, _)| *id != top_level_id) - { + print_wit_packages( + resolve, + root_deps_dir, + resolve + .packages + .iter() + .filter(|(id, _)| *id != top_level_id), + ) + .await +} + +async fn print_wit_packages<'a>( + resolve: &Resolve, + root_deps_dir: &Path, + packages: impl IntoIterator, +) -> Result<()> { + for (id, pkg) in packages { let dep_path = root_deps_dir.join(name_from_package_name(&pkg.name)); tokio::fs::create_dir_all(&dep_path).await?; let mut printer = WitPrinter::default(); @@ -474,6 +495,57 @@ async fn print_wit_from_resolve( Ok(()) } +/// Populate dependency list to a given directory, aggregating subdirectory dependencies. +/// Behaves like [`populate_dependencies`], but is intended for workspace-scoped aggregation +/// +/// [`OutputType::Wit`] will merge all decoded dependencies into a single [`Resolve`], printing +/// each package. +/// [`OutputType::Wasm`] behaves identically to [`populate_dependencies`]. +pub async fn populate_dependencies_workspace( + path: impl AsRef, + deps: &DependencyResolutionMap, + output: OutputType, +) -> Result<()> { + let deps_path = prepare_deps_dir(path.as_ref()).await?; + let deps = deps.decode_dependencies().await?; + + if let OutputType::Wit = output { + let mut merged = Resolve { + all_features: true, + ..Resolve::default() + }; + for decoded in deps.into_values() { + match decoded { + DecodedDependency::Wit { + resolution, + package, + } => { + let name = resolution.name().to_string(); + merged + .push_group(package) + .with_context(|| format!("failed to merge `{name}`"))?; + } + DecodedDependency::Wasm { + resolution, + decoded, + } => { + let name = resolution.name().to_string(); + let resolve = match decoded { + wit_component::DecodedWasm::WitPackage(resolve, _) => resolve, + wit_component::DecodedWasm::Component(resolve, _) => resolve, + }; + merged + .merge(resolve) + .with_context(|| format!("failed to merge world for `{name}`"))?; + } + } + } + return print_wit_packages(&merged, &deps_path, merged.packages.iter()).await; + } + + write_wasm_deps(&deps_path, &deps).await +} + /// Given a package name, returns a valid directory/file name for it (thanks windows!) fn name_from_package_name(package_name: &PackageName) -> String { let package_name_str = package_name.to_string(); diff --git a/crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml b/crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml new file mode 100644 index 0000000..acf504d --- /dev/null +++ b/crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml @@ -0,0 +1,6 @@ +[workspace] +# Mix glob with literal entries +members = [ + "example-*/wit", + "example-c/wit/nested", +] diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 887eb03..c0ff439 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -4,7 +4,7 @@ use std::{ }; use anstream::eprintln; -use anyhow::{Context, anyhow, ensure}; +use anyhow::{Context, anyhow, bail, ensure}; use clap::{Args, Parser, Subcommand, ValueEnum}; use futures_util::TryStreamExt; use tokio::io::AsyncWriteExt; @@ -19,7 +19,10 @@ use wasm_pkg_common::{ package::PackageSpec, registry::Registry, }; -use wasm_pkg_core::lock::LockFile; +use wasm_pkg_core::{ + lock::LockFile, + manifest::{Manifest, workspace::WorkspaceRootConfig}, +}; use wit_component::DecodedWasm; mod oci; @@ -284,6 +287,10 @@ struct PublishArgs { #[arg(long)] dry_run: bool, + /// Publish all packages in the workspace + #[arg(long)] + workspace: bool, + /// Disable semver compatibility checks. #[arg(long)] skip_semver_check: bool, @@ -299,9 +306,16 @@ struct PublishArgs { } impl PublishArgs { - pub async fn run(self) -> anyhow::Result<()> { + pub async fn run(mut self) -> anyhow::Result<()> { let publish_opts = self.publish_opts()?; + let _root = self.workspace_root().await?; let path = match &self.paths[..] { + [] => { + anyhow::bail!( + "no publish targets: pass one or more paths, or run from a workspace \ + (see `[workspace] members` in `wkg.toml`)" + ); + } [path] => path, paths => { let mut lock_file = LockFile::load(false).await?; @@ -387,6 +401,31 @@ impl PublishArgs { Ok(()) } + fn publish_opts(&mut self) -> anyhow::Result { + let package = match self.package.clone() { + Some(_) if self.paths.len() != 1 => { + anyhow::bail!( + "`--package` is currently only supported when providing one path argument" + ); + } + Some(PackageSpec { + package, + version: Some(v), + }) => Some((package, v)), + Some(PackageSpec { version: None, .. }) => { + anyhow::bail!("version is required when manually overriding the package ID"); + } + None => None, + }; + + Ok(PublishOpts { + package, + registry: self.registry_args.registry.clone(), + dry_run: self.dry_run, + skip_semver_check: self.skip_semver_check, + }) + } + fn handle_publish_result( &self, res: Result<(PackageRef, Version), wasm_pkg_common::Error>, @@ -407,28 +446,25 @@ impl PublishArgs { Ok(()) } - fn publish_opts(&self) -> anyhow::Result { - let package = match self.package.clone() { - Some(_) if self.paths.len() > 2 => { - anyhow::bail!( - "`--package` is currently unsupported when providing more than one path argument" - ); - } - Some(PackageSpec { - package, - version: Some(v), - }) => Some((package, v)), - Some(PackageSpec { version: None, .. }) => { - anyhow::bail!("version is required when manually overriding the package ID"); - } - None => None, + async fn workspace_root(&mut self) -> anyhow::Result> { + match self.workspace { + true if !self.paths.is_empty() => anyhow::bail!( + "`--workspace` selects every workspace member; do not also pass explicit \ + path arguments" + ), + + true => {} + false => return Ok(None), + } + let cwd = std::env::current_dir()?; + let Some(root) = Manifest::load_root_workspace(&cwd).await? else { + bail!( + "`--workspace` called but unable to find workspace root from {}", + cwd.display(), + ) }; - Ok(PublishOpts { - package, - registry: self.registry_args.registry.clone(), - dry_run: self.dry_run, - skip_semver_check: self.skip_semver_check, - }) + self.paths = root.members.clone(); + Ok(Some(root)) } } diff --git a/crates/wkg/src/overlay.rs b/crates/wkg/src/overlay.rs index 4a3e8b0..0f40085 100644 --- a/crates/wkg/src/overlay.rs +++ b/crates/wkg/src/overlay.rs @@ -20,10 +20,8 @@ use crate::wit::build_wit_dir; /// A [`CachingClient`] and [`PublishPlan`] wired to a temporary local backend pub(crate) struct PublishVerifier { - #[expect(dead_code, reason = "workspaces")] pub(crate) client: CachingClient, pub(crate) plan: PublishPlan, - #[expect(dead_code, reason = "workspaces")] pub(crate) packages: BTreeSet, pub(crate) data: HashMap>, /// Held so the temp local backend outlives the returned client. diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 052596d..38c7cdc 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -1,4 +1,5 @@ //! Args and commands for interacting with WIT files and dependencies +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use anstream::eprintln; @@ -7,13 +8,16 @@ use clap::{Args, Subcommand}; use tempfile::NamedTempFile; use wasm_pkg_client::caching::{CachingClient, FileCache}; use wasm_pkg_common::package::{PackageRef, Version}; +use wasm_pkg_core::wit::WIT_DEPS_DIR; use wasm_pkg_core::{ - lock::LockFile, - manifest::Manifest, + lock::{LOCK_FILE_NAME, LockFile, LockedPackage}, + manifest::{MANIFEST_FILE_NAME, Manifest, workspace::WorkspaceRootConfig}, + resolver::DependencyResolutionMap, wit::{self, OutputType}, }; use crate::Common; +use crate::overlay::PublishVerifier; /// Commands for interacting with wit #[derive(Debug, Subcommand)] @@ -43,7 +47,7 @@ pub struct BuildArgs { pub dir: PathBuf, /// The name of the file that should be written. This can also be a full path. Defaults to the - /// current directory with the name of the package + /// current directory with the name of the package. #[clap(short = 'o', long = "output")] pub output: Option, @@ -60,8 +64,8 @@ pub struct BuildArgs { #[derive(Debug, Args)] pub struct FetchArgs { /// The directory containing the WIT files to fetch dependencies for. - #[clap(short = 'd', long = "wit-dir", default_value = "wit")] - pub dir: PathBuf, + /// Falls back to workspace manifest if empty. + pub dir: Option, /// The desired output type of the dependencies. Valid options are "wit" or "wasm" (wasm is the /// WIT package binary format). @@ -150,24 +154,165 @@ pub async fn temp_wit_file(package: &PackageRef, bytes: &[u8]) -> anyhow::Result impl FetchArgs { pub async fn run(self) -> anyhow::Result<()> { - check_dir(&self.dir).await?; - let client = self.common.get_client().await?; - let manifest = Manifest::load().await?; - let mut lock_file = LockFile::load(false).await?; - wit::fetch_dependencies( - &manifest, - self.dir, + let cwd = std::env::current_dir()?; + let root = Manifest::load_root_workspace(&cwd).await?; + + let dirs = if let Some(dir) = self.dir.clone() { + vec![dir] + } else { + root.as_ref() + .map(|r| r.members.clone()) + .unwrap_or_else(|| vec![PathBuf::from("wit")]) + }; + let manifest = match root.as_ref() { + Some(root) => { + let manifest_path = root.root_dir().join(MANIFEST_FILE_NAME); + Manifest::load_from_path(manifest_path).await? + } + None => Manifest::load().await?, + }; + let output = self.output_type.unwrap_or_default(); + + for dir in &dirs { + check_dir(dir).await?; + } + + match root { + Some(root) => { + self.run_workspace_fetch(&dirs, output, &manifest, root) + .await + } + None => self.fetch_into_lock(&dirs, &manifest, output).await, + } + } + + // fetch dependneces for a given workspace root, merging dependencies trees for included packages + async fn run_workspace_fetch( + &self, + dirs: &[PathBuf], + output: OutputType, + manifest: &Manifest, + root: WorkspaceRootConfig, + ) -> anyhow::Result<()> { + // Load/create the root lock file. This is the file that will be committed back to disk + let lock_path = root.root_dir().join(LOCK_FILE_NAME); + let mut lock_file = load_or_create_lock(&lock_path).await?; + + let verifier = PublishVerifier::try_new( + root.members.as_ref(), + "tmp_local_fetch", + self.common.load_config().await?, + self.common.load_cache().await?, &mut lock_file, - client, - self.output_type.unwrap_or_default(), + false, ) .await?; - // Now write out the lock file since everything else succeeded + + // Resolve dependencies for every requested member through the publish verifier + let mut merged = DependencyResolutionMap::default(); + for dir in dirs { + let resolved = + wit::resolve_dependencies(manifest, dir, Some(&lock_file), verifier.client.clone()) + .await + .with_context(|| { + format!("failed to resolve dependencies for {}", dir.display()) + })?; + for (pkg, resolution) in resolved.as_ref() { + if verifier.packages.contains(pkg) { + continue; + } + merged.insert(pkg.clone(), resolution.clone()); + } + } + + lock_file.update_dependencies(&merged); + lock_file + .write() + .await + .with_context(|| format!("failed to commit lock file at {}", lock_path.display()))?; + + // Ensure `/wkg/` exists and drop the aggregated deps into `/wkg/deps`. + // `populate_dependencies` canonicalizes its argument, so the parent must exist. + let out_dir = root.out_dir(); + tokio::fs::create_dir_all(&out_dir) + .await + .with_context(|| format!("failed to create {}", out_dir.display()))?; + wit::populate_dependencies_workspace(&out_dir, &merged, output) + .await + .with_context(|| { + format!( + "failed to populate workspace deps at {}", + out_dir.join(WIT_DEPS_DIR).display(), + ) + }) + } + + /// Iterate `dirs` and run [`wit::fetch_dependencies`] unioning each call's resolved lock entries into a + /// single set. + /// `fetch_dependencies` replaces [`LockFile`] packages on every call so we snapshot + /// between calls to avoid losing earlier entries. + async fn fetch_into_lock( + &self, + dirs: &[PathBuf], + manifest: &Manifest, + output: OutputType, + ) -> anyhow::Result<()> { + let client = self.common.get_client().await?; + let mut lock_file = LockFile::load(false).await?; + + let mut union: BTreeSet = BTreeSet::new(); + merge_locked_packages(&mut union, std::mem::take(&mut lock_file.packages)); + for dir in dirs { + wit::fetch_dependencies(manifest, dir, &mut lock_file, client.clone(), output) + .await + .with_context(|| format!("failed to fetch dependencies for {}", dir.display()))?; + merge_locked_packages(&mut union, std::mem::take(&mut lock_file.packages)); + } + lock_file.packages = union; + lock_file.write().await?; Ok(()) } } +/// Open `/wkg.lock` for read-write, creating it as an empty lockfile if missing. +/// Mirrors `LockFile::load`'s create-if-absent semantics but at an explicit path. +async fn load_or_create_lock(path: &Path) -> anyhow::Result { + if !tokio::fs::try_exists(path).await? { + let mut empty = LockFile::new_with_path([], path).await?; + empty.write().await?; + drop(empty); + } + LockFile::load_from_path(path, false) + .await + .with_context(|| format!("failed to load lock file at {}", path.display())) +} + +/// Merge `incoming` locked packages into `acc`, unioning per-package version +/// entries when the same `(name, registry)` key appears in both sets. Newer +/// version entries win on conflicting requirements within a package. +fn merge_locked_packages(acc: &mut BTreeSet, incoming: BTreeSet) { + for pkg in incoming { + if let Some(existing) = acc.take(&pkg) { + let mut merged = existing; + for v in pkg.versions { + if let Some(slot) = merged + .versions + .iter_mut() + .find(|e| e.requirement == v.requirement) + { + *slot = v; + } else { + merged.versions.push(v); + } + } + acc.insert(merged); + } else { + acc.insert(pkg); + } + } +} + impl UpdateArgs { pub async fn run(self) -> anyhow::Result<()> { check_dir(&self.dir).await?; diff --git a/crates/wkg/tests/common.rs b/crates/wkg/tests/common.rs index 0aca4de..24d92dd 100644 --- a/crates/wkg/tests/common.rs +++ b/crates/wkg/tests/common.rs @@ -61,6 +61,32 @@ pub async fn start_registry() -> (Config, Registry, ContainerAsync (config, registry, container) } +pub const TRANSITIVE_LOCAL_NAMESPACES: &[&str] = + &["example-a", "example-b", "example-c", "example-d"]; + +/// Maps every namespace in [`TRANSITIVE_LOCAL_NAMESPACES`] to `registry`. +pub fn map_transitive_local_namespaces(config: &Config, registry: &Registry) -> Config { + let mut mapped = config.clone(); + for ns in TRANSITIVE_LOCAL_NAMESPACES { + mapped = map_namespace(&mapped, ns, registry); + } + mapped +} + +/// runs `wkg publish --workspace` for [`TRANSITIVE_LOCAL_NAMESPACES`] packages +pub async fn publish_transitive_local(config: &Config) -> Fixture { + let fixture = load_fixture_from(transitive_local_fixture()).await; + let status = fixture + .command_with_config(config) + .await + .args(["publish", "--workspace"]) + .status() + .await + .expect("spawn wkg publish"); + assert!(status.success(), "`wkg publish --workspace` should succeed",); + fixture +} + /// Clones the given config, mapping the namespace to the given registry at the top level pub fn map_namespace(config: &Config, namespace: &str, registry: &Registry) -> Config { let mut config = config.clone(); @@ -116,18 +142,27 @@ pub fn fixture_dir() -> PathBuf { .join("fixtures") } +pub fn transitive_local_fixture() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../wasm-pkg-core/tests/fixtures/transitive-local") +} + /// Loads the fixture with the given name into a temporary directory. This will copy the fixture /// from the tests/fixtures directory into a temporary directory and return the tempdir containing /// that directory (and its path) pub async fn load_fixture(fixture: &str) -> Fixture { + load_fixture_from(fixture_dir().join(fixture)).await +} + +pub async fn load_fixture_from(src: impl AsRef) -> Fixture { + let src = src.as_ref(); let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); - let fixture_path = fixture_dir().join(fixture); // This will error if it doesn't exist, which is what we want - tokio::fs::metadata(&fixture_path) + tokio::fs::metadata(src) .await .expect("Fixture does not exist or couldn't be read"); - let copied_path = temp_dir.path().join(fixture_path.file_name().unwrap()); - copy_dir(&fixture_path, &copied_path) + let copied_path = temp_dir.path().join(src.file_name().unwrap()); + copy_dir(src, &copied_path) .await .expect("Failed to copy fixture"); Fixture { diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index 92a9899..6fbcfb6 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -1,6 +1,7 @@ use wasm_pkg_client::{Version, VersionInfo}; -use crate::common::copy_dir; +#[cfg(feature = "docker-tests")] +use crate::common::{map_transitive_local_namespaces, publish_transitive_local}; mod common; @@ -91,48 +92,19 @@ async fn build_and_publish_with_metadata() { #[cfg(feature = "docker-tests")] #[tokio::test] -async fn publish_multiple_transitive_local_packages() { - use std::path::PathBuf; - +async fn publish_workspace_packages() { let (config, registry, _container) = common::start_registry().await; - let namespaces = ["example-a", "example-b", "example-c", "example-d"]; - - // copy the transitive-local fixtures from wasm-pkg-core into a temp dir - let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); - let src_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../wasm-pkg-core/tests/fixtures/transitive-local"); - let fixture_root = temp_dir.path().join("transitive-local"); - copy_dir(&src_root, &fixture_root).await.unwrap(); - - let mut mapped = config.clone(); - for ns in namespaces { - mapped = common::map_namespace(&mapped, ns, ®istry); - } - let config_path = temp_dir.path().join("config.toml"); - mapped.to_file(&config_path).await.expect("write config"); - - // pass all fixture dirs to a single `wkg publish` invocation - let mut dirs: Vec = namespaces - .iter() - .map(|name| fixture_root.join(name).join("wit")) - .collect(); - // TODO use glob suchas in `wasm_pkgs_core::resolver::tests::transitive_local_paths` - dirs.push(fixture_root.join("example-c/wit/nested")); - - let mut publish = tokio::process::Command::new(env!("CARGO_BIN_EXE_wkg")); - publish - .current_dir(temp_dir.path()) - .env("WKG_CACHE_DIR", temp_dir.path().join("cache")) - .env("WKG_CONFIG_FILE", &config_path) - .arg("publish"); - for dir in &dirs { - publish.arg(dir); - } - let status = publish.status().await.expect("spawn wkg publish"); - assert!(status.success(), "wkg publish should succeed"); - let client = wasm_pkg_client::Client::new(mapped); + let mapped = map_transitive_local_namespaces(&config, ®istry); + let fixture = publish_transitive_local(&mapped).await; + + assert!( + fixture.fixture_path.join("wkg.toml").exists(), + "fixture must include the workspace manifest copied from \ + crates/wasm-pkg-core/tests/fixtures/transitive-local/wkg.toml", + ); + let client = wasm_pkg_client::Client::new(mapped); let expected_version = "0.1.0".parse::().unwrap(); for name in [ "example-a:foo", @@ -146,12 +118,60 @@ async fn publish_multiple_transitive_local_packages() { .list_all_versions(&pkg) .await .unwrap_or_else(|e| panic!("list versions for {name}: {e:#}")); + std::assert_matches!( + &versions[..], + [VersionInfo { version, .. }] if version == &expected_version, + "{name} should have exactly one published version", + ); + } +} + +#[cfg(feature = "docker-tests")] +#[tokio::test] +async fn fetch_workspace_packages() { + use wasm_pkg_core::lock::{LOCK_FILE_NAME, LockFile}; + + let (config, registry, _container) = common::start_registry().await; + let mapped = map_transitive_local_namespaces(&config, ®istry); + let _publisher = publish_transitive_local(&mapped).await; + + let fixture = common::load_fixture("fetch-workspace").await; + let status = fixture + .command_with_config(&mapped) + .await + .arg("fetch") + .status() + .await + .expect("spawn wkg fetch"); + assert!( + status.success(), + "`wkg fetch` in fetch-workspace should succeed" + ); + + let lock_path = fixture.fixture_path.join(LOCK_FILE_NAME); + assert!(lock_path.exists(), "`wkg fetch` should create wkg.lock",); + let lock = LockFile::load_from_path(&lock_path, true) + .await + .expect("load fetch-workspace wkg.lock"); + let expected_version = "0.1.0".parse::().unwrap(); + for expected_pkgs in ["example-a:foo", "example-d:foo"] { + let pkg = expected_pkgs.parse().unwrap(); + let entry = lock + .packages + .iter() + .find(|p| p.name == pkg) + .unwrap_or_else(|| { + panic!( + "wkg.lock should contain {expected_pkgs}; had: {:?}", + lock.packages + .iter() + .map(|p| p.name.to_string()) + .collect::>() + ) + }); assert!( - matches!( - &versions[..], - [VersionInfo { version, .. }] if version == &expected_version, - ), - "{name} should have exactly one published version, got {versions:?}", + entry.versions.iter().any(|v| v.version == expected_version), + "{expected_pkgs} should be locked at {expected_version}; entry: {entry:?}", ); } } diff --git a/crates/wkg/tests/fixtures/fetch-workspace/fetch-a/wit/world.wit b/crates/wkg/tests/fixtures/fetch-workspace/fetch-a/wit/world.wit new file mode 100644 index 0000000..1c7b007 --- /dev/null +++ b/crates/wkg/tests/fixtures/fetch-workspace/fetch-a/wit/world.wit @@ -0,0 +1,5 @@ +package fetch:a@0.1.0; + +world app { + import example-a:foo/foo@0.1.0; +} diff --git a/crates/wkg/tests/fixtures/fetch-workspace/fetch-d/wit/world.wit b/crates/wkg/tests/fixtures/fetch-workspace/fetch-d/wit/world.wit new file mode 100644 index 0000000..9bb1cba --- /dev/null +++ b/crates/wkg/tests/fixtures/fetch-workspace/fetch-d/wit/world.wit @@ -0,0 +1,5 @@ +package fetch:d@0.1.0; + +world app { + import example-d:foo/bar@0.1.0; +} diff --git a/crates/wkg/tests/fixtures/fetch-workspace/wkg.toml b/crates/wkg/tests/fixtures/fetch-workspace/wkg.toml new file mode 100644 index 0000000..91c3cd9 --- /dev/null +++ b/crates/wkg/tests/fixtures/fetch-workspace/wkg.toml @@ -0,0 +1,2 @@ +[workspace] +members = ["fetch-*/wit"]