diff --git a/Cargo.lock b/Cargo.lock index ac75e1ce64..77ef5d88ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6891,6 +6891,7 @@ dependencies = [ "assert_matches", "crossbeam-channel", "deno_task_shell", + "dunce", "fancy_display", "fs-err", "itertools 0.14.0", diff --git a/crates/pixi_cli/src/install.rs b/crates/pixi_cli/src/install.rs index 5bc2b1e46a..1aa3f25ddc 100644 --- a/crates/pixi_cli/src/install.rs +++ b/crates/pixi_cli/src/install.rs @@ -250,5 +250,38 @@ pub async fn execute(args: Args) -> miette::Result<()> { eprintln!("{message}."); + // Under the `hierarchical-tasks` preview feature, the workspace owns a + // tree of member workspaces. `pixi install` at the root is eager: we + // walk every reachable member and install its default environment, + // updating each member's own lockfile in the process. This matches + // the user-approved behaviour of "lazy install for tasks, eager + // install for `pixi install`". + // + // Errors from any member abort the command with the path prefix of + // the failing member so users can see exactly which project broke. + for (path, member_ws) in workspace.walk_members() { + let member_envs = vec![member_ws.default_environment()]; + let prefix = path.join("::"); + get_update_lock_file_and_prefixes( + &member_envs, + UpdateMode::Revalidate, + UpdateLockFileOptions { + lock_file_usage: args.lock_file_usage.to_usage(), + no_install: false, + max_concurrent_solves: member_ws.config().max_concurrent_solves(), + }, + ReinstallPackages::default(), + &filter, + ) + .await + .map_err(|e| e.wrap_err(format!("failed to install member workspace `{prefix}`")))?; + + eprintln!( + "{} The {} member's default environment has been installed", + console::style(console::Emoji("✔ ", "")).green(), + console::style(&prefix).bold(), + ); + } + Ok(()) } diff --git a/crates/pixi_cli/src/run.rs b/crates/pixi_cli/src/run.rs index 0f605a7e49..ebc5d2fc65 100644 --- a/crates/pixi_cli/src/run.rs +++ b/crates/pixi_cli/src/run.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet, hash_map::Entry}, convert::identity, ffi::OsString, + path::PathBuf, string::String, }; @@ -19,8 +20,8 @@ use pixi_config::{ConfigCli, ConfigCliActivation}; use pixi_core::{ Workspace, WorkspaceLocator, environment::sanity_check_workspace, - lock_file::{ReinstallPackages, UpdateLockFileOptions, UpdateMode}, - workspace::{Environment, errors::UnsupportedPlatformError}, + lock_file::{LockFileDerivedData, ReinstallPackages, UpdateLockFileOptions, UpdateMode}, + workspace::{Environment, HasWorkspaceRef, errors::UnsupportedPlatformError}, }; use pixi_manifest::{FeaturesExt, TaskName}; use pixi_progress::global_multi_progress; @@ -217,6 +218,39 @@ pub async fn execute(args: Args) -> miette::Result<()> { )?; tracing::debug!("Task graph: {}", task_graph); + // Under the `hierarchical-tasks` preview feature, tasks may run in + // member workspaces (e.g. `pixi run a::test`). Each member has its + // own lockfile and its own `.pixi/envs/` install dir — so we can't + // reuse the root's `lock_file` for a member task. Collect the + // distinct member workspaces referenced by the graph and lazily + // update each one's lockfile. We look these up by workspace root + // path when executing tasks. The root workspace always uses the + // `lock_file` already computed above. + // + // When no member is referenced by the graph (the common case), this + // map is empty and behaviour is identical to pre-preview pixi. + let mut member_lock_files: HashMap> = HashMap::new(); + { + let root_path = workspace.root().to_path_buf(); + let mut seen: HashSet = HashSet::from([root_path.clone()]); + for task_id in task_graph.topological_order() { + let node_ws: &Workspace = task_graph[task_id].run_environment.workspace(); + let node_root = node_ws.root().to_path_buf(); + if !seen.insert(node_root.clone()) { + continue; + } + let member_lf = node_ws + .update_lock_file(UpdateLockFileOptions { + lock_file_usage: args.lock_and_install_config.lock_file_usage()?, + no_install: args.lock_and_install_config.no_install(), + max_concurrent_solves: node_ws.config().max_concurrent_solves(), + }) + .await? + .0; + member_lock_files.insert(node_root, member_lf); + } + } + // Print dry-run message if dry-run mode is enabled if args.dry_run { pixi_progress::println!( @@ -288,9 +322,22 @@ pub async fn execute(args: Args) -> miette::Result<()> { continue; } + // Under `hierarchical-tasks`, the task may run in a member + // workspace; pick that workspace's lockfile instead of the + // root's. For the common case (task belongs to the root or the + // preview is off) this resolves to the root's `lock_file`. + let task_lock_file: &LockFileDerivedData<'_> = + if executable_task.workspace.root() == workspace.root() { + &lock_file + } else { + member_lock_files + .get(executable_task.workspace.root()) + .expect("member lockfile must have been populated before the task loop runs") + }; + // check task cache let task_cache = match executable_task - .can_skip(lock_file.as_lock_file()) + .can_skip(task_lock_file.as_lock_file()) .await .into_diagnostic()? { @@ -322,8 +369,11 @@ pub async fn execute(args: Args) -> miette::Result<()> { Entry::Vacant(entry) => { // Check if we allow installs if args.lock_and_install_config.allow_installs() { - // Ensure there is a valid prefix - lock_file + // Ensure there is a valid prefix. `task_lock_file` + // already matches `executable_task`'s workspace, so + // the install dir, lockfile, and command dispatcher + // all target the right workspace (root or member). + task_lock_file .prefix( &executable_task.run_environment, UpdateMode::QuickValidate, @@ -334,17 +384,25 @@ pub async fn execute(args: Args) -> miette::Result<()> { } // Clear the current progress reports. + task_lock_file.command_dispatcher.clear_reporter().await; progress.on_clear(); + // Clear caches based on the filesystem. The tasks might change files on disk. - lock_file.command_dispatcher.clear_filesystem_caches().await; + task_lock_file + .command_dispatcher + .clear_filesystem_caches() + .await; + let task_workspace = executable_task.workspace; let command_env = get_task_env( &executable_task.run_environment, args.clean_env || executable_task.task().clean_env(), - Some(lock_file.as_lock_file()), - workspace.config().force_activate(), - workspace.config().experimental_activation_cache_usage(), + Some(task_lock_file.as_lock_file()), + task_workspace.config().force_activate(), + task_workspace + .config() + .experimental_activation_cache_usage(), ) .await?; entry.insert(command_env) diff --git a/crates/pixi_cli/src/task.rs b/crates/pixi_cli/src/task.rs index 80255b305c..7b93901fcf 100644 --- a/crates/pixi_cli/src/task.rs +++ b/crates/pixi_cli/src/task.rs @@ -337,13 +337,19 @@ async fn list_tasks( return print_tasks_json(workspace_ctx.workspace()); } - let tasks_per_env = workspace_ctx + let mut tasks_per_env = workspace_ctx .list_tasks( args.environment .and_then(|e| EnvironmentName::from_str(&e.to_string()).ok()), ) .await?; + // Hierarchical-tasks: inject member tasks into the default environment + // listing with their fully-qualified `a::b::task` names. These tasks do + // not belong to any single environment — they live on member nodes — so + // we surface them under the default environment for display purposes. + inject_member_tasks_for_display(&mut tasks_per_env, workspace_ctx.workspace()); + if tasks_per_env.is_empty() { eprintln!("No tasks found",); return Ok(()); @@ -536,3 +542,34 @@ impl From<&Task> for TaskInfo { } } } + +/// Walks the workspace's member tree and appends every reachable member task +/// to `tasks_per_env` under the default environment, using the fully +/// qualified `a::b::task` name. Does nothing when the tree is empty (the +/// usual case when `hierarchical-tasks` is disabled). +fn inject_member_tasks_for_display( + tasks_per_env: &mut HashMap>, + workspace: &Workspace, +) { + if workspace.members().is_empty() { + return; + } + + let default_env_name = workspace.default_environment().name().clone(); + let bucket = tasks_per_env.entry(default_env_name).or_default(); + + // Each member is its own standalone Workspace (Model 2). Its tasks + // live on the member's default environment — we surface them here + // under the root's default-env listing with fully-qualified names so + // `pixi task list` shows the entire reachable set without a caller + // needing to enter each member. + for (path, member_ws) in workspace.walk_members() { + let prefix = path.join("::"); + if let Ok(member_tasks) = member_ws.default_environment().tasks(None) { + for (task_name, task) in member_tasks { + let qualified = format!("{prefix}::{}", task_name.as_str()); + bucket.insert(TaskName::from(qualified), task.clone()); + } + } + } +} diff --git a/crates/pixi_core/src/workspace/discovery.rs b/crates/pixi_core/src/workspace/discovery.rs index 8fec99223f..be5cc5c2fa 100644 --- a/crates/pixi_core/src/workspace/discovery.rs +++ b/crates/pixi_core/src/workspace/discovery.rs @@ -1,4 +1,7 @@ -use std::{path::PathBuf, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; use itertools::Itertools; use miette::{Diagnostic, NamedSource, Report}; @@ -10,7 +13,7 @@ use pixi_manifest::{ use thiserror::Error; use crate::workspace::WorkspaceRegistry; -use crate::workspace::{Workspace, WorkspaceRegistryError}; +use crate::workspace::{MemberWorkspace, Workspace, WorkspaceRegistryError}; /// Defines where the search for the workspace should start. #[derive(Debug, Clone, Default)] @@ -136,6 +139,10 @@ pub enum WorkspaceLocatorError { #[error(transparent)] #[diagnostic(transparent)] InvalidRequiresPixi(#[from] Box), + + #[error(transparent)] + #[diagnostic(transparent)] + MemberDiscovery(#[from] Box), } impl WorkspaceLocator { @@ -293,7 +300,30 @@ impl WorkspaceLocator { ); } - let workspace = Workspace::from_manifests(discovered_manifests); + let mut workspace = Workspace::from_manifests(discovered_manifests); + + // If the `hierarchical-tasks` preview feature is enabled on the + // discovered root workspace, recursively load each discovered + // member's own standalone Workspace and attach it to the root's + // member tree. Under Model 2 every member has its own [workspace] + // and is fully self-contained — we just load it via the same + // locator machinery, then key the results by member name. + if workspace + .workspace + .value + .workspace + .preview + .is_enabled(pixi_manifest::KnownPreviewFeature::HierarchicalTasks) + { + // Find the MemberTree that was populated during manifest + // discovery and load each member. This runs the full + // `WorkspaceLocator` for each member directory, which means + // each member gets the same discovery/canonicalisation/ + // requires-pixi treatment as any standalone workspace. + let workspace_dir = workspace.root().to_path_buf(); + let members = load_members_recursively(&workspace_dir)?; + workspace.set_members(members); + } Ok(workspace) } @@ -341,6 +371,60 @@ impl WorkspaceLocator { } } +/// Recursively load each discovered member as its own standalone +/// [`Workspace`], keyed by the member's `[workspace].name`. +/// +/// This is called by [`WorkspaceLocator::locate`] once the root workspace +/// has been constructed and the `hierarchical-tasks` preview feature is +/// enabled. Each member goes through the same [`WorkspaceLocator`] path as +/// a standalone workspace — so the member's own manifest discovery, +/// canonicalisation, and `requires-pixi` check all run unchanged. +/// +/// The `workspace_dir` argument is the already-canonicalised root of the +/// outer workspace. Member discovery only descends from here; it never +/// climbs back upward, so a member's own upward walk reliably stops at +/// the member's `[workspace]` (and not the outer root). +fn load_members_recursively( + workspace_dir: &Path, +) -> Result, WorkspaceLocatorError> { + // Structural discovery: names + directories only, no Workspace load yet. + let tree = pixi_manifest::discover_members(workspace_dir) + .map_err(Box::new) + .map_err(WorkspaceLocatorError::MemberDiscovery)?; + + fn build( + nodes: &indexmap::IndexMap, + ) -> Result, WorkspaceLocatorError> { + let mut out = indexmap::IndexMap::with_capacity(nodes.len()); + for (name, node) in nodes { + // Load this member as its own standalone Workspace. We + // deliberately avoid `with_consider_environment` / warning + // emission so member loads don't pollute the root's output. + let member_ws = WorkspaceLocator::for_cli() + .with_consider_environment(false) + .with_emit_warnings(false) + .with_search_start(DiscoveryStart::SearchRoot(node.dir.clone())) + .locate()?; + + // Recurse into this member's own nested members. Each level + // owns its own loaded Workspace; the tree mirrors the + // structure returned by `discover_members`. + let children = build(&node.children)?; + + out.insert( + name.clone(), + MemberWorkspace { + workspace: member_ws, + children, + }, + ); + } + Ok(out) + } + + build(tree.members()) +} + #[cfg(test)] mod test { use std::path::Path; diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index acc8dead82..ee0afcad7f 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -182,6 +182,27 @@ pub struct Workspace { /// Optional backend override for testing purposes backend_override: Option, + + /// Nested member workspaces, populated only when the workspace enables + /// the `hierarchical-tasks` preview feature. Empty otherwise. Each + /// member is itself a fully-loaded, standalone [`Workspace`] (with its + /// own envs, lockfile, `.pixi/` install dir) — the root simply + /// aggregates them so hierarchical task addresses (`a::b::task`) can + /// dispatch into the right member workspace. + members: indexmap::IndexMap, +} + +/// A member workspace inside the root's hierarchical tree. The `workspace` +/// field is a fully standalone pixi workspace — it can also be used +/// directly (e.g. when the user runs pixi from inside the member's +/// directory, the upward walk resolves to the member itself and the outer +/// aggregation is invisible). +#[derive(Clone, Debug)] +pub struct MemberWorkspace { + /// The member's own loaded Workspace. + pub workspace: Workspace, + /// Further nested member workspaces under this one. + pub children: indexmap::IndexMap, } impl Debug for Workspace { @@ -240,6 +261,11 @@ impl Workspace { .collect::>(); let config = Config::load(&root); + // `from_manifests` is called for every Workspace construction, + // including member workspaces loaded recursively. The member tree + // is attached separately by `WorkspaceLocator::locate` so that the + // recursive load uses the same discovery/locator plumbing as any + // standalone workspace, without reimplementing it here. Self { root, manifest_location_name, @@ -253,7 +279,60 @@ impl Workspace { repodata_gateway: Default::default(), concurrent_downloads_semaphore: OnceCell::default(), backend_override: None, + members: indexmap::IndexMap::new(), + } + } + + /// Hierarchical tree of nested member workspaces discovered under this + /// workspace. Empty unless the workspace enabled the + /// `hierarchical-tasks` preview feature. + pub fn members(&self) -> &indexmap::IndexMap { + &self.members + } + + /// Walks a member path (e.g. `["a", "c"]`) and returns the matching + /// member's [`Workspace`], or `None` if any segment does not exist. + pub fn resolve_member(&self, path: I) -> Option<&Workspace> + where + I: IntoIterator, + S: AsRef, + { + let mut iter = path.into_iter(); + let first = iter.next()?; + let mut cursor = self.members.get(first.as_ref())?; + for seg in iter { + cursor = cursor.children.get(seg.as_ref())?; } + Some(&cursor.workspace) + } + + /// Yields every reachable member workspace as `(path_segments, workspace)` + /// in depth-first, insertion order. `path_segments` is the chain of + /// member names from the root (e.g. `vec!["a", "c"]` for `a::c`). + pub fn walk_members(&self) -> Vec<(Vec, &Workspace)> { + fn visit<'a>( + prefix: &[String], + members: &'a indexmap::IndexMap, + out: &mut Vec<(Vec, &'a Workspace)>, + ) { + for (name, node) in members { + let mut path = prefix.to_vec(); + path.push(name.clone()); + out.push((path.clone(), &node.workspace)); + visit(&path, &node.children, out); + } + } + let mut out = Vec::new(); + visit(&[], &self.members, &mut out); + out + } + + /// Attaches the given member workspace tree. Used by + /// [`crate::WorkspaceLocator`] once each member has been loaded via + /// its own [`Workspace::from_path`] / [`crate::WorkspaceLocator::locate`] + /// call. Not intended for direct use outside of workspace loading. + pub(crate) fn set_members(&mut self, members: indexmap::IndexMap) { + self.members = members; } /// Loads a project from manifest file. The `manifest_path` is expected to diff --git a/crates/pixi_manifest/src/lib.rs b/crates/pixi_manifest/src/lib.rs index c124ecbe67..b1cf319af2 100644 --- a/crates/pixi_manifest/src/lib.rs +++ b/crates/pixi_manifest/src/lib.rs @@ -11,6 +11,7 @@ mod features_ext; mod has_features_iter; mod has_manifest_ref; mod manifests; +mod members; mod package; mod preview; pub mod pypi; @@ -45,6 +46,7 @@ pub use manifests::{ AssociateProvenance, ManifestKind, ManifestProvenance, ManifestSource, PackageManifest, ProvenanceError, WithProvenance, WorkspaceManifest, WorkspaceManifestMut, }; +pub use members::{MemberDiscoveryError, MemberNode, MemberTree, discover_members}; use miette::Diagnostic; pub use package::Package; pub use preview::{KnownPreviewFeature, Preview}; diff --git a/crates/pixi_manifest/src/members.rs b/crates/pixi_manifest/src/members.rs new file mode 100644 index 0000000000..651be8cf4e --- /dev/null +++ b/crates/pixi_manifest/src/members.rs @@ -0,0 +1,493 @@ +//! Recursive downward discovery of nested member workspaces under a +//! workspace root. +//! +//! This module implements the structural discovery side of the +//! `hierarchical-tasks` preview feature described in issue +//! [#5003](https://github.com/prefix-dev/pixi/issues/5003). +//! +//! **Model 2 — federated member workspaces.** A "member" is a subdirectory +//! that contains its own pixi-compatible manifest (`pixi.toml`, +//! `pyproject.toml`, or `mojoproject.toml`) with a top-level `[workspace]` +//! block (or `[tool.pixi.workspace]` for pyproject). Members have their +//! own environments, channels, and lockfile — they are fully standalone +//! pixi projects. Running `pixi run test` inside a member directory +//! treats that member as the root workspace, without any knowledge of +//! the outer aggregation. +//! +//! This module produces only **structural metadata** — names and paths. +//! Actual loading of each member as a `pixi_core::Workspace` happens in a +//! later layer, keyed off the tree returned here. + +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use indexmap::IndexMap; +use miette::{Diagnostic, NamedSource}; +use pixi_consts::consts; +use thiserror::Error; + +use crate::{ManifestKind, ManifestProvenance, ProvenanceError, TomlError, utils::WithSourceCode}; + +/// A tree of member workspaces rooted under a workspace directory. +#[derive(Debug, Default, Clone)] +pub struct MemberTree { + members: IndexMap, +} + +/// A single member node in the tree. Holds only structural metadata — +/// the member is loaded as a full `pixi_core::Workspace` later. +#[derive(Debug, Clone)] +pub struct MemberNode { + /// The workspace name declared in this member's manifest (from + /// `[workspace].name`, `[tool.pixi.workspace].name`, or `[project].name` + /// as a pyproject fallback). + pub name: String, + /// Absolute path to the manifest file for this member. + pub manifest_path: PathBuf, + /// Absolute path to the directory containing the manifest. + pub dir: PathBuf, + /// Nested child members (unbounded depth). + pub children: IndexMap, +} + +impl MemberTree { + /// Returns true if no members were discovered. + pub fn is_empty(&self) -> bool { + self.members.is_empty() + } + + /// The top-level members. + pub fn members(&self) -> &IndexMap { + &self.members + } + + /// Walks a member path (e.g. `["a", "c"]`) and returns the matching node, + /// or `None` if any segment does not exist. + pub fn resolve(&self, path: I) -> Option<&MemberNode> + where + I: IntoIterator, + S: AsRef, + { + let mut iter = path.into_iter(); + let first = iter.next()?; + let mut cursor = self.members.get(first.as_ref())?; + for seg in iter { + cursor = cursor.children.get(seg.as_ref())?; + } + Some(cursor) + } + + /// Yields every reachable member as `(path_segments, node)` in + /// depth-first, insertion order. + pub fn walk(&self) -> Vec<(Vec, &MemberNode)> { + fn visit<'a>( + prefix: &[String], + members: &'a IndexMap, + out: &mut Vec<(Vec, &'a MemberNode)>, + ) { + for node in members.values() { + let mut path = prefix.to_vec(); + path.push(node.name.clone()); + out.push((path.clone(), node)); + visit(&path, &node.children, out); + } + } + + let mut out = Vec::new(); + visit(&[], &self.members, &mut out); + out + } +} + +/// Errors returned by [`discover_members`]. +#[derive(Debug, Error, Diagnostic)] +pub enum MemberDiscoveryError { + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error(transparent)] + #[diagnostic(transparent)] + Toml(Box>>>), + + #[error(transparent)] + #[diagnostic(transparent)] + ProvenanceError(#[from] ProvenanceError), + + #[error("duplicate member workspace name `{name}`: found at `{first}` and `{second}`")] + DuplicateSibling { + name: String, + first: PathBuf, + second: PathBuf, + }, +} + +/// Discovers nested member workspaces rooted at `workspace_dir`. +/// +/// Walks the directory tree under `workspace_dir`, skipping common build / +/// cache / VCS directories. When a directory contains a pixi-compatible +/// manifest with a `[workspace]` block that declares a name, that +/// directory becomes a member and descent continues inside it for further +/// nested members. +/// +/// Manifests without `[workspace]` (for example, a `pixi.toml` that only +/// declares `[package]` or is otherwise a non-workspace manifest) are +/// transparent — discovery walks through them as if they weren't there +/// and keeps searching deeper. +/// +/// Returns an empty tree when no members are found. The root manifest at +/// `workspace_dir/pixi.toml` (or equivalent) is never treated as a member — +/// only descendants are considered. +pub fn discover_members(workspace_dir: &Path) -> Result { + let mut tree = MemberTree::default(); + walk(workspace_dir, &mut tree.members)?; + Ok(tree) +} + +fn walk( + dir: &Path, + members: &mut IndexMap, +) -> Result<(), MemberDiscoveryError> { + let mut entries: Vec = fs_err::read_dir(dir)? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| p.is_dir()) + .filter(|p| should_descend(p)) + .collect(); + entries.sort(); + + for entry in entries { + let Some(provenance) = provenance_from_dir(&entry) else { + walk(&entry, members)?; + continue; + }; + + let parsed = parse_member_manifest(&provenance)?; + + // Under Model 2, a directory is a member iff its manifest has + // [workspace]. Anything else — including `[package]`-only + // manifests — is transparent: we walk right through it to find + // any deeper members. + let Some(name) = parsed.workspace_name else { + walk(&entry, members)?; + continue; + }; + + // Recurse into this member for further nested members. + let mut children = IndexMap::new(); + walk(&entry, &mut children)?; + + let node = MemberNode { + name: name.clone(), + manifest_path: provenance.path.clone(), + dir: entry.clone(), + children, + }; + + if let Some(existing) = members.get(&name) { + return Err(MemberDiscoveryError::DuplicateSibling { + name, + first: existing.dir.clone(), + second: entry, + }); + } + + members.insert(name, node); + } + + Ok(()) +} + +/// Structured view of a member manifest used during discovery. +/// +/// `workspace_name` is `Some(name)` only when the manifest contains a +/// `[workspace]` block that declares a name. Anything else is `None`. +struct ParsedMember { + workspace_name: Option, +} + +fn parse_member_manifest( + provenance: &ManifestProvenance, +) -> Result { + let contents = provenance.read()?.map(Arc::::from); + let source_name = provenance.absolute_path().to_string_lossy().into_owned(); + let inner: Arc = match &contents { + crate::ManifestSource::PixiToml(s) + | crate::ManifestSource::PyProjectToml(s) + | crate::ManifestSource::MojoProjectToml(s) => s.clone(), + }; + + let toml = match toml_span::parse(inner.as_ref()) { + Ok(t) => t, + Err(e) => { + let source = NamedSource::new(source_name, inner).with_language("toml"); + return Err(MemberDiscoveryError::Toml(Box::new(WithSourceCode { + error: TomlError::from(e), + source, + }))); + } + }; + + let (workspace_ptr, workspace_name_ptr): (&'static str, &'static str) = match provenance.kind { + ManifestKind::Pixi | ManifestKind::MojoProject => ("/workspace", "/workspace/name"), + ManifestKind::Pyproject => ("/tool/pixi/workspace", "/tool/pixi/workspace/name"), + }; + + // `[workspace]` is the sole membership signal. + if toml.pointer(workspace_ptr).is_none() { + return Ok(ParsedMember { + workspace_name: None, + }); + } + + let workspace_name = toml + .pointer(workspace_name_ptr) + .and_then(|v| v.as_str().map(|s| s.to_string())) + .or_else(|| { + if matches!(provenance.kind, ManifestKind::Pyproject) { + toml.pointer("/project/name") + .and_then(|v| v.as_str().map(|s| s.to_string())) + } else { + None + } + }); + + Ok(ParsedMember { workspace_name }) +} + +/// Directories we never descend into. Dot-prefixed names are skipped +/// separately in [`should_descend`]. +const SKIP_DIR_NAMES: &[&str] = &[ + "node_modules", + "target", + "dist", + "build", + "__pycache__", + "venv", +]; + +fn should_descend(path: &Path) -> bool { + let Some(name) = path.file_name().and_then(|s| s.to_str()) else { + return false; + }; + if name.starts_with('.') { + return false; + } + !SKIP_DIR_NAMES.contains(&name) +} + +fn provenance_from_dir(dir: &Path) -> Option { + let pixi = dir.join(consts::WORKSPACE_MANIFEST); + let pyproject = dir.join(consts::PYPROJECT_MANIFEST); + let mojo = dir.join(consts::MOJOPROJECT_MANIFEST); + if pixi.is_file() { + Some(ManifestProvenance::new(pixi, ManifestKind::Pixi)) + } else if pyproject.is_file() { + Some(ManifestProvenance::new(pyproject, ManifestKind::Pyproject)) + } else if mojo.is_file() { + Some(ManifestProvenance::new(mojo, ManifestKind::MojoProject)) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write(path: &Path, contents: &str) { + if let Some(parent) = path.parent() { + fs_err::create_dir_all(parent).unwrap(); + } + fs_err::write(path, contents).unwrap(); + } + + /// Minimal valid `[workspace]` block for a member fixture. + fn member_workspace_toml(name: &str) -> String { + format!("[workspace]\nname = \"{name}\"\nchannels = []\nplatforms = []\n") + } + + fn member_workspace_with_task(name: &str, task_body: &str) -> String { + format!( + "[workspace]\nname = \"{name}\"\nchannels = []\nplatforms = []\n\n[tasks]\n{task_body}\n" + ) + } + + #[test] + fn empty_workspace_no_members() { + let tmp = tempfile::tempdir().unwrap(); + let tree = discover_members(tmp.path()).unwrap(); + assert!(tree.is_empty()); + } + + #[test] + fn discovers_top_level_members() { + let tmp = tempfile::tempdir().unwrap(); + write(&tmp.path().join("a/pixi.toml"), &member_workspace_toml("a")); + write(&tmp.path().join("b/pixi.toml"), &member_workspace_toml("b")); + + let tree = discover_members(tmp.path()).unwrap(); + let names: Vec<_> = tree.members().keys().cloned().collect(); + assert_eq!(names, vec!["a".to_string(), "b".to_string()]); + assert!(tree.resolve(["a"]).is_some()); + assert!(tree.resolve(["b"]).is_some()); + assert!(tree.resolve(["c"]).is_none()); + } + + #[test] + fn discovers_nested_members() { + let tmp = tempfile::tempdir().unwrap(); + write(&tmp.path().join("a/pixi.toml"), &member_workspace_toml("a")); + write( + &tmp.path().join("a/c/pixi.toml"), + &member_workspace_toml("c"), + ); + + let tree = discover_members(tmp.path()).unwrap(); + let a = tree.resolve(["a"]).expect("a should exist"); + assert!(a.children.contains_key("c")); + assert!(tree.resolve(["a", "c"]).is_some()); + } + + #[test] + fn intermediate_dir_without_manifest_is_transparent() { + let tmp = tempfile::tempdir().unwrap(); + write( + &tmp.path().join("b/c/pixi.toml"), + &member_workspace_toml("c"), + ); + + let tree = discover_members(tmp.path()).unwrap(); + assert!(tree.resolve(["c"]).is_some()); + assert!(tree.resolve(["b"]).is_none()); + } + + #[test] + fn package_only_manifest_is_transparent() { + // A manifest with only [package] and no [workspace] is NOT a + // member and discovery descends through it as if the manifest + // weren't there. + let tmp = tempfile::tempdir().unwrap(); + write( + &tmp.path().join("tools/pixi.toml"), + "[package]\nname = \"tools\"\nversion = \"0.1.0\"\n", + ); + write( + &tmp.path().join("tools/inner/pixi.toml"), + &member_workspace_toml("inner"), + ); + + let tree = discover_members(tmp.path()).unwrap(); + assert!(tree.resolve(["inner"]).is_some()); + assert!(tree.resolve(["tools"]).is_none()); + } + + #[test] + fn skips_common_build_and_hidden_dirs() { + let tmp = tempfile::tempdir().unwrap(); + write( + &tmp.path().join("target/pixi.toml"), + &member_workspace_toml("should_not_appear"), + ); + write( + &tmp.path().join(".pixi/pixi.toml"), + &member_workspace_toml("should_not_appear_either"), + ); + write( + &tmp.path().join("node_modules/pixi.toml"), + &member_workspace_toml("also_nope"), + ); + write( + &tmp.path().join("ok/pixi.toml"), + &member_workspace_toml("ok"), + ); + + let tree = discover_members(tmp.path()).unwrap(); + let names: Vec<_> = tree.members().keys().cloned().collect(); + assert_eq!(names, vec!["ok".to_string()]); + } + + #[test] + fn duplicate_sibling_name_errors() { + let tmp = tempfile::tempdir().unwrap(); + write( + &tmp.path().join("a/pixi.toml"), + &member_workspace_toml("same"), + ); + write( + &tmp.path().join("b/pixi.toml"), + &member_workspace_toml("same"), + ); + + let err = discover_members(tmp.path()).unwrap_err(); + assert!( + matches!(err, MemberDiscoveryError::DuplicateSibling { ref name, .. } if name == "same") + ); + } + + #[test] + fn nested_workspaces_are_expected() { + // A member may itself contain members with [workspace]. This is + // the expected Model-2 shape; no error is returned. + let tmp = tempfile::tempdir().unwrap(); + write(&tmp.path().join("a/pixi.toml"), &member_workspace_toml("a")); + write( + &tmp.path().join("a/c/pixi.toml"), + &member_workspace_toml("c"), + ); + let tree = discover_members(tmp.path()).unwrap(); + assert!(tree.resolve(["a", "c"]).is_some()); + } + + #[test] + fn deeply_nested_transparent_chain() { + let tmp = tempfile::tempdir().unwrap(); + write(&tmp.path().join("a/pixi.toml"), &member_workspace_toml("a")); + // mid has no manifest; c sits under a/mid/c + write( + &tmp.path().join("a/mid/c/pixi.toml"), + &member_workspace_toml("c"), + ); + + let tree = discover_members(tmp.path()).unwrap(); + assert!(tree.resolve(["a", "c"]).is_some()); + } + + #[test] + fn walk_yields_paths_in_insertion_order() { + let tmp = tempfile::tempdir().unwrap(); + write(&tmp.path().join("a/pixi.toml"), &member_workspace_toml("a")); + write( + &tmp.path().join("a/c/pixi.toml"), + &member_workspace_toml("c"), + ); + write(&tmp.path().join("b/pixi.toml"), &member_workspace_toml("b")); + + let tree = discover_members(tmp.path()).unwrap(); + let paths: Vec> = tree.walk().into_iter().map(|(p, _)| p).collect(); + assert_eq!( + paths, + vec![ + vec!["a".to_string()], + vec!["a".to_string(), "c".to_string()], + vec!["b".to_string()], + ] + ); + } + + #[test] + fn member_with_tasks_still_discovered_structurally() { + // Discovery doesn't parse or return tasks — task extraction + // happens during Workspace loading in pixi_core. This test just + // confirms a tasks block doesn't confuse discovery. + let tmp = tempfile::tempdir().unwrap(); + write( + &tmp.path().join("a/pixi.toml"), + &member_workspace_with_task("a", "greet = \"echo hi\""), + ); + + let tree = discover_members(tmp.path()).unwrap(); + assert!(tree.resolve(["a"]).is_some()); + } +} diff --git a/crates/pixi_manifest/src/preview.rs b/crates/pixi_manifest/src/preview.rs index 349302071a..7b3425c064 100644 --- a/crates/pixi_manifest/src/preview.rs +++ b/crates/pixi_manifest/src/preview.rs @@ -73,4 +73,6 @@ impl Preview { pub enum KnownPreviewFeature { /// Build feature, to enable conda source builds PixiBuild, + /// Hierarchical task addressing across nested member packages + HierarchicalTasks, } diff --git a/crates/pixi_task/Cargo.toml b/crates/pixi_task/Cargo.toml index ef08b77f85..ec9ae9e5fc 100644 --- a/crates/pixi_task/Cargo.toml +++ b/crates/pixi_task/Cargo.toml @@ -14,6 +14,7 @@ anyhow = { workspace = true } assert_matches = { workspace = true } crossbeam-channel = { workspace = true } deno_task_shell = { workspace = true } +dunce = { workspace = true } fancy_display = { workspace = true } fs-err = { workspace = true } itertools = { workspace = true } diff --git a/crates/pixi_task/src/executable_task.rs b/crates/pixi_task/src/executable_task.rs index 856fe405e3..f453efb136 100644 --- a/crates/pixi_task/src/executable_task.rs +++ b/crates/pixi_task/src/executable_task.rs @@ -89,6 +89,11 @@ pub enum CanSkip { /// A task that contains enough information to be able to execute it. The /// lifetime `'p` refers to the lifetime of the project that contains the /// tasks. +/// +/// For hierarchical-tasks member tasks, `workspace` is the **member's** +/// workspace (sourced from `run_environment.workspace()`), not the root +/// aggregator's. That means activation, lockfile access, env install dir, +/// and the default `cwd` all correctly target the member. #[derive(Clone, Debug)] pub struct ExecutableTask<'p> { pub workspace: &'p Workspace, @@ -101,6 +106,11 @@ pub struct ExecutableTask<'p> { impl<'p> ExecutableTask<'p> { /// Constructs a new executable task from a task graph node. + /// + /// The resulting task's `workspace` is taken from the node's + /// `run_environment` — not from the graph's root project — so that + /// member tasks execute against their own member workspace instead of + /// the aggregator root. pub fn from_task_graph( task_graph: &TaskGraph<'p>, task_id: TaskId, @@ -109,7 +119,7 @@ impl<'p> ExecutableTask<'p> { let node = &task_graph[task_id]; Self { - workspace: task_graph.project(), + workspace: node.run_environment.workspace(), name: node.name.clone(), task: node.task.clone(), run_environment: node.run_environment.clone(), @@ -287,6 +297,11 @@ impl<'p> ExecutableTask<'p> { } /// Returns the working directory for this task. + /// + /// For member tasks under the `hierarchical-tasks` preview feature, + /// `self.workspace` is already the member's workspace (see + /// [`Self::from_task_graph`]), so `workspace.root()` naturally yields + /// the member's directory — no extra plumbing required. pub fn working_directory(&self) -> Result { Ok(match self.task.working_directory() { Some(cwd) if cwd.is_absolute() => cwd.to_path_buf(), diff --git a/crates/pixi_task/src/task_environment.rs b/crates/pixi_task/src/task_environment.rs index 5c9b3cc7a8..be9c80f627 100644 --- a/crates/pixi_task/src/task_environment.rs +++ b/crates/pixi_task/src/task_environment.rs @@ -9,6 +9,30 @@ use thiserror::Error; use crate::error::{AmbiguousTaskError, MissingTaskError}; +/// The separator used between member path segments and the task name in +/// qualified task addresses, e.g. `member_a::build` or `a::c::test`. +pub const MEMBER_TASK_SEPARATOR: &str = "::"; + +/// Splits a task name on [`MEMBER_TASK_SEPARATOR`]. Returns `Some((member_path, +/// task_name))` when the input contains at least one separator, otherwise +/// `None`. +/// +/// Examples: +/// - `"build"` → `None` +/// - `"a::build"` → `Some((vec!["a"], "build"))` +/// - `"a::c::test"` → `Some((vec!["a", "c"], "test"))` +pub fn parse_qualified_task_name(s: &str) -> Option<(Vec<&str>, &str)> { + if !s.contains(MEMBER_TASK_SEPARATOR) { + return None; + } + let mut parts: Vec<&str> = s.split(MEMBER_TASK_SEPARATOR).collect(); + if parts.len() < 2 { + return None; + } + let task = parts.pop().expect("checked len >= 2"); + Some((parts, task)) +} + /// Defines where the task was defined when looking for a task. #[derive(Debug, Clone)] pub enum FindTaskSource<'p> { @@ -122,6 +146,41 @@ impl<'p, D: TaskDisambiguation<'p>> SearchEnvironments<'p, D> { source: FindTaskSource<'p>, task_specific_environment: Option>, ) -> Result, FindTaskError> { + // `member::task` / `a::b::task` addressing for the hierarchical-tasks + // preview feature (Model 2 — federated member workspaces). + // + // If the name contains `::` and the first segment matches a known + // top-level member, we resolve the member path to the member's + // standalone Workspace and dispatch the lookup into that member's + // **own** default environment. The returned `Environment` carries + // a reference to the member workspace — so downstream task + // execution (activation, lockfile, install dir) naturally targets + // the member, not the root. + // + // Names without `::`, or with a first segment that isn't a + // member, fall through to the normal task search below — + // preserving backwards compatibility for any task name that + // happens to contain `::`. + if let Some((member_path, task_name)) = parse_qualified_task_name(name.as_str()) + && self.project.members().contains_key(member_path[0]) + && let Some(member_ws) = self.project.resolve_member(member_path.iter().copied()) + { + let member_env = member_ws.default_environment(); + let task_name_lookup = TaskName::from(task_name); + match member_env.task(&task_name_lookup, self.platform) { + Ok(task) => return Ok((member_env, task)), + Err(_) => { + // Member path resolves but no task with that name. + // Surface a MissingTask error tied to the + // fully-qualified address so the user sees exactly + // what we searched for. + return Err(FindTaskError::MissingTask(MissingTaskError { + task_name: name, + })); + } + } + } + // If no explicit environment was specified if self.explicit_environment.is_none() && task_specific_environment.is_none() { let default_env = self.project.default_environment(); @@ -227,6 +286,8 @@ impl<'p, D: TaskDisambiguation<'p>> SearchEnvironments<'p, D> { mod tests { use std::path::Path; + use pixi_core::workspace::HasWorkspaceRef; + use super::*; #[test] @@ -488,4 +549,162 @@ mod tests { .expect("should resolve to an environment"); assert_eq!(result.0.name().as_str(), "test"); } + + // ---- Hierarchical-tasks (`a::b::task`) end-to-end routing tests ---- + + /// Writes a workspace root + member layout used by the hierarchical-tasks + /// integration tests and returns the tempdir guard (drop = cleanup). + /// + /// Under Model 2 every member has its own `[workspace]` block — each + /// is a fully standalone pixi project. The root's role is purely to + /// aggregate so `a::c::test` resolves through the member tree. + fn build_hierarchical_fixture(preview_on: bool) -> tempfile::TempDir { + let tmp = tempfile::tempdir().unwrap(); + let preview_line = if preview_on { + "preview = [\"hierarchical-tasks\"]" + } else { + "preview = []" + }; + fs_err::write( + tmp.path().join("pixi.toml"), + format!( + "[workspace]\nname = \"ht-root\"\nchannels = []\nplatforms = [\"linux-64\", \"osx-64\", \"osx-arm64\", \"win-64\"]\n{preview_line}\n\n[tasks]\ngreet = \"echo hi\"\nall_tests = {{ depends-on = [\"a::test\", \"a::c::test\", \"b::test\"] }}\n" + ), + ) + .unwrap(); + + for (rel, name, task) in [ + ("a", "a", "echo a"), + ("b", "b", "echo b"), + ("a/c", "c", "echo c"), + ] { + let dir = tmp.path().join(rel); + fs_err::create_dir_all(&dir).unwrap(); + fs_err::write( + dir.join("pixi.toml"), + format!( + "[workspace]\nname = \"{name}\"\nchannels = []\nplatforms = [\"linux-64\", \"osx-64\", \"osx-arm64\", \"win-64\"]\n\n[tasks]\ntest = \"{task}\"\n" + ), + ) + .unwrap(); + } + tmp + } + + fn locate_workspace(root: &Path) -> Workspace { + pixi_core::WorkspaceLocator::for_cli() + .with_consider_environment(false) + .with_emit_warnings(false) + .with_search_start(pixi_core::workspace::DiscoveryStart::SearchRoot( + root.to_path_buf(), + )) + .locate() + .expect("workspace should locate") + } + + #[test] + fn qualified_task_routes_to_member() { + let tmp = build_hierarchical_fixture(true); + let project = locate_workspace(tmp.path()); + let search = SearchEnvironments::from_opt_env(&project, None, None); + + let (env, task) = search + .find_task("a::test".into(), FindTaskSource::CmdArgs, None) + .expect("a::test must resolve"); + // Model 2: the returned Environment belongs to the **member's** + // workspace, not the root. Verify by comparing workspace roots. + let expected_root = dunce::canonicalize(tmp.path().join("a")).unwrap(); + assert_eq!( + env.workspace().root(), + expected_root, + "member tasks must run in the member's own workspace" + ); + assert!( + env.name().is_default(), + "member task should run in the member's default env" + ); + // Confirm we routed to the member's task, not the workspace's. + // The member's `test` command is "echo a"; the root has no `test`. + use pixi_manifest::task::CmdArgs; + match task.as_command() { + Some(CmdArgs::Single(s)) => assert_eq!(s.source(), "echo a"), + other => panic!("expected `echo a`, got {other:?}"), + } + } + + #[test] + fn nested_qualified_task_routes_to_grandchild() { + let tmp = build_hierarchical_fixture(true); + let project = locate_workspace(tmp.path()); + let search = SearchEnvironments::from_opt_env(&project, None, None); + + let (env, task) = search + .find_task("a::c::test".into(), FindTaskSource::CmdArgs, None) + .expect("a::c::test must resolve"); + // The returned env must belong to the inner member `a/c`, not to + // `a` or the root. + let expected_root = dunce::canonicalize(tmp.path().join("a/c")).unwrap(); + assert_eq!(env.workspace().root(), expected_root); + use pixi_manifest::task::CmdArgs; + match task.as_command() { + Some(CmdArgs::Single(s)) => assert_eq!(s.source(), "echo c"), + other => panic!("expected `echo c`, got {other:?}"), + } + } + + #[test] + fn qualified_task_with_unknown_leaf_returns_missing() { + let tmp = build_hierarchical_fixture(true); + let project = locate_workspace(tmp.path()); + let search = SearchEnvironments::from_opt_env(&project, None, None); + + let err = search + .find_task("a::does_not_exist".into(), FindTaskSource::CmdArgs, None) + .expect_err("unknown leaf task must fail"); + assert!(matches!(err, FindTaskError::MissingTask(_))); + } + + #[test] + fn unqualified_task_still_works_with_preview_on() { + let tmp = build_hierarchical_fixture(true); + let project = locate_workspace(tmp.path()); + let search = SearchEnvironments::from_opt_env(&project, None, None); + + // Root has `greet`; should resolve against the workspace as normal. + let (env, _task) = search + .find_task("greet".into(), FindTaskSource::CmdArgs, None) + .expect("root task must still resolve"); + assert!(env.name().is_default()); + } + + #[test] + fn qualified_task_is_unknown_when_preview_off() { + let tmp = build_hierarchical_fixture(false); + let project = locate_workspace(tmp.path()); + let search = SearchEnvironments::from_opt_env(&project, None, None); + + // Preview off → member tree is empty, so `a::test` falls through + // to the normal task search and reports a missing task (there is + // no root task literally named `a::test`). + let err = search + .find_task("a::test".into(), FindTaskSource::CmdArgs, None) + .expect_err("preview-off must not resolve member tasks"); + assert!(matches!(err, FindTaskError::MissingTask(_))); + } + + #[test] + fn parse_qualified_task_name_edge_cases() { + assert_eq!(parse_qualified_task_name("build"), None); + assert_eq!( + parse_qualified_task_name("a::build"), + Some((vec!["a"], "build")) + ); + assert_eq!( + parse_qualified_task_name("a::b::c::t"), + Some((vec!["a", "b", "c"], "t")) + ); + // `::foo` splits to ["", "foo"]: degenerate but handled — caller + // will fail at member-resolve time since `""` isn't a member. + assert_eq!(parse_qualified_task_name("::foo"), Some((vec![""], "foo"))); + } } diff --git a/crates/pixi_task/src/task_graph.rs b/crates/pixi_task/src/task_graph.rs index 8506f74503..2282422146 100644 --- a/crates/pixi_task/src/task_graph.rs +++ b/crates/pixi_task/src/task_graph.rs @@ -79,7 +79,11 @@ pub struct TaskNode<'p> { /// The name of the task or `None` if the task is a custom task. pub name: Option, - /// The environment to run the task in + /// The environment to run the task in. For hierarchical-tasks member + /// tasks (`a::b::task`), this `Environment` belongs to the **member's** + /// workspace — so `run_environment.workspace().root()` is the member's + /// directory, and all downstream execution (cwd, activation, lockfile + /// path, install dir) automatically targets that member. pub run_environment: Environment<'p>, /// A reference to a project task, or a owned custom task. @@ -173,7 +177,11 @@ impl TaskNode<'_> { /// different executable tasks. #[derive(Debug)] pub struct TaskGraph<'p> { - /// The project that this graph references + /// The project this graph was built from. Under the `hierarchical-tasks` + /// preview feature, individual nodes may point at different member + /// workspaces via their `run_environment.workspace()` — `project` is + /// the outermost (aggregator) workspace only. + #[allow(dead_code)] project: &'p Workspace, /// The tasks in the graph @@ -199,6 +207,7 @@ impl<'p> Index for TaskGraph<'p> { } impl<'p> TaskGraph<'p> { + #[allow(dead_code)] pub(crate) fn project(&self) -> &'p Workspace { self.project } @@ -304,11 +313,13 @@ impl<'p> TaskGraph<'p> { Some(ArgValues::FreeFormArgs(free_args)) }; + let task_name_owned: TaskName = task_name.clone().into(); + if skip_deps { return Ok(Self { project, nodes: vec![TaskNode { - name: Some(task_name.into()), + name: Some(task_name_owned.clone()), task: Cow::Borrowed(task), run_environment: run_env, args: arg_values, @@ -321,7 +332,7 @@ impl<'p> TaskGraph<'p> { project, search_envs, TaskNode { - name: Some(task_name.into()), + name: Some(task_name_owned), task: Cow::Borrowed(task), run_environment: run_env, args: arg_values, diff --git a/docs/workspace/hierarchical_tasks.md b/docs/workspace/hierarchical_tasks.md new file mode 100644 index 0000000000..dcf23549b5 --- /dev/null +++ b/docs/workspace/hierarchical_tasks.md @@ -0,0 +1,199 @@ +# Hierarchical tasks + +!!! warning "Preview feature" + Hierarchical tasks are currently a **preview** feature and may change. + Opt in by enabling the `hierarchical-tasks` preview flag in your + workspace's `pixi.toml`: + + ```toml + [workspace] + # ... other workspace fields ... + preview = ["hierarchical-tasks"] + + ``` + + + With the flag disabled, everything in this page is a no-op and your + workspace behaves exactly as before. + +## Why + +When you have a monorepo of related projects — or a project that pulls +in other projects as git submodules — you often want to: + +- Keep each sub-project runnable on its own (`pixi run test` inside a + sub-project behaves normally, with no dependency on the outer + repository). +- Aggregate the sub-projects from the outer root — list their tasks, + depend on them, and invoke them without `cd`-ing. + +The `hierarchical-tasks` preview feature adds this aggregation layer. +The design is inspired by the proposal in +[issue #5003](https://github.com/prefix-dev/pixi/issues/5003) (especially +the comments from user `phuicy`) and mirrors the way git submodules and +CMake's `add_subdirectory()` compose a larger project from smaller ones. + +## How members are defined + +A **member** is a subdirectory of your workspace whose manifest contains +its own `[workspace]` block (or `[tool.pixi.workspace]` for pyproject). +Each member is a **fully standalone pixi workspace**: + +- Its own environments, channels, platforms. +- Its own lockfile (`/pixi.lock`). +- Its own install directory (`/.pixi/envs/...`). + +Because a member _is_ a workspace, you can `cd` into it and run `pixi +run test`, `pixi install`, `pixi shell`, etc. — all the usual pixi +commands — without any dependence on the outer aggregation root. + +Manifests with no `[workspace]` block (for example, a `pyproject.toml` +that is only a Python package, or a pixi manifest that only declares +`[package]`) are **transparent** — discovery walks right through them +as if they weren't there and keeps searching deeper. + +## Discovery rules + +When the preview flag is enabled, pixi walks the workspace directory +tree downward from the root manifest: + +- Members form a **tree**. A member discovered inside another member + becomes a child of the outer member. Intermediate directories that + contain no member manifest are transparent. +- Within a given parent, member names must be **unique** (same rule as + directory names on a filesystem). Two different parents may contain + members with the same name — their full paths differ, so their task + addresses differ too. +- Common build and cache directories are skipped: `.pixi`, `.git`, any + `.`-prefixed directory, `target`, `node_modules`, `dist`, `build`, + `venv`, `__pycache__`. + +## Which workspace is "the root"? — nearest-ancestor wins + +**Pixi's upward discovery rule is unchanged: the root workspace is the +nearest-ancestor `[workspace]` from your current directory.** The only +thing Model 2 changes is that a root may now _aggregate_ member +workspaces below it. + +This means what you see depends on where you stand: + +| CWD | Root | Addressable tasks | +| ------------ | ------------------- | ----------------------------------------------------------- | +| `/repo` | `/repo/pixi.toml` | `greet`, `all_tests`, `a::test`, `a::c::test`, `b::test` | +| `/repo/a` | `/repo/a/pixi.toml` | `test`, `c::test` (if `a` has the preview flag on) | +| `/repo/a/c` | `/repo/a/c/pixi.toml` | `test` | + +This is the **same mental model as git submodules**: `git status` inside +a submodule reports that submodule's state, not the outer repository's. +If you want the outer aggregation visible, run your pixi command from +the outer root. + +## Running a member task + +Address any member task using the `::` separator. Paths can be as deep +as the tree allows: + +```console +$ pixi run a::test # runs `test` in member `a` +$ pixi run a::c::test # runs `test` in member `c`, a child of `a` +``` + +Plain task names (no `::`) resolve against the **root** workspace's +tasks — the one you're currently in. A name with `::` that doesn't +match a known top-level member also falls through to the normal task +search, so task names that happen to contain `::` keep working. + +### Working directory + +Because each member is its own workspace, a member task's working +directory defaults to the member's own directory. An explicit `cwd:` +declared on the task is resolved relative to the member's directory, +not the aggregation root's. + +### Environment + +A member task runs in the **member's own default environment** — using +the member's lockfile and its `.pixi/envs/...` install dir. This is the +key difference from plain pixi: different tasks in a single `pixi run` +invocation may execute in different workspaces, each with its own +activated environment. + +## Cross-member dependencies + +A task in the root may declare `depends-on` entries that reference +members using the same `::` syntax: + +```toml +# /pixi.toml +[tasks] +all_tests = { depends-on = ["a::test", "a::c::test", "b::test"] } +``` + +Running `pixi run all_tests` resolves each dependency to its own member +workspace and runs it there, in member order. Cross-member cycle +detection applies across the whole graph. + +## Listing tasks + +`pixi task list` surfaces all reachable member tasks under their +fully-qualified names alongside the usual workspace tasks: + +```console +$ pixi task list +Tasks that can run on this machine: +----------------------------------- +a::c::test, a::test, all_tests, b::test, greet +``` + +## Installing environments + +Install timing differs between `pixi run` and `pixi install`: + +- **`pixi run a::test` is lazy.** Only the member actually referenced + by the task (and its dependencies, if any) gets its lockfile updated + and its environment installed. Other members are untouched. + +- **`pixi install` at the root is eager.** It walks every reachable + member workspace and installs each one's default environment, writing + a `pixi.lock` in each. This is useful for a cold checkout where you + want everything ready in one go. + +## Example + +```text +my-workspace/ +├── pixi.toml # [workspace] preview = ["hierarchical-tasks"] +│ # [tasks] all_tests = { depends-on = [...] } +├── a/ +│ ├── pixi.toml # [workspace] name = "a" + [tasks] +│ └── c/ +│ └── pixi.toml # [workspace] name = "c" + [tasks] +└── b/ + └── pixi.toml # [workspace] name = "b" + [tasks] +``` + +```console +$ pixi run a::test # from my-workspace: runs in a/ +$ pixi run a::c::test # from my-workspace: runs in a/c/ +$ pixi run all_tests # from my-workspace: runs all three, each in its own workspace + +$ cd a/c +$ pixi run test # standalone: upward walk stops at a/c/pixi.toml; + # outer aggregation is invisible from inside +``` + +## Scope and limitations + +This preview covers the **task layer**: + +- **Yes**: downward member discovery, `a::b::task` addressing, + cross-member `depends-on`, per-member working directory, per-member + lockfiles and install dirs, `pixi task list` showing the full tree, + `pixi install` eagerly installing every member. +- **Not yet**: merging dependencies across members into a single solve, + a unified lockfile spanning members, public/private dependency scope + between members, `--package`/`-w` flag for addressing, `pixi task +add a::new_task` (edit the member's `pixi.toml` directly). + +These are tracked as follow-ups on +[issue #5003](https://github.com/prefix-dev/pixi/issues/5003). diff --git a/mkdocs.yml b/mkdocs.yml index d96c514529..2814d5265c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -128,6 +128,7 @@ nav: - Concepts: - Environments: workspace/environment.md - Tasks: workspace/advanced_tasks.md + - Hierarchical Tasks (preview): workspace/hierarchical_tasks.md - Multi Platform: workspace/multi_platform_configuration.md - Multi Environment: workspace/multi_environment.md - Lock File: workspace/lockfile.md