diff --git a/docs/wiki/Projects.md b/docs/wiki/Projects.md index edf41b9..a20bb1e 100644 --- a/docs/wiki/Projects.md +++ b/docs/wiki/Projects.md @@ -43,6 +43,10 @@ prefix = ".venv" # optional: install location (default ".ven `version` accepts a label: `latest`, an exact version (`1.3.0`), or a range (`>=1.2,<2`). +Two optional sub-tables, `[tool.nepenthe.package-mappings]` and +`[tool.nepenthe.extras-mappings]`, adjust how PyPI requirement names are matched +against conda packages — see [Overriding the name map](#overriding-the-name-map). + ## `sync` — install the referenced environment ```bash @@ -120,6 +124,39 @@ regenerated from the upstream source with a single command — see occasional `missing` as "couldn't confirm" rather than "definitely absent" for an obscure or very new package. +### Overriding the name map + +The vendored map only knows what conda-forge knows. A project that depends on +distributions packaged elsewhere — or that packages its own — can say how those +names translate, without waiting for the vendored table to be regenerated: + +```toml +[tool.nepenthe.package-mappings] +example-package = "example-conda-package" + +[tool.nepenthe.extras-mappings] +"example-package[extra]" = ["example-package", "example-package-extra"] +``` + +`package-mappings` renames a distribution, taking precedence over the vendored +entry (or supplying one where there is none). + +`extras-mappings` handles a case a rename cannot express. conda has no +equivalent of a PyPI extras group, so a distribution that splits its optional +features into separate conda packages maps *one* requirement onto *several* +packages. `check` then requires every one of them: the requirement is **ok** +only when all are present and satisfy its version specifier, and reports the +first that is **missing** or in **conflict**. `nepenthe try` expands the same +way, adding one conda spec per package. + +Keys are normalized, so casing, separator style and the order of extras in the +key do not matter. An extras group with no mapping resolves to the distribution +name alone — the same result the requirement would have produced without the +group. + +Both tables also apply to `nepenthe try --project`, which reads them without +requiring the rest of the `[tool.nepenthe]` stanza. + ### Updating the name map The vendored table lives at `rust/src/data/pypi_to_conda.tsv` and is reproducible diff --git a/rust/python/lib.rs b/rust/python/lib.rs index d0b2292..8562d6c 100644 --- a/rust/python/lib.rs +++ b/rust/python/lib.rs @@ -604,7 +604,8 @@ fn try_solve<'py>( let mut specs = with_.unwrap_or_default(); if let Some(path) = &project { let deps = project::read_dependencies(path).map_err(err)?; - specs.extend(project::requirements_to_conda_specs(&deps)); + let overrides = project::read_name_overrides(path).map_err(err)?; + specs.extend(project::requirements_to_conda_specs(&deps, &overrides)); } if specs.is_empty() { return Err(PyValueError::new_err( diff --git a/rust/src/cli.rs b/rust/src/cli.rs index 8b2f3ff..8c8cb28 100644 --- a/rust/src/cli.rs +++ b/rust/src/cli.rs @@ -1356,7 +1356,10 @@ async fn try_solve(args: TryArgs) -> CliResult { let mut specs = args.with.clone(); if let Some(project) = &args.project { let deps = crate::project::read_dependencies(project)?; - specs.extend(crate::project::requirements_to_conda_specs(&deps)); + let overrides = crate::project::read_name_overrides(project)?; + specs.extend(crate::project::requirements_to_conda_specs( + &deps, &overrides, + )); } if specs.is_empty() { return Err("nothing to try: pass --with and/or --project ".into()); diff --git a/rust/src/name_map.rs b/rust/src/name_map.rs index 4ae6144..427ad78 100644 --- a/rust/src/name_map.rs +++ b/rust/src/name_map.rs @@ -52,6 +52,108 @@ pub fn len() -> usize { table().len() } +/// The lookup key for a requirement that carries an extras group: the +/// normalized distribution name followed by its sorted, normalized extras. +pub fn extras_key(name: &str, extras: &[String]) -> String { + let mut extras: Vec = extras.iter().map(|e| normalize_name(e)).collect(); + extras.sort(); + extras.dedup(); + format!("{}[{}]", normalize_name(name), extras.join(",")) +} + +/// Project-supplied additions to the vendored table. +/// +/// The vendored table covers what conda-forge knows about. A project that +/// depends on distributions packaged outside conda-forge — or that packages its +/// own — needs to say how those names translate, without waiting for the +/// vendored table to be regenerated. +/// +/// Two kinds of override are supported: +/// +/// - **package**: a distribution name maps to a different conda package name, +/// overriding the vendored entry (or supplying one where there is none). +/// - **extras**: a requirement's extras group maps to *several* conda packages. +/// conda has no equivalent of an extras group, so a distribution that splits +/// its optional features into separate conda packages cannot be expressed as +/// a rename. +/// +/// An extras group with no override resolves to the distribution name alone, +/// which is what the extras-less requirement would have produced. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Overrides { + packages: BTreeMap, + extras: BTreeMap>, +} + +impl Overrides { + /// Build from raw (un-normalized) `package` and `extras` tables. `packages` + /// is keyed by distribution name; `extras` by a `name[extra,…]` string. + /// Keys are normalized, so casing, separator style and extras order in the + /// source are irrelevant. An `extras` key without a `[…]` group is ignored. + pub fn new( + packages: impl IntoIterator, + extras: impl IntoIterator)>, + ) -> Self { + let packages = packages + .into_iter() + .map(|(pypi, conda)| (normalize_name(&pypi), conda)) + .collect(); + let extras = extras + .into_iter() + .filter_map(|(key, conda)| Some((normalize_extras_key(&key)?, conda))) + .collect(); + Self { packages, extras } + } + + /// Whether any override is configured. + pub fn is_empty(&self) -> bool { + self.packages.is_empty() && self.extras.is_empty() + } + + /// The conda package name(s) a requirement resolves to. `extras` is the + /// requirement's extras group, empty when it has none. + /// + /// An extras override wins; otherwise a package override; otherwise the + /// vendored table; otherwise the normalized name itself. + pub fn conda_names(&self, pypi_name: &str, extras: &[String]) -> Vec { + if !extras.is_empty() { + if let Some(names) = self.extras.get(&extras_key(pypi_name, extras)) { + return names.clone(); + } + } + vec![self.conda_name(pypi_name)] + } + + /// The single conda package name a distribution name resolves to, ignoring + /// any extras group. + pub fn conda_name(&self, pypi_name: &str) -> String { + let normalized = normalize_name(pypi_name); + if let Some(conda) = self.packages.get(&normalized) { + return conda.clone(); + } + pypi_to_conda(&normalized) + .map(str::to_string) + .unwrap_or(normalized) + } +} + +/// Normalize a `name[extra,…]` key. Returns `None` when there is no extras +/// group, or when the group is empty or unterminated. +fn normalize_extras_key(key: &str) -> Option { + let (name, rest) = key.split_once('[')?; + let extras = rest.strip_suffix(']')?; + let extras: Vec = extras + .split(',') + .map(str::trim) + .filter(|e| !e.is_empty()) + .map(str::to_string) + .collect(); + if name.trim().is_empty() || extras.is_empty() { + return None; + } + Some(extras_key(name.trim(), &extras)) +} + fn parse_tsv(text: &str) -> BTreeMap { text.lines() .filter_map(|line| { @@ -173,6 +275,72 @@ mod tests { assert!(len() > 50); } + #[test] + fn overrides_default_to_the_vendored_table() { + let overrides = Overrides::default(); + assert!(overrides.is_empty()); + assert_eq!(overrides.conda_name("opencv-python"), "opencv"); + assert_eq!(overrides.conda_name("numpy"), "numpy"); + assert_eq!(overrides.conda_name("Flask_SQLAlchemy"), "flask-sqlalchemy"); + } + + #[test] + fn a_package_override_wins_over_the_vendored_table() { + let overrides = Overrides::new( + [("OpenCV_Python".to_string(), "example-opencv".to_string())], + [], + ); + assert!(!overrides.is_empty()); + // The key is normalized, so any spelling of the requirement matches. + assert_eq!(overrides.conda_name("opencv-python"), "example-opencv"); + } + + #[test] + fn an_extras_override_expands_to_several_packages() { + let overrides = Overrides::new( + [], + [( + "Example.Package[Two, One]".to_string(), + vec![ + "example-package".to_string(), + "example-package-extra".to_string(), + ], + )], + ); + // Extras order and separator style in the key do not matter. + assert_eq!( + overrides.conda_names("example-package", &["one".to_string(), "two".to_string()]), + vec!["example-package", "example-package-extra"] + ); + // A different extras group is not the same key. + assert_eq!( + overrides.conda_names("example-package", &["one".to_string()]), + vec!["example-package"] + ); + } + + #[test] + fn an_unmapped_extras_group_resolves_to_the_name_alone() { + let overrides = Overrides::default(); + assert_eq!( + overrides.conda_names("opencv-python", &["extra".to_string()]), + vec!["opencv"] + ); + } + + #[test] + fn extras_keys_without_a_group_are_ignored() { + let overrides = Overrides::new( + [], + [ + ("example-package".to_string(), vec!["a".to_string()]), + ("example-package[]".to_string(), vec!["b".to_string()]), + ("[extra]".to_string(), vec!["c".to_string()]), + ], + ); + assert!(overrides.is_empty()); + } + #[test] fn reduce_keeps_only_divergent_pairs() { let yaml = r#" diff --git a/rust/src/project.rs b/rust/src/project.rs index 7430dbe..30f71f7 100644 --- a/rust/src/project.rs +++ b/rust/src/project.rs @@ -117,6 +117,14 @@ pub struct ProjectRef { /// Prefix to install into (defaults to `.venv`). #[serde(default)] pub prefix: Option, + /// Conda package name to use for a PyPI distribution name, overriding the + /// [vendored mapping](crate::name_map). + #[serde(default, rename = "package-mappings")] + pub package_mappings: std::collections::BTreeMap, + /// Conda packages a requirement's extras group expands to, keyed by + /// `name[extra,…]`. + #[serde(default, rename = "extras-mappings")] + pub extras_mappings: std::collections::BTreeMap>, } impl ProjectRef { @@ -151,6 +159,11 @@ impl ProjectRef { fn registry(&self) -> Registry { Registry::new(SpecStore::new(), self.registry.clone()) } + + /// The project's additions to the vendored PyPI→conda name mapping. + pub fn name_overrides(&self) -> name_map::Overrides { + name_map::Overrides::new(self.package_mappings.clone(), self.extras_mappings.clone()) + } } /// A parsed `pyproject.toml`: the `[tool.nepenthe]` reference plus the project's @@ -227,6 +240,19 @@ pub fn read_dependencies(pyproject: &Path) -> Result, ProjectError> Ok(raw.project.map(|p| p.dependencies).unwrap_or_default()) } +/// Read the name-mapping overrides from a `pyproject.toml`, without requiring a +/// `[tool.nepenthe]` stanza. Returns the empty set when there is none, so `try` +/// honours a project's overrides but does not demand the stanza. +pub fn read_name_overrides(pyproject: &Path) -> Result { + let text = std::fs::read_to_string(pyproject)?; + let raw: RawPyProject = toml::from_str(&text).map_err(|e| ProjectError::Toml(e.to_string()))?; + Ok(raw + .tool + .and_then(|t| t.nepenthe) + .map(|n| n.name_overrides()) + .unwrap_or_default()) +} + /// Install (or update) the environment referenced by a project into its prefix, /// resolving the version label against the registry. Performs network I/O; await /// inside a tokio runtime. @@ -336,8 +362,14 @@ impl CheckReport { /// When a PyPI name differs from its conda counterpart (e.g. `opencv-python` vs /// `opencv`), the [grayskull-derived mapping](crate::name_map) is consulted so /// the dependency still resolves; only names absent under both spellings report -/// as [`Missing`](DependencyStatus::Missing). -pub fn check_dependencies(dependencies: &[String], packages: &[PackageId]) -> CheckReport { +/// as [`Missing`](DependencyStatus::Missing). `overrides` takes precedence over +/// the vendored mapping, and can expand a requirement's extras group into +/// several conda packages, all of which must then be present and satisfied. +pub fn check_dependencies( + dependencies: &[String], + packages: &[PackageId], + overrides: &name_map::Overrides, +) -> CheckReport { let by_name: std::collections::BTreeMap = packages .iter() .map(|p| (normalize_name(&p.name), p)) @@ -349,10 +381,9 @@ pub fn check_dependencies(dependencies: &[String], packages: &[PackageId]) -> Ch None => DependencyStatus::Skipped { reason: "not a name+version requirement (direct URL or unparseable)".to_string(), }, - Some((name, specifier)) => match resolve_package(&name, &by_name) { - None => DependencyStatus::Missing { name }, - Some(package) => check_version(name, &specifier, &package.version), - }, + Some((name, extras, specifier)) => { + check_resolved(&overrides.conda_names(&name, &extras), &specifier, &by_name) + } }; checked.push(CheckedDependency { requirement: requirement.clone(), @@ -364,17 +395,27 @@ pub fn check_dependencies(dependencies: &[String], packages: &[PackageId]) -> Ch } } -/// Resolve a normalized requirement name to an environment package: first by a -/// direct name match, then via the PyPI→conda [name mapping](crate::name_map). -fn resolve_package<'a>( - name: &str, - by_name: &std::collections::BTreeMap, -) -> Option<&'a PackageId> { - if let Some(package) = by_name.get(name) { - return Some(package); - } - let conda = name_map::pypi_to_conda(name)?; - by_name.get(&normalize_name(conda)).copied() +/// Check every conda package a requirement resolved to. The requirement is +/// satisfied only when all of them are; the first failure is reported. +fn check_resolved( + names: &[String], + specifier: &str, + by_name: &std::collections::BTreeMap, +) -> DependencyStatus { + let mut last = DependencyStatus::Skipped { + reason: "requirement resolved to no conda package".to_string(), + }; + for name in names { + let normalized = normalize_name(name); + let Some(package) = by_name.get(&normalized) else { + return DependencyStatus::Missing { name: normalized }; + }; + last = check_version(normalized, specifier, &package.version); + if !matches!(last, DependencyStatus::Satisfied { .. }) { + return last; + } + } + last } /// Pull the environment's lock from the registry and check the project's @@ -394,7 +435,11 @@ pub async fn check( let bytes = registry.pull(&coords, &reference.label())?; let lock = install::parse_lock(&bytes)?; let packages = install::lock_packages(&lock, &reference.environment, &coords.platform)?; - Ok(check_dependencies(&project.dependencies, &packages)) + Ok(check_dependencies( + &project.dependencies, + &packages, + &reference.name_overrides(), + )) } /// Test a pinned version against a requirement's specifier. @@ -429,10 +474,10 @@ fn check_version(name: String, specifier: &str, found: &str) -> DependencyStatus } } -/// Extract a `(normalized name, version specifier)` from a PEP 508 requirement. -/// Returns `None` for direct-reference (`name @ url`) or unparseable entries. -/// Environment markers (`; python_version …`) and extras (`[extra]`) are dropped. -fn parse_requirement(requirement: &str) -> Option<(String, String)> { +/// Extract a requirement's normalized name, extras group and version specifier +/// from a PEP 508 string. Returns `None` for direct-reference (`name @ url`) or +/// unparseable entries. Environment markers (`; python_version …`) are dropped. +fn parse_requirement(requirement: &str) -> Option<(String, Vec, String)> { // Drop any environment marker. let base = requirement.split(';').next().unwrap_or("").trim(); if base.is_empty() { @@ -451,38 +496,50 @@ fn parse_requirement(requirement: &str) -> Option<(String, String)> { return None; } let rest = base[name_end..].trim(); - // Drop an optional extras group (`[extra1,extra2]`) before the specifier. - let specifier = if let Some(after_open) = rest.strip_prefix('[') { + // Split off an optional extras group (`[extra1,extra2]`) before the specifier. + let (extras, specifier) = if let Some(after_open) = rest.strip_prefix('[') { match after_open.find(']') { - Some(close) => after_open[close + 1..].to_string(), + Some(close) => { + let extras = after_open[..close] + .split(',') + .map(str::trim) + .filter(|e| !e.is_empty()) + .map(str::to_string) + .collect(); + (extras, after_open[close + 1..].to_string()) + } None => return None, } } else { - rest.to_string() + (Vec::new(), rest.to_string()) }; // Conda version specs carry no internal whitespace. let specifier: String = specifier.split_whitespace().collect(); - Some((normalize_name(name), specifier)) + Some((normalize_name(name), extras, specifier)) } /// Convert a project's PEP 508 `[project.dependencies]` into conda match-specs /// suitable for a trial solve. Each requirement's name is mapped PyPI→conda via -/// the [name map](crate::name_map) (so `opencv-python` becomes `opencv`), and -/// its version specifier is reused (conda and PEP 440 share the common -/// comparison operators). Direct-URL / unparseable entries are skipped. -pub fn requirements_to_conda_specs(dependencies: &[String]) -> Vec { +/// the [name map](crate::name_map) (so `opencv-python` becomes `opencv`), with +/// `overrides` consulted first; its version specifier is reused (conda and PEP +/// 440 share the common comparison operators). A requirement whose extras group +/// maps to several conda packages yields one spec per package, each carrying +/// the requirement's specifier. Direct-URL / unparseable entries are skipped. +pub fn requirements_to_conda_specs( + dependencies: &[String], + overrides: &name_map::Overrides, +) -> Vec { let mut specs = Vec::new(); for requirement in dependencies { - let Some((name, specifier)) = parse_requirement(requirement) else { + let Some((name, extras, specifier)) = parse_requirement(requirement) else { continue; }; - let conda = name_map::pypi_to_conda(&name) - .map(str::to_string) - .unwrap_or(name); - if specifier.is_empty() { - specs.push(conda); - } else { - specs.push(format!("{conda} {specifier}")); + for conda in overrides.conda_names(&name, &extras) { + if specifier.is_empty() { + specs.push(conda); + } else { + specs.push(format!("{conda} {specifier}")); + } } } specs @@ -512,6 +569,12 @@ environment = "myenv" registry = "file:///srv/nepenthe" version = "1.3.0" python = "3.11" + +[tool.nepenthe.package-mappings] +example-package = "example-conda-package" + +[tool.nepenthe.extras-mappings] +"example-package[extra]" = ["example-package", "example-package-extra"] "#; let dir = std::env::temp_dir().join(format!("nepenthe-proj-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -527,6 +590,18 @@ python = "3.11" assert!(matches!(project.nepenthe.label(), Label::Exact(v) if v == "1.3.0")); assert_eq!(project.dependencies, vec!["numpy>=2", "requests"]); + let overrides = project.nepenthe.name_overrides(); + assert_eq!( + overrides.conda_name("example-package"), + "example-conda-package" + ); + assert_eq!( + overrides.conda_names("example-package", &["extra".to_string()]), + vec!["example-package", "example-package-extra"] + ); + // `try` reads the same overrides without requiring the stanza to resolve. + assert_eq!(read_name_overrides(&path).unwrap(), overrides); + std::fs::remove_dir_all(&dir).ok(); } @@ -541,26 +616,38 @@ python = "3.11" } #[test] - fn parse_requirement_extracts_name_and_specifier() { + fn parse_requirement_extracts_name_extras_and_specifier() { assert_eq!( parse_requirement("numpy>=2.2"), - Some(("numpy".to_string(), ">=2.2".to_string())) + Some(("numpy".to_string(), vec![], ">=2.2".to_string())) ); assert_eq!( parse_requirement("requests"), - Some(("requests".to_string(), String::new())) + Some(("requests".to_string(), vec![], String::new())) ); assert_eq!( parse_requirement("Flask-SQLAlchemy >= 3, <4"), - Some(("flask-sqlalchemy".to_string(), ">=3,<4".to_string())) + Some(("flask-sqlalchemy".to_string(), vec![], ">=3,<4".to_string())) ); assert_eq!( parse_requirement("ruff[extra]==0.6.0"), - Some(("ruff".to_string(), "==0.6.0".to_string())) + Some(( + "ruff".to_string(), + vec!["extra".to_string()], + "==0.6.0".to_string() + )) + ); + assert_eq!( + parse_requirement("ruff[one, two]"), + Some(( + "ruff".to_string(), + vec!["one".to_string(), "two".to_string()], + String::new() + )) ); assert_eq!( parse_requirement("pandas==1.5.0; python_version < '3.12'"), - Some(("pandas".to_string(), "==1.5.0".to_string())) + Some(("pandas".to_string(), vec![], "==1.5.0".to_string())) ); // Direct URL references are skipped. assert_eq!(parse_requirement("foo @ https://example.com/foo.whl"), None); @@ -576,7 +663,7 @@ python = "3.11" "scipy>=1.10".to_string(), // missing "foo @ https://x/foo".to_string(), // skipped ]; - let report = check_dependencies(&deps, &packages); + let report = check_dependencies(&deps, &packages, &name_map::Overrides::default()); assert_eq!(report.satisfied(), 2); assert_eq!(report.conflicts(), 1); assert_eq!(report.missing(), 1); @@ -597,7 +684,11 @@ python = "3.11" fn check_matches_pypi_names_to_conda_packages() { // PEP 503 normalization lets `Ruamel.YAML` match a conda `ruamel-yaml`. let packages = vec![pkg("ruamel-yaml", "0.18.6")]; - let report = check_dependencies(&["Ruamel.YAML>=0.18".to_string()], &packages); + let report = check_dependencies( + &["Ruamel.YAML>=0.18".to_string()], + &packages, + &name_map::Overrides::default(), + ); assert_eq!(report.satisfied(), 1); } @@ -606,11 +697,56 @@ python = "3.11" // The PyPI name `opencv-python` maps to the conda package `opencv` via // the vendored grayskull table; without it this would report missing. let packages = vec![pkg("opencv", "4.10.0")]; - let report = check_dependencies(&["opencv-python>=4".to_string()], &packages); + let report = check_dependencies( + &["opencv-python>=4".to_string()], + &packages, + &name_map::Overrides::default(), + ); assert_eq!(report.satisfied(), 1, "{:?}", report.dependencies); assert_eq!(report.missing(), 0); } + #[test] + fn check_expands_an_extras_group_and_requires_every_package() { + let overrides = name_map::Overrides::new( + [], + [( + "example-package[extra]".to_string(), + vec![ + "example-package".to_string(), + "example-package-extra".to_string(), + ], + )], + ); + let deps = vec!["example-package[extra]>=1.2".to_string()]; + + let packages = vec![ + pkg("example-package", "1.3.0"), + pkg("example-package-extra", "1.3.0"), + ]; + let report = check_dependencies(&deps, &packages, &overrides); + assert_eq!(report.satisfied(), 1, "{:?}", report.dependencies); + + // The base package alone no longer satisfies the requirement. + let packages = vec![pkg("example-package", "1.3.0")]; + let report = check_dependencies(&deps, &packages, &overrides); + assert!(matches!( + &report.dependencies[0].status, + DependencyStatus::Missing { name } if name == "example-package-extra" + )); + } + + #[test] + fn check_prefers_a_package_override_over_the_vendored_table() { + let overrides = name_map::Overrides::new( + [("opencv-python".to_string(), "example-opencv".to_string())], + [], + ); + let packages = vec![pkg("example-opencv", "4.10.0")]; + let report = check_dependencies(&["opencv-python>=4".to_string()], &packages, &overrides); + assert_eq!(report.satisfied(), 1, "{:?}", report.dependencies); + } + #[test] fn requirements_convert_to_conda_match_specs() { let deps = vec![ @@ -619,7 +755,7 @@ python = "3.11" "opencv-python<5".to_string(), // pypi→conda mapped "torch @ https://example.com/torch".to_string(), // url ref → skipped ]; - let specs = requirements_to_conda_specs(&deps); + let specs = requirements_to_conda_specs(&deps, &name_map::Overrides::default()); assert_eq!( specs, vec![