diff --git a/Cargo.lock b/Cargo.lock index 3f59caf7..4d1319f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "once_cell", "version_check", "zerocopy", @@ -140,9 +140,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -150,14 +150,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -187,7 +188,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if 1.0.4", + "cfg-if", "constant_time_eq", "cpufeatures", ] @@ -276,12 +277,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.4" @@ -300,7 +295,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures", "rand_core 0.10.1", ] @@ -491,7 +486,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0667304c32ea56cb4cd6d2d7c0cfe9a2f8041229db8c033af7f8d69492429def" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -509,7 +504,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -683,7 +678,7 @@ version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -851,12 +846,9 @@ dependencies = [ [[package]] name = "gearhash" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" -dependencies = [ - "cfg-if 0.1.10", -] +checksum = "616e8f476a586f1b078d9eece21f7a071f27997e709e2b471f5697deea53bda9" [[package]] name = "getopts" @@ -873,7 +865,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -886,7 +878,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "js-sys", "libc", "r-efi 5.3.0", @@ -900,7 +892,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "js-sys", "libc", "r-efi 6.0.0", @@ -1470,7 +1462,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "combine", "jni-macros", "jni-sys", @@ -1529,7 +1521,7 @@ version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "futures-util", "once_cell", "wasm-bindgen", @@ -1556,6 +1548,7 @@ dependencies = [ "base64", "digest", "eyre", + "hf-hub", "itertools 0.13.0", "monostate", "regex", @@ -2509,7 +2502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if 1.0.4", + "cfg-if", "getrandom 0.2.17", "libc", "untrusted", @@ -2847,7 +2840,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "cpufeatures", "digest", ] @@ -3099,7 +3092,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", ] [[package]] @@ -3436,7 +3429,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "static_assertions", ] @@ -3695,7 +3688,7 @@ version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" dependencies = [ - "cfg-if 1.0.4", + "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", diff --git a/Cargo.toml b/Cargo.toml index 127c6a4c..95cd0594 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,6 @@ members = [ "kernels-common", ] resolver = "2" + +[workspace.dependencies] +hf-hub = { version = "1.1.0", git = "https://github.com/huggingface/hf-hub", rev = "8b00e9eab4a27a614e414692d71791d8e33f7149", features = ["blocking"] } diff --git a/kernel-builder/Cargo.toml b/kernel-builder/Cargo.toml index 4aab33e1..96c8f9e1 100644 --- a/kernel-builder/Cargo.toml +++ b/kernel-builder/Cargo.toml @@ -18,7 +18,7 @@ clap-markdown = "0.1.5" clap_complete = "4" eyre = "0.6.12" git2 = "0.20" -hf-hub = { version = "1.1.0", git = "https://github.com/huggingface/hf-hub", rev = "8b00e9eab4a27a614e414692d71791d8e33f7149", features = ["blocking"] } +hf-hub = { workspace = true } indicatif = "0.17" itertools = "0.13" minijinja = "2.5" diff --git a/kernel-builder/src/hf.rs b/kernel-builder/src/hf.rs index 0ed6c8fd..35ae5c18 100644 --- a/kernel-builder/src/hf.rs +++ b/kernel-builder/src/hf.rs @@ -1,19 +1,10 @@ use eyre::{Context, Result}; use hf_hub::{HFClientSync, HFRepositorySync, RepoType}; +use kernels_common::hf::HFKernelsClientBuilder; /// Build a sync HF API client. pub fn api() -> Result { - let mut builder = hf_hub::HFClient::builder(); - - if let Ok(endpoint) = std::env::var("HF_ENDPOINT") { - builder = builder.endpoint(endpoint); - } - - if let Ok(token) = std::env::var("HF_TOKEN") { - builder = builder.token(token); - } - - builder + HFKernelsClientBuilder::new() .build_sync() .context("Cannot create Hugging Face API client") } diff --git a/kernels-common/Cargo.toml b/kernels-common/Cargo.toml index 2709de5a..9cfb0a4a 100644 --- a/kernels-common/Cargo.toml +++ b/kernels-common/Cargo.toml @@ -11,6 +11,7 @@ repository = "https://github.com/huggingface/kernels" base64 = "0.22" digest = "0.11" eyre = "0.6.12" +hf-hub = { workspace = true } itertools = "0.13" monostate = "0.1" regex = "1" diff --git a/kernels-common/src/hf.rs b/kernels-common/src/hf.rs new file mode 100644 index 00000000..099d5baa --- /dev/null +++ b/kernels-common/src/hf.rs @@ -0,0 +1,487 @@ +//! Construction of Hugging Face Hub clients. +//! +//! The hf-hub crate stopped handling standard huggingface_hub environment +//! variables. This module adds a wrapper for the builder that adds back +//! environment variable support, as closely aligned to huggingface_hub +//! as possible. + +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +use hf_hub::{HFClient, HFError}; +use thiserror::Error; + +/// Default Hub endpoint, matching `huggingface_hub`. +const DEFAULT_ENDPOINT: &str = "https://huggingface.co"; + +/// Name of the token file inside the Hugging Face home directory. +const TOKEN_FILENAME: &str = "token"; + +/// Environment variable interpreted as a bool. +fn env_bool(var: &str) -> bool { + is_true(env_string(var).as_deref()) +} + +/// Truthy values. +fn is_true(value: Option<&str>) -> bool { + // See: https://github.com/huggingface/huggingface_hub/blob/f14866648507aa58afc9554c712a4e1f70dd5c3e/src/huggingface_hub/constants.py#L12 + matches!( + value.map(str::to_ascii_uppercase).as_deref(), + Some("1" | "ON" | "YES" | "TRUE") + ) +} + +/// Environment variable interpreted as a string. +/// +/// Empty values count as unset. A leading `~` is expmanded to the user's +/// home directory. +fn env_path(var: &str) -> Option { + env::var_os(var) + .filter(|value| !value.is_empty()) + .map(|value| expand_tilde(PathBuf::from(value))) +} + +/// Environment variable interpreted as a string. +/// +/// An empty value counts as unset. +fn env_string(var: &str) -> Option { + env::var(var).ok().filter(|value| !value.is_empty()) +} + +/// Expand a leading `~` to the home directory. +pub(crate) fn expand_tilde(path: PathBuf) -> PathBuf { + expand_tilde_with_home(path, env::home_dir()) +} + +fn expand_tilde_with_home(path: PathBuf, home: Option) -> PathBuf { + // Normally one would use `expand_tilde`. But this is a variant + // that takes a home directory to make it testable. + + // Only matches a whole leading `~` component, so `~user/x` does not. + let Ok(rest) = path.strip_prefix("~") else { + return path; + }; + let Some(home) = home else { + return path; + }; + if rest == Path::new("") { + return home; + } + home.join(rest) +} + +/// Error building a Hub client. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum HFKernelsClientError { + /// The kernel cache directory could not be detepmined. + #[error( + "cannot determine the kernel cache directory, set `KERNELS_CACHE`, `HF_HUB_CACHE`, or `HF_HOME`" + )] + UnknownCacheDir, + + /// The underlying `hf-hub` client could not be constructed. + #[error("cannot create Hugging Face Hub client")] + Client(#[from] HFError), +} + +/// Hugging Face Hub client builder, with standard huggingface-hub environment variable support. +#[derive(Clone, Debug, Default)] +pub struct HFKernelsClientBuilder { + endpoint: Option, + token: Option, + cache_dir: Option, + user_agent: Option, +} + +impl HFKernelsClientBuilder { + /// Create a new builder. + pub fn new() -> Self { + Self::default() + } + + /// Override the Hub endpoint. + pub fn endpoint(mut self, endpoint: impl Into) -> Self { + self.endpoint = Some(endpoint.into()); + self + } + + /// Override the authentication token. + pub fn token(mut self, token: impl Into) -> Self { + self.token = Some(token.into()); + self + } + + /// Override the cache directory. + pub fn cache_dir(mut self, cache_dir: impl Into) -> Self { + self.cache_dir = Some(cache_dir.into()); + self + } + + /// Override the `User-Agent` header. + pub fn user_agent(mut self, user_agent: impl Into) -> Self { + self.user_agent = Some(user_agent.into()); + self + } + + /// Build an asynchronous client. + pub fn build(self) -> Result { + Ok(self.hf_client_builder()?.build()?) + } + + /// Build a blocking client. + pub fn build_sync(self) -> Result { + Ok(self.hf_client_builder()?.build_sync()?) + } + + fn hf_client_builder(self) -> Result { + let cache_dir = match self.cache_dir { + Some(cache_dir) => cache_dir, + None => kernels_cache().ok_or(HFKernelsClientError::UnknownCacheDir)?, + }; + + let mut builder = HFClient::builder() + .endpoint(select_endpoint(self.endpoint, env_string("HF_ENDPOINT"))) + .cache_dir(cache_dir) + .user_agent(self.user_agent.unwrap_or_else(default_user_agent)); + + // `hf-hub` has no way to unset a token, so only set it when we have + // one. + if let Some(token) = resolve_token(self.token) { + builder = builder.token(token); + } + + Ok(builder) + } +} + +/// The Hugging Face home directory. +fn hf_home() -> Option { + resolve_hf_home( + env_path("HF_HOME"), + env_path("XDG_CACHE_HOME"), + env::home_dir(), + ) +} + +/// The Hub cache directory. +fn hf_hub_cache() -> Option { + resolve_hf_hub_cache( + env_path("HF_HUB_CACHE"), + env_path("HUGGINGFACE_HUB_CACHE"), + hf_home(), + ) +} + +/// The kernels cache directory. +fn kernels_cache() -> Option { + resolve_kernels_cache(env_path("KERNELS_CACHE"), hf_hub_cache()) +} + +fn resolve_hf_home( + hf_home: Option, + xdg_cache_home: Option, + home: Option, +) -> Option { + if let Some(hf_home) = hf_home { + return Some(hf_home); + } + let cache_home = match xdg_cache_home { + Some(xdg_cache_home) => xdg_cache_home, + None => home?.join(".cache"), + }; + Some(cache_home.join("huggingface")) +} + +fn resolve_hf_hub_cache( + hf_hub_cache: Option, + huggingface_hub_cache: Option, + hf_home: Option, +) -> Option { + hf_hub_cache + .or(huggingface_hub_cache) + .or_else(|| Some(hf_home?.join("hub"))) +} + +fn resolve_kernels_cache( + kernels_cache: Option, + hf_hub_cache: Option, +) -> Option { + kernels_cache.or(hf_hub_cache) +} + +/// The default user agent. +fn default_user_agent() -> String { + let version = env!("CARGO_PKG_VERSION"); + match env_string("HF_HUB_USER_AGENT_ORIGIN") { + Some(origin) => format!("kernels/{version}; {origin}"), + None => format!("kernels/{version}"), + } +} + +/// Select the endpoint to use. +fn select_endpoint(explicit: Option, hf_endpoint: Option) -> String { + explicit + .or(hf_endpoint) + .unwrap_or_else(|| DEFAULT_ENDPOINT.to_string()) +} + +/// Resolve the token from the environment and the token file. +fn resolve_token(explicit: Option) -> Option { + select_token( + explicit, + env_bool("HF_HUB_DISABLE_IMPLICIT_TOKEN"), + env_string("HF_TOKEN"), + env_string("HUGGING_FACE_HUB_TOKEN"), + token_path().as_deref(), + ) +} + +/// Path of the token file. +fn token_path() -> Option { + env_path("HF_TOKEN_PATH").or_else(|| Some(hf_home()?.join(TOKEN_FILENAME))) +} + +/// A token stored in a file; blank files count as absent. +fn read_token_file(path: &Path) -> Option { + let token = fs::read_to_string(path).ok()?.trim().to_string(); + (!token.is_empty()).then_some(token) +} + +/// Select the token to use. +fn select_token( + explicit: Option, + disable_implicit_token: bool, + hf_token: Option, + huggingface_hub_token: Option, + token_file: Option<&Path>, +) -> Option { + // An explicitly configured token is not an implicit one, so it is used + // even when implicit tokens are disabled. + if explicit.is_some() { + return explicit; + } + if disable_implicit_token { + return None; + } + hf_token + .or(huggingface_hub_token) + .or_else(|| read_token_file(token_file?)) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + fn text(value: &str) -> Option { + Some(value.to_string()) + } + + #[test] + fn endpoint_precedence() { + assert_eq!( + select_endpoint(text("https://explicit"), text("https://env")), + "https://explicit" + ); + assert_eq!(select_endpoint(None, text("https://env")), "https://env"); + assert_eq!(select_endpoint(None, None), DEFAULT_ENDPOINT); + } + + #[test] + fn token_precedence() { + assert_eq!( + select_token(text("explicit"), false, text("hf"), text("legacy"), None), + text("explicit") + ); + assert_eq!( + select_token(None, false, text("hf"), text("legacy"), None), + text("hf") + ); + assert_eq!( + select_token(None, false, None, text("legacy"), None), + text("legacy") + ); + assert_eq!(select_token(None, false, None, None, None), None); + } + + /// Disabling implicit tokens must not discard a token the caller passed + /// in deliberately. + #[test] + fn disabling_implicit_tokens_keeps_explicit_token() { + assert_eq!( + select_token(text("explicit"), true, text("hf"), None, None), + text("explicit") + ); + assert_eq!(select_token(None, true, text("hf"), None, None), None); + } + + #[test] + fn token_falls_back_to_token_file() -> std::io::Result<()> { + let dir = TempDir::new()?; + let token_file = dir.path().join("token"); + fs::write(&token_file, " file-token\n")?; + + assert_eq!( + select_token(None, false, None, None, Some(&token_file)), + text("file-token") + ); + // The environment still wins over the file. + assert_eq!( + select_token(None, false, text("hf"), None, Some(&token_file)), + text("hf") + ); + Ok(()) + } + + #[test] + fn blank_and_missing_token_files_are_ignored() -> std::io::Result<()> { + let dir = TempDir::new()?; + let blank = dir.path().join("blank"); + fs::write(&blank, " \n")?; + + assert_eq!(select_token(None, false, None, None, Some(&blank)), None); + assert_eq!( + select_token(None, false, None, None, Some(&dir.path().join("missing"))), + None + ); + Ok(()) + } + + /// Only `huggingface_hub`'s truthy values count, so `HF_HUB_OFFLINE=0` + /// does not switch offline mode on. + #[test] + fn only_documented_values_are_true() { + for value in ["1", "ON", "on", "YES", "yes", "TRUE", "true", "True"] { + assert!(is_true(Some(value)), "{value} should be true"); + } + for value in ["0", "OFF", "NO", "FALSE", "false", "AUTO", "", "maybe"] { + assert!(!is_true(Some(value)), "{value} should be false"); + } + assert!(!is_true(None)); + } + + /// The cache directory must never silently become relative to the + /// working directory, the way `hf-hub`'s own default does. + #[test] + fn explicit_cache_dir_is_used() { + let builder = HFKernelsClientBuilder::new() + .cache_dir("/tmp/kernels-cache") + .token("test-token"); + let client = builder.build().expect("client should build"); + assert_eq!(client.cache_dir(), Path::new("/tmp/kernels-cache")); + } + + fn path(path: &str) -> Option { + Some(PathBuf::from(path)) + } + + #[test] + fn hf_home_prefers_explicit_setting() { + assert_eq!( + resolve_hf_home(path("/hf-home"), path("/xdg"), path("/home/user")), + path("/hf-home") + ); + } + + #[test] + fn hf_home_falls_back_to_xdg_cache_home() { + assert_eq!( + resolve_hf_home(None, path("/xdg"), path("/home/user")), + path("/xdg/huggingface") + ); + } + + #[test] + fn hf_home_falls_back_to_home_directory() { + assert_eq!( + resolve_hf_home(None, None, path("/home/user")), + path("/home/user/.cache/huggingface") + ); + } + + #[test] + fn hf_home_is_unknown_without_home_directory() { + assert_eq!(resolve_hf_home(None, None, None), None); + } + + #[test] + fn hub_cache_precedence() { + assert_eq!( + resolve_hf_hub_cache(path("/hub"), path("/legacy"), path("/hf-home")), + path("/hub") + ); + assert_eq!( + resolve_hf_hub_cache(None, path("/legacy"), path("/hf-home")), + path("/legacy") + ); + assert_eq!( + resolve_hf_hub_cache(None, None, path("/hf-home")), + path("/hf-home/hub") + ); + assert_eq!(resolve_hf_hub_cache(None, None, None), None); + } + + #[test] + fn kernels_cache_overrides_hub_cache() { + assert_eq!( + resolve_kernels_cache(path("/kernels"), path("/hub")), + path("/kernels") + ); + assert_eq!(resolve_kernels_cache(None, path("/hub")), path("/hub")); + assert_eq!(resolve_kernels_cache(None, None), None); + } + + /// Expands `input` against a fixed home directory. + fn expanded(input: &str) -> PathBuf { + expand_tilde_with_home(PathBuf::from(input), path("/home/user")) + } + + #[test] + fn tilde_is_expanded_to_the_home_directory() { + assert_eq!(expanded("~"), PathBuf::from("/home/user")); + assert_eq!( + expanded("~/.cache/hub"), + PathBuf::from("/home/user/.cache/hub") + ); + } + + #[test] + fn only_a_leading_tilde_component_is_expanded() { + // Another user's home directory needs a passwd lookup. + assert_eq!(expanded("~other/hub"), PathBuf::from("~other/hub")); + // A tilde that is not leading is part of the name. + assert_eq!(expanded("/cache/~/hub"), PathBuf::from("/cache/~/hub")); + assert_eq!(expanded("/absolute/hub"), PathBuf::from("/absolute/hub")); + assert_eq!(expanded("relative/hub"), PathBuf::from("relative/hub")); + } + + /// `expanduser` leaves the path alone when there is no home directory, + /// rather than failing. + #[test] + fn tilde_is_kept_without_a_home_directory() { + assert_eq!( + expand_tilde_with_home(PathBuf::from("~/hub"), None), + PathBuf::from("~/hub") + ); + } + + /// An explicitly empty variable must not resolve to a relative path. + #[test] + fn empty_variables_count_as_unset() { + // SAFETY: single-threaded access to this variable; no other test + // reads or writes it. + unsafe { env::set_var("KERNELS_DATA_TEST_EMPTY", "") }; + assert_eq!(env_path("KERNELS_DATA_TEST_EMPTY"), None); + + unsafe { env::set_var("KERNELS_DATA_TEST_EMPTY", "/value") }; + assert_eq!(env_path("KERNELS_DATA_TEST_EMPTY"), path("/value")); + + unsafe { env::remove_var("KERNELS_DATA_TEST_EMPTY") }; + assert_eq!(env_path("KERNELS_DATA_TEST_EMPTY"), None); + } +} diff --git a/kernels-common/src/lib.rs b/kernels-common/src/lib.rs index 5dc92f3b..30149065 100644 --- a/kernels-common/src/lib.rs +++ b/kernels-common/src/lib.rs @@ -1,6 +1,7 @@ pub mod config; pub mod digest; pub mod git; +pub mod hf; pub mod lock; pub mod metadata; pub mod version; diff --git a/nix-builder/tests/Dockerfile.test-kernel b/nix-builder/tests/Dockerfile.test-kernel index 74f63ca1..fc8d05b3 100644 --- a/nix-builder/tests/Dockerfile.test-kernel +++ b/nix-builder/tests/Dockerfile.test-kernel @@ -62,12 +62,16 @@ RUN CUDA_MAJOR_MINOR=$(echo ${CUDA_VERSION} | cut -d'.' -f1,2) && \ # Add additional dependencies. RUN uv add "apache-tvm-ffi~=0.1.9" numpy pytest -# Copy kernels-common and kernels source to install from source. -COPY kernels-common ./kernels-common -COPY kernels ./kernels-src +# Copy the Cargo workspace root plus the members needed to build the +# `kernels` Python package, preserving their relative layout. +COPY Cargo.toml Cargo.lock ./kernels-workspace/ +COPY kernel-abi-check ./kernels-workspace/kernel-abi-check +COPY kernel-builder ./kernels-workspace/kernel-builder +COPY kernels-common ./kernels-workspace/kernels-common +COPY kernels ./kernels-workspace/kernels # Install kernels (Rust/maturin package) from source. -RUN uv add ./kernels-src +RUN uv add ./kernels-workspace/kernels # Copy kernels and tests COPY relu-kernel ./relu-kernel