diff --git a/Cargo.toml b/Cargo.toml index 398c2c51..d917f4ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,11 @@ rust.unused_import_braces = "warn" rust.unused_lifetimes = "warn" rust.unused_macro_rules = "warn" rust.unused_qualifications = "warn" +rust.unexpected_cfgs = { level = "warn", check-cfg = [ + "cfg(disable_dispatch_sse4_2)", + "cfg(disable_dispatch_avx2)", + "cfg(disable_dispatch_avx512)", +] } clippy.too_many_arguments = "allow" diff --git a/fearless_simd/Cargo.toml b/fearless_simd/Cargo.toml index 18ed941c..218bb050 100644 --- a/fearless_simd/Cargo.toml +++ b/fearless_simd/Cargo.toml @@ -26,7 +26,7 @@ rustdoc-args = [ [features] -default = ["std", "dispatch_sse4_2", "dispatch_avx2", "dispatch_avx512"] +default = ["std"] # Get floating point functions from the standard library (likely using your targets libc). # Also allows using `Level::new` on all platforms, to detect which target features are enabled std = [] @@ -37,19 +37,6 @@ libm = ["dep:libm"] # This is primarily used for tests force_support_fallback = [] -# Enable automatic multiversioning for SSE4.2 through `dispatch!` and `Level::dispatch`. -# Has no effect on architectures other than x86 and x86-64. -# Disabling this does not remove the `Sse4_2` token or explicit `kernel!` support. -dispatch_sse4_2 = [] -# Enable automatic multiversioning for AVX2 through `dispatch!` and `Level::dispatch`. -# Has no effect on architectures other than x86 and x86-64. -# Disabling this does not remove the `Avx2` token or explicit `kernel!` support. -dispatch_avx2 = [] -# Enable automatic multiversioning for AVX-512 through `dispatch!` and `Level::dispatch`. -# Has no effect on architectures other than x86 and x86-64. -# Disabling this doeces not remove the `Avx512` token or explicit `kernel!` support. -dispatch_avx512 = [] - [lints] workspace = true diff --git a/fearless_simd/README.md b/fearless_simd/README.md index 893e99aa..78cfed4e 100644 --- a/fearless_simd/README.md +++ b/fearless_simd/README.md @@ -188,13 +188,18 @@ runtime. x86 CPUs are not guaranteed to have any SIMD particular instruction set, so `fearless_simd` compiles a version of each function generic over [`Simd`] for each instruction set, and [`dispatch`] selects the best one at runtime. -This is strictly required to take advantage of SIMD, but results in an increased binary size on x86. +This is necessary to take advantage of SIMD, but results in an increased binary size on x86. If binary size is a concern, the increase can be partially mitigated by setting [`codegen-units=1`](https://nnethercote.github.io/perf-book/build-configuration.html#codegen-units) or [`lto=true`](https://nnethercote.github.io/perf-book/build-configuration.html#link-time-optimization) in your Cargo.toml, at the cost of longer build times. -As a last resort, you can turn off multiversioning for specific SIMD instruction sets, see "Feature Flags" below. +As a last resort, you can turn off multiversioning for specific SIMD instruction sets by passing +`--cfg disable_dispatch_sse4_2`, `--cfg disable_dispatch_avx2`, or `--cfg disable_dispatch_avx512` in `RUSTFLAGS`. +These configuration flags only control automatic multiversioning. Disabling one does not remove its token type, its +[`Simd`] implementation, or explicit [`kernel`] support; for example, an `Avx2` token can still be used to call an +AVX2 kernel when the CPU supports it. + Note that later extensions can be beneficial even if you are only using 128-bit vectors: AVX2 and AVX-512 provide more efficient instructions for some operations, and AVX-512 also more than doubles the number of vector registers of all sizes. @@ -210,13 +215,6 @@ The following crate [feature flags](https://doc.rust-lang.org/cargo/reference/fe Also allows using [`Level::new`] on all platforms, to detect which target features are enabled. - `libm`: Use floating point implementations from [libm]. Useful for `#[no_std]`. - `force_support_fallback`: Force scalar fallback, to be supported, even if your compilation target has a better baseline. -- `dispatch_sse4_2`, `dispatch_avx2`, `dispatch_avx512` (enabled by default): Enable automatic x86 multiversioning for the - corresponding instruction set in [`dispatch`] and [`Level::dispatch`]. - -The x86 feature flags only control automatic multiversioning. Disabling one does not remove its token type, its -[`Simd`] implementation, or explicit [`kernel`] support; for example, an `Avx2` token can still be used to call an -AVX2 kernel when the CPU supports it. Because Cargo features are additive, opt out of x86 multiversioning with -`default-features = false`, then enable the pieces you want, for example `features = ["std", "dispatch_sse4_2"]`. At least one of `std` and `libm` is required; `std` overrides `libm`. diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index abbac0c5..33fb183b 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -85,34 +85,7 @@ impl Simd for Sse4_2 { type mask64s = mask64x2; #[inline(always)] fn level(self) -> Level { - #[cfg(not(all( - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "f16c", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt", - target_feature = "xsave" - )))] - return Level::Sse4_2(self); - #[cfg(all( - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "f16c", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt", - target_feature = "xsave" - ))] - { - Level::baseline() - } + Level::Sse4_2(self) } #[inline] fn vectorize R, R>(self, f: F) -> R { diff --git a/fearless_simd/src/lib.rs b/fearless_simd/src/lib.rs index f51a3a10..e1e2d584 100644 --- a/fearless_simd/src/lib.rs +++ b/fearless_simd/src/lib.rs @@ -155,13 +155,18 @@ //! x86 CPUs are not guaranteed to have any SIMD particular instruction set, so `fearless_simd` compiles a version //! of each function generic over [`Simd`] for each instruction set, and [`dispatch`] selects the best one at runtime. //! -//! This is strictly required to take advantage of SIMD, but results in an increased binary size on x86. +//! This is necessary to take advantage of SIMD, but results in an increased binary size on x86. //! If binary size is a concern, the increase can be partially mitigated by setting //! [`codegen-units=1`](https://nnethercote.github.io/perf-book/build-configuration.html#codegen-units) //! or [`lto=true`](https://nnethercote.github.io/perf-book/build-configuration.html#link-time-optimization) in your Cargo.toml, //! at the cost of longer build times. //! -//! As a last resort, you can turn off multiversioning for specific SIMD instruction sets, see "Feature Flags" below. +//! As a last resort, you can turn off multiversioning for specific SIMD instruction sets by passing +//! `--cfg disable_dispatch_sse4_2`, `--cfg disable_dispatch_avx2`, or `--cfg disable_dispatch_avx512` in `RUSTFLAGS`. +//! These configuration flags only control automatic multiversioning. Disabling one does not remove its token type, its +//! [`Simd`] implementation, or explicit [`kernel`] support; for example, an `Avx2` token can still be used to call an +//! AVX2 kernel when the CPU supports it. +//! //! Note that later extensions can be beneficial even if you are only using 128-bit vectors: //! AVX2 and AVX-512 provide more efficient instructions for some operations, //! and AVX-512 also more than doubles the number of vector registers of all sizes. @@ -177,13 +182,6 @@ //! Also allows using [`Level::new`] on all platforms, to detect which target features are enabled. //! - `libm`: Use floating point implementations from [libm]. Useful for `#[no_std]`. //! - `force_support_fallback`: Force scalar fallback, to be supported, even if your compilation target has a better baseline. -//! - `dispatch_sse4_2`, `dispatch_avx2`, `dispatch_avx512` (enabled by default): Enable automatic x86 multiversioning for the -//! corresponding instruction set in [`dispatch`] and [`Level::dispatch`]. -//! -//! The x86 feature flags only control automatic multiversioning. Disabling one does not remove its token type, its -//! [`Simd`] implementation, or explicit [`kernel`] support; for example, an `Avx2` token can still be used to call an -//! AVX2 kernel when the CPU supports it. Because Cargo features are additive, opt out of x86 multiversioning with -//! `default-features = false`, then enable the pieces you want, for example `features = ["std", "dispatch_sse4_2"]`. //! //! At least one of `std` and `libm` is required; `std` overrides `libm`. //! @@ -305,29 +303,6 @@ pub enum Level { /// Scalar fallback level, i.e. no supported SIMD features are to be used. /// /// This can be created with [`Level::fallback`]. - // We only want to compile the fallback implementation if: - // - We're on a supported architecture, but don't statically support the lowest alternative level; OR - // - We're on an unsupported architecture; OR - // - The fallback is forcibly enabled - #[cfg(any( - all(target_arch = "aarch64", not(target_feature = "neon")), - all( - any(target_arch = "x86", target_arch = "x86_64"), - not(all( - target_feature = "sse4.2", - target_feature = "cmpxchg16b", - target_feature = "popcnt" - )) - ), - all(target_arch = "wasm32", not(target_feature = "simd128")), - not(any( - target_arch = "x86", - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "wasm32" - )), - feature = "force_support_fallback" - ))] Fallback(Fallback), /// The Neon instruction set on 64 bit ARM. #[cfg(target_arch = "aarch64")] @@ -339,22 +314,7 @@ pub enum Level { /// Also known as x86-64-v2. /// /// All production CPUs with SSE4.2 also support the other two extensions, so it is safe to require them. - // We don't need to support this if the compilation target definitely supports something better. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - not(all( - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "f16c", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt", - target_feature = "xsave" - )) - ))] + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] Sse4_2(Sse4_2), /// Ice Lake-class AVX-512 on (32 and 64 bit) x86. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] @@ -438,18 +398,6 @@ impl Level { && std::arch::is_x86_feature_detected!("cmpxchg16b") && std::arch::is_x86_feature_detected!("popcnt") { - #[cfg(not(all( - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "f16c", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt", - target_feature = "xsave" - )))] return unsafe { Self::Sse4_2(Sse4_2::new_unchecked()) }; } } @@ -500,34 +448,8 @@ impl Level { /// Check whether this is the `Fallback` level; that is, whether no better feature level could /// be statically or dynamically detected. This is useful if there's a scalarized version of /// your algorithm that runs faster if SIMD isn't supported. - /// - /// This method is always available, even in cases where `Fallback` is not; for instance, if - /// you're targeting a platform that always supports some level of SIMD. In such cases, it will - /// always return false. pub fn is_fallback(self) -> bool { - #[cfg(any( - all(target_arch = "aarch64", not(target_feature = "neon")), - all( - any(target_arch = "x86", target_arch = "x86_64"), - not(all( - target_feature = "sse4.2", - target_feature = "cmpxchg16b", - target_feature = "popcnt" - )) - ), - all(target_arch = "wasm32", not(target_feature = "simd128")), - not(any( - target_arch = "x86", - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "wasm32" - )), - feature = "force_support_fallback" - ))] - return matches!(self, Self::Fallback(_)); - - #[allow(unreachable_code, reason = "Fallback unreachable in some cfgs.")] - false + matches!(self, Self::Fallback(_)) } /// If this is a proof that Neon (or better) is available, access that instruction set. @@ -593,23 +515,7 @@ impl Level { // Safety: The Avx2 struct represents the x86-64-v3 feature set being enabled, which // includes the `sse4.2`, `cmpxchg16b`, and `popcnt` features required by Sse4_2. Self::Avx2(_avx) => unsafe { Some(Sse4_2::new_unchecked()) }, - #[cfg(not(all( - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "f16c", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt", - target_feature = "xsave" - )))] Self::Sse4_2(sse42) => Some(sse42), - #[allow( - unreachable_patterns, - reason = "This arm is reachable on baseline x86/x86_64." - )] _ => None, } } @@ -814,6 +720,65 @@ impl Level { } } + #[doc(hidden)] + #[inline] + pub fn __dispatch_target(self) -> Self { + // Dispatch compiles only the selected multiversioned backends, but public tokens can + // still name lower levels even when the ambient target baseline makes those backends + // redundant. Normalize the proof to the best dispatchable level, while leaving exact + // token identity available for `kernel!` and explicit token use. + #[cfg(feature = "force_support_fallback")] + #[allow( + irrefutable_let_patterns, + reason = "On targets without supported SIMD, Fallback is the only Level variant." + )] + if let Self::Fallback(fallback) = self { + return Self::Fallback(fallback); + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + #[allow(unused_variables, reason = "Unused with all cfgs active")] + let baseline = Self::baseline(); + + #[cfg(not(disable_dispatch_avx512))] + if let Some(avx512) = self.as_avx512().or_else(|| baseline.as_avx512()) { + return Self::Avx512(avx512); + } + + #[cfg(not(disable_dispatch_avx2))] + if let Some(avx2) = self.as_avx2().or_else(|| baseline.as_avx2()) { + return Self::Avx2(avx2); + } + + #[cfg(not(disable_dispatch_sse4_2))] + if let Some(sse4_2) = self.as_sse4_2().or_else(|| baseline.as_sse4_2()) { + return Self::Sse4_2(sse4_2); + } + } + + #[cfg(target_arch = "aarch64")] + { + let baseline = Self::baseline(); + if let Some(neon) = self.as_neon().or_else(|| baseline.as_neon()) { + return Self::Neon(neon); + } + } + + #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] + { + let baseline = Self::baseline(); + if let Some(wasm) = self + .as_wasm_simd128() + .or_else(|| baseline.as_wasm_simd128()) + { + return Self::WasmSimd128(wasm); + } + } + + Self::Fallback(Fallback::new()) + } + /// Create a scalar fallback level, which uses no SIMD instructions. /// /// This is primarily intended for tests; most users should prefer [`Level::new`] or [`Level::baseline`]. diff --git a/fearless_simd/src/macros.rs b/fearless_simd/src/macros.rs index 09a9a150..c6d06cee 100644 --- a/fearless_simd/src/macros.rs +++ b/fearless_simd/src/macros.rs @@ -47,11 +47,8 @@ /// [`Simd`]: crate::Simd #[macro_export] macro_rules! dispatch { - // This falls through to the next branch, but with `forced_fallback_arm` turned into a boolean literal - // indicating whether or not the `force_support_fallback` crate feature is enabled. - ($level:expr, $simd:pat => $op:expr) => {{ $crate::internal_unstable_dispatch_inner!($level, $simd => $op) }}; - (@impl $level:expr, $simd:pat => $op:expr; $forced_fallback_arm: literal) => {{ - match $level { + ($level:expr, $simd:pat => $op:expr) => {{ + match $crate::Level::__dispatch_target($level) { #[cfg(target_arch = "aarch64")] $crate::Level::Neon(neon) => { $crate::__fearless_simd_dispatch_with_token!(neon, $simd => $op) @@ -60,63 +57,10 @@ macro_rules! dispatch { $crate::Level::WasmSimd128(wasm) => { $crate::__fearless_simd_dispatch_with_token!(wasm, $simd => $op) } - // Do not even compile x86-64-v2 (SSE4.2) codepaths when x86-64-v3 (AVX2) is guaranteed - // to reduce binary size and build times - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - not(all( - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "f16c", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt", - target_feature = "xsave" - )) - ))] + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] $crate::Level::Sse4_2(sse4_2) => { $crate::__fearless_simd_dispatch_dispatch_sse4_2!(sse4_2, $simd => $op) } - // Do not even compile x86-64-v3 (AVX2) codepaths when Ice Lake (AVX-512) is guaranteed - // to reduce binary size and build times - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - not(all( - target_feature = "aes", - target_feature = "avx512bitalg", - target_feature = "avx512bw", - target_feature = "avx512cd", - target_feature = "avx512dq", - target_feature = "avx512f", - target_feature = "avx512ifma", - target_feature = "avx512vbmi", - target_feature = "avx512vbmi2", - target_feature = "avx512vl", - target_feature = "avx512vnni", - target_feature = "avx512vpopcntdq", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "fma", - target_feature = "gfni", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "pclmulqdq", - target_feature = "popcnt", - target_feature = "rdrand", - target_feature = "rdseed", - target_feature = "sha", - target_feature = "vaes", - target_feature = "vpclmulqdq", - target_feature = "xsave", - target_feature = "xsavec", - target_feature = "xsaveopt", - target_feature = "xsaves", - )) - ))] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] $crate::Level::Avx2(avx2) => { $crate::__fearless_simd_dispatch_dispatch_avx2!(avx2, $simd => $op) @@ -125,38 +69,20 @@ macro_rules! dispatch { $crate::Level::Avx512(avx512) => { $crate::__fearless_simd_dispatch_dispatch_avx512!(avx512, $simd => $op) } - #[cfg(any( - all(target_arch = "aarch64", not(target_feature = "neon")), - all( - any(target_arch = "x86", target_arch = "x86_64"), - not(all( - target_feature = "sse4.2", - target_feature = "cmpxchg16b", - target_feature = "popcnt" - )) - ), - all(target_arch = "wasm32", not(target_feature = "simd128")), - not(any( - target_arch = "x86", - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "wasm32" - )), - $forced_fallback_arm - ))] $crate::Level::Fallback(fb) => { - // This vectorize call does nothing for Fallback, but it is reasonable to be consistent here. - $crate::__fearless_simd_dispatch_with_token!(fb, $simd => $op) + $crate::__fearless_simd_dispatch_dispatch_fallback!(fb, $simd => $op) } _ => unreachable!(), } }}; } -// The x86 multiversion helpers are split into cfg-selected macro definitions +// The dispatch helpers are split into cfg-selected macro definitions // because exported macro bodies are expanded in the downstream crate, -// so putting `#[cfg(feature = "...")]` directly inside `dispatch!` would test -// the downstream crate's features instead of `fearless_simd`'s features. +// so selecting the helper definitions here preserves `fearless_simd`'s +// dispatch configuration. Helpers for pruned levels expand to `unreachable!()` +// without mentioning `$op`, so the pruned SIMD body is not typechecked or +// compiled. /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] @@ -187,140 +113,230 @@ macro_rules! __fearless_simd_dispatch_with_token { /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(feature = "dispatch_avx512")] -macro_rules! __fearless_simd_dispatch_dispatch_avx512 { - ($avx512:expr, $simd:pat => $op:expr) => { - $crate::__fearless_simd_dispatch_with_token!($avx512, $simd => $op) - }; -} - -/// Implementation detail of [`crate::dispatch`]; this is not public API. -#[macro_export] -#[doc(hidden)] -#[cfg(not(feature = "dispatch_avx512"))] -macro_rules! __fearless_simd_dispatch_dispatch_avx512 { - ($avx512:expr, $simd:pat => $op:expr) => {{ - $crate::__fearless_simd_dispatch_dispatch_avx2_from_superset!($avx512, $simd => $op) +macro_rules! __fearless_simd_dispatch_pruned { + ($proof:expr) => {{ + let __fearless_simd_proof = $proof; + let _ = __fearless_simd_proof; + unreachable!("this SIMD level was pruned from dispatch") }}; } /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(feature = "dispatch_avx2")] -macro_rules! __fearless_simd_dispatch_dispatch_avx2 { - ($avx2:expr, $simd:pat => $op:expr) => { - $crate::__fearless_simd_dispatch_with_token!($avx2, $simd => $op) +#[cfg(any( + all(target_arch = "aarch64", not(target_feature = "neon")), + all( + any(target_arch = "x86", target_arch = "x86_64"), + any( + disable_dispatch_sse4_2, + not(all( + target_feature = "sse4.2", + target_feature = "cmpxchg16b", + target_feature = "popcnt", + )), + ), + ), + all(target_arch = "wasm32", not(target_feature = "simd128")), + not(any( + target_arch = "x86", + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "wasm32" + )), + feature = "force_support_fallback" +))] +macro_rules! __fearless_simd_dispatch_dispatch_fallback { + ($fallback:expr, $simd:pat => $op:expr) => { + $crate::__fearless_simd_dispatch_with_token!($fallback, $simd => $op) }; } /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(not(feature = "dispatch_avx2"))] -macro_rules! __fearless_simd_dispatch_dispatch_avx2 { - ($avx2:expr, $simd:pat => $op:expr) => {{ - $crate::__fearless_simd_dispatch_dispatch_sse4_2_from_superset!($avx2, $simd => $op) - }}; -} - -/// Implementation detail of [`crate::dispatch`]; this is not public API. -#[macro_export] -#[doc(hidden)] -#[cfg(feature = "dispatch_avx2")] -macro_rules! __fearless_simd_dispatch_dispatch_avx2_from_superset { - ($proof:expr, $simd:pat => $op:expr) => {{ - let __fearless_simd_proof = $proof; - let __fearless_simd_token = $crate::Simd::level(__fearless_simd_proof) - .as_avx2() - .expect("a superset x86 SIMD level should provide an AVX2 token"); - $crate::__fearless_simd_dispatch_with_token!(__fearless_simd_token, $simd => $op) - }}; -} - -/// Implementation detail of [`crate::dispatch`]; this is not public API. -#[macro_export] -#[doc(hidden)] -#[cfg(not(feature = "dispatch_avx2"))] -macro_rules! __fearless_simd_dispatch_dispatch_avx2_from_superset { - ($proof:expr, $simd:pat => $op:expr) => {{ - $crate::__fearless_simd_dispatch_dispatch_sse4_2_from_superset!($proof, $simd => $op) - }}; +#[cfg(not(any( + all(target_arch = "aarch64", not(target_feature = "neon")), + all( + any(target_arch = "x86", target_arch = "x86_64"), + any( + disable_dispatch_sse4_2, + not(all( + target_feature = "sse4.2", + target_feature = "cmpxchg16b", + target_feature = "popcnt", + )), + ), + ), + all(target_arch = "wasm32", not(target_feature = "simd128")), + not(any( + target_arch = "x86", + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "wasm32" + )), + feature = "force_support_fallback" +)))] +macro_rules! __fearless_simd_dispatch_dispatch_fallback { + ($fallback:expr, $simd:pat => $op:expr) => {{ $crate::__fearless_simd_dispatch_pruned!($fallback) }}; } /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(feature = "dispatch_sse4_2")] -macro_rules! __fearless_simd_dispatch_dispatch_sse4_2 { - ($sse4_2:expr, $simd:pat => $op:expr) => { - $crate::__fearless_simd_dispatch_with_token!($sse4_2, $simd => $op) +#[cfg(not(disable_dispatch_avx512))] +macro_rules! __fearless_simd_dispatch_dispatch_avx512 { + ($avx512:expr, $simd:pat => $op:expr) => { + $crate::__fearless_simd_dispatch_with_token!($avx512, $simd => $op) }; } /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(not(feature = "dispatch_sse4_2"))] -macro_rules! __fearless_simd_dispatch_dispatch_sse4_2 { - ($sse4_2:expr, $simd:pat => $op:expr) => {{ - let __fearless_simd_proof = $sse4_2; - let _ = __fearless_simd_proof; - $crate::__fearless_simd_dispatch_with_token!($crate::Fallback::new(), $simd => $op) - }}; +#[cfg(disable_dispatch_avx512)] +macro_rules! __fearless_simd_dispatch_dispatch_avx512 { + ($avx512:expr, $simd:pat => $op:expr) => {{ $crate::__fearless_simd_dispatch_pruned!($avx512) }}; } /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(feature = "dispatch_sse4_2")] -macro_rules! __fearless_simd_dispatch_dispatch_sse4_2_from_superset { - ($proof:expr, $simd:pat => $op:expr) => {{ - let __fearless_simd_proof = $proof; - let __fearless_simd_token = $crate::Simd::level(__fearless_simd_proof) - .as_sse4_2() - .expect("a superset x86 SIMD level should provide an SSE4.2 token"); - $crate::__fearless_simd_dispatch_with_token!(__fearless_simd_token, $simd => $op) - }}; +#[cfg(all( + not(disable_dispatch_avx2), + any( + disable_dispatch_avx512, + not(all( + target_feature = "aes", + target_feature = "avx512bitalg", + target_feature = "avx512bw", + target_feature = "avx512cd", + target_feature = "avx512dq", + target_feature = "avx512f", + target_feature = "avx512ifma", + target_feature = "avx512vbmi", + target_feature = "avx512vbmi2", + target_feature = "avx512vl", + target_feature = "avx512vnni", + target_feature = "avx512vpopcntdq", + target_feature = "bmi1", + target_feature = "bmi2", + target_feature = "cmpxchg16b", + target_feature = "fma", + target_feature = "gfni", + target_feature = "lzcnt", + target_feature = "movbe", + target_feature = "pclmulqdq", + target_feature = "popcnt", + target_feature = "rdrand", + target_feature = "rdseed", + target_feature = "sha", + target_feature = "vaes", + target_feature = "vpclmulqdq", + target_feature = "xsave", + target_feature = "xsavec", + target_feature = "xsaveopt", + target_feature = "xsaves", + )) + ) +))] +macro_rules! __fearless_simd_dispatch_dispatch_avx2 { + ($avx2:expr, $simd:pat => $op:expr) => { + $crate::__fearless_simd_dispatch_with_token!($avx2, $simd => $op) + }; } /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(not(feature = "dispatch_sse4_2"))] -macro_rules! __fearless_simd_dispatch_dispatch_sse4_2_from_superset { - ($proof:expr, $simd:pat => $op:expr) => {{ - let __fearless_simd_proof = $proof; - let _ = __fearless_simd_proof; - $crate::__fearless_simd_dispatch_with_token!($crate::Fallback::new(), $simd => $op) - }}; +#[cfg(any( + disable_dispatch_avx2, + all( + not(disable_dispatch_avx512), + target_feature = "aes", + target_feature = "avx512bitalg", + target_feature = "avx512bw", + target_feature = "avx512cd", + target_feature = "avx512dq", + target_feature = "avx512f", + target_feature = "avx512ifma", + target_feature = "avx512vbmi", + target_feature = "avx512vbmi2", + target_feature = "avx512vl", + target_feature = "avx512vnni", + target_feature = "avx512vpopcntdq", + target_feature = "bmi1", + target_feature = "bmi2", + target_feature = "cmpxchg16b", + target_feature = "fma", + target_feature = "gfni", + target_feature = "lzcnt", + target_feature = "movbe", + target_feature = "pclmulqdq", + target_feature = "popcnt", + target_feature = "rdrand", + target_feature = "rdseed", + target_feature = "sha", + target_feature = "vaes", + target_feature = "vpclmulqdq", + target_feature = "xsave", + target_feature = "xsavec", + target_feature = "xsaveopt", + target_feature = "xsaves", + ), +))] +macro_rules! __fearless_simd_dispatch_dispatch_avx2 { + ($avx2:expr, $simd:pat => $op:expr) => {{ $crate::__fearless_simd_dispatch_pruned!($avx2) }}; } -// This macro turns whether the `force_support_fallback` macro is enabled into a boolean literal -// in `dispatch`, which allows it to be used correctly cross-crate. -// This trickery is required because macros are expanded in the context of the calling crate, including for -// evaluating `cfg`s. - /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(feature = "force_support_fallback")] -macro_rules! internal_unstable_dispatch_inner { - ($level:expr, $simd:pat => $op:expr) => { - $crate::dispatch!( - @impl $level, $simd => $op; true - ) +#[cfg(all( + not(disable_dispatch_sse4_2), + any( + disable_dispatch_avx2, + not(all( + target_feature = "avx2", + target_feature = "bmi1", + target_feature = "bmi2", + target_feature = "cmpxchg16b", + target_feature = "f16c", + target_feature = "fma", + target_feature = "lzcnt", + target_feature = "movbe", + target_feature = "popcnt", + target_feature = "xsave" + )) + ) +))] +macro_rules! __fearless_simd_dispatch_dispatch_sse4_2 { + ($sse4_2:expr, $simd:pat => $op:expr) => { + $crate::__fearless_simd_dispatch_with_token!($sse4_2, $simd => $op) }; } /// Implementation detail of [`crate::dispatch`]; this is not public API. #[macro_export] #[doc(hidden)] -#[cfg(not(feature = "force_support_fallback"))] -macro_rules! internal_unstable_dispatch_inner { - ($level:expr, $simd:pat => $op:expr) => { - $crate::dispatch!(@impl $level, $simd => $op; false) - }; +#[cfg(any( + disable_dispatch_sse4_2, + all( + not(disable_dispatch_avx2), + target_feature = "avx2", + target_feature = "bmi1", + target_feature = "bmi2", + target_feature = "cmpxchg16b", + target_feature = "f16c", + target_feature = "fma", + target_feature = "lzcnt", + target_feature = "movbe", + target_feature = "popcnt", + target_feature = "xsave" + ), +))] +macro_rules! __fearless_simd_dispatch_dispatch_sse4_2 { + ($sse4_2:expr, $simd:pat => $op:expr) => {{ $crate::__fearless_simd_dispatch_pruned!($sse4_2) }}; } #[cfg(test)] @@ -360,17 +376,25 @@ mod tests { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn expected_x86_dispatch_backend(level: Level) -> X86DispatchBackend { - if cfg!(feature = "dispatch_avx512") && level.as_avx512().is_some() { + if cfg!(not(disable_dispatch_avx512)) && level.as_avx512().is_some() { X86DispatchBackend::Avx512 - } else if cfg!(feature = "dispatch_avx2") && level.as_avx2().is_some() { + } else if cfg!(not(disable_dispatch_avx2)) && level.as_avx2().is_some() { X86DispatchBackend::Avx2 - } else if cfg!(feature = "dispatch_sse4_2") && level.as_sse4_2().is_some() { + } else if cfg!(not(disable_dispatch_sse4_2)) && level.as_sse4_2().is_some() { X86DispatchBackend::Sse4_2 } else { X86DispatchBackend::Fallback } } + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[allow(dead_code, reason = "Compile test")] + fn x86_level_variants_remain_available() { + let _ = Level::Sse4_2(unsafe { crate::Sse4_2::new_unchecked() }); + let _ = Level::Avx2(unsafe { crate::Avx2::new_unchecked() }); + let _ = Level::Avx512(unsafe { crate::Avx512::new_unchecked() }); + } + #[allow(dead_code, reason = "Compile test")] fn dispatch_generic() { fn generic(_: S, x: T) -> T { @@ -404,18 +428,51 @@ mod tests { #[cfg(all( feature = "std", any(target_arch = "x86", target_arch = "x86_64"), - not(feature = "dispatch_avx512") + any( + disable_dispatch_sse4_2, + disable_dispatch_avx2, + disable_dispatch_avx512 + ) ))] #[test] - fn disabled_avx512_multiversioning_does_not_filter_avx512_token_access() { - if crate::x86_detects_icelake_avx512() { - assert!( - Level::new().as_avx512().is_some(), - "`dispatch_avx512` controls dispatch multiversioning, not AVX-512 token access" - ); + fn disabled_x86_multiversioning_does_not_filter_token_access() { + let level = Level::new(); + + // If dispatch disabling accidentally removed token access, these method calls stop compiling. + #[cfg(disable_dispatch_sse4_2)] + let _ = level.as_sse4_2().is_some(); + + #[cfg(disable_dispatch_avx2)] + let _ = level.as_avx2().is_some(); + + #[cfg(disable_dispatch_avx512)] + let _ = level.as_avx512().is_some(); + } + + /// Mostly useful with `RUSTFLAGS='-C target-cpu=x86-64-v3'` and higher, doesn't do much otherwise + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[test] + fn high_baseline_dispatches_lower_tokens() { + if let Some(sse4_2) = Level::baseline().as_sse4_2() { + let actual = dispatch!(sse4_2.level(), simd => x86_dispatch_backend(simd)); + + assert_eq!(actual, expected_x86_dispatch_backend(Level::baseline())); } } + #[cfg(all( + not(feature = "force_support_fallback"), + any(target_arch = "x86", target_arch = "x86_64") + ))] + #[test] + fn fallback_dispatches_as_baseline() { + let actual = dispatch!(Level::Fallback(crate::Fallback::new()), simd => + x86_dispatch_backend(simd) + ); + + assert_eq!(actual, expected_x86_dispatch_backend(Level::baseline())); + } + mod no_import_simd { /// We should be able to use [`dispatch`] in a scope which doesn't import anything. #[test] diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 724c12ef..b5329ed0 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -137,43 +137,9 @@ impl Level for X86 { fn make_level_body(&self) -> TokenStream { let level_tok = self.token(); - match self { - Self::Sse4_2 => quote! { - #[cfg(not(all( - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "f16c", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt", - target_feature = "xsave" - )))] - return Level::#level_tok(self); - #[cfg(all( - target_feature = "avx2", - target_feature = "bmi1", - target_feature = "bmi2", - target_feature = "cmpxchg16b", - target_feature = "f16c", - target_feature = "fma", - target_feature = "lzcnt", - target_feature = "movbe", - target_feature = "popcnt", - target_feature = "xsave" - ))] - { - Level::baseline() - } - }, - Self::Avx2 => quote! { - Level::#level_tok(self) - }, - Self::Avx512 => quote! { - Level::#level_tok(self) - }, + + quote! { + Level::#level_tok(self) } }