diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index e6b8c537d8451..8a4ebfc360ddc 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -20,6 +20,8 @@ mod simd; use cranelift_codegen::ir::{ AtomicRmwOp, BlockArg, ExceptionTableData, ExceptionTableItem, ExceptionTag, }; +use rustc_codegen_ssa::target_features; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::ty; use rustc_middle::ty::GenericArgsRef; use rustc_middle::ty::layout::ValidityRequirement; @@ -834,6 +836,29 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, caller_location); } + sym::target_feature_available_at_call_site => { + intrinsic_args!(fx, args => (); intrinsic); + + let enabled = match target_features::target_feature_available_at_call_site_intrinsic( + fx.tcx, + source_info.span, + fx.instance.def_id(), + generic_args, + ) { + target_features::TargetFeatureAvailableAtCallSite::Known(enabled) => enabled, + // SSA already handles the easy case where the caller function has the feature enabled. + // Cranelift doesn't (yet) have the equivalent LLVM pass to replace a marker post-inlining, + // so report that the feature is unavailable at the call site. + target_features::TargetFeatureAvailableAtCallSite::CheckBackend(_) => false, + }; + + let ret_val = CValue::by_val( + fx.bcx.ins().iconst(types::I8, i64::from(enabled)), + fx.layout_of(fx.tcx.types.bool), + ); + ret.write_cvalue(fx, ret_val); + } + _ if intrinsic.as_str().starts_with("atomic_fence") => { intrinsic_args!(fx, args => (); intrinsic); diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 78a4c7e88c895..fab6fa836b4d8 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -24,7 +24,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; -use rustc_middle::{bug, span_bug}; +use rustc_middle::{bug, mir, span_bug}; use rustc_span::{Span, Symbol, sym}; use rustc_target::callconv::{ArgAbi, PassMode}; @@ -195,6 +195,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc &mut self, instance: Instance<'tcx>, args: &[OperandRef<'tcx, RValue<'gcc>>], + _: &[mir::Operand<'tcx>], result_layout: ty::layout::TyAndLayout<'tcx>, result_place: Option>>, span: Span, @@ -594,6 +595,16 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc IntrinsicResult::WroteIntoPlace } + fn codegen_target_feature_available_at_call_site( + &mut self, + _rust_feature_name: &str, + ) -> RValue<'gcc> { + // SSA already handles the easy case where the caller function has the feature enabled. + // GCC doesn't (yet) have the equivalent LLVM pass to replace a marker post-inlining, + // so report that the feature is unavailable at the call site. + self.const_bool(false) + } + fn codegen_llvm_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 621f2cd3f9fc7..cf9dd1a6bd52c 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -36,7 +36,7 @@ use crate::back::write::to_llvm_code_model; use crate::builder::gpu_offload::{OffloadGlobals, OffloadKernelGlobals}; use crate::callee::get_fn; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; -use crate::llvm::{self, Metadata, MetadataKindId, Module, Type, Value}; +use crate::llvm::{self, Metadata, MetadataKindId, Module, TargetMachine, Type, Value}; use crate::{attributes, common, coverageinfo, debuginfo, llvm_util}; /// `TyCtxt` (and related cache datastructures) can't be move between threads. @@ -92,6 +92,7 @@ pub(crate) type CodegenCx<'ll, 'tcx> = GenericCx<'ll, FullCx<'ll, 'tcx>>; pub(crate) struct FullCx<'ll, 'tcx> { pub tcx: TyCtxt<'tcx>, pub scx: SimpleCx<'ll>, + pub tm: &'ll TargetMachine, pub use_dll_storage_attrs: bool, pub tls_model: llvm::ThreadLocalMode, @@ -660,6 +661,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { FullCx { tcx, scx: SimpleCx::new(llmod, llcx, tcx.data_layout.pointer_size()), + tm: llvm_module.tm.raw(), use_dll_storage_attrs, tls_model, codegen_unit, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 7bd604bdbbd76..330e7561b7854 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -17,7 +17,7 @@ use rustc_codegen_ssa::traits::*; use rustc_hir as hir; use rustc_hir::def_id::LOCAL_CRATE; use rustc_hir::find_attr; -use rustc_middle::mir::BinOp; +use rustc_middle::mir::{self, BinOp}; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; use rustc_middle::ty::offload_meta::OffloadMetadata; use rustc_middle::ty::{self, GenericArgsRef, Instance, SimdAlign, Ty, TyCtxt, TypingEnv}; @@ -32,6 +32,7 @@ use rustc_target::spec::{Arch, LlvmAbi}; use tracing::debug; use crate::abi::FnAbiLlvmExt; +use crate::attributes; use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::builder::gpu_offload::{ @@ -177,6 +178,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &mut self, instance: ty::Instance<'tcx>, args: &[OperandRef<'tcx, &'ll Value>], + _: &[mir::Operand<'tcx>], result_layout: ty::layout::TyAndLayout<'tcx>, result_place: Option>, span: Span, @@ -923,6 +925,47 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } + fn codegen_target_feature_available_at_call_site( + &mut self, + rust_feature_name: &str, + ) -> &'ll Value { + // SSA already returned `true` when this Rust feature is enabled on the current + // function. If there is no LLVM mapping, we cannot do any better from here. + let Some(llvm_feature) = crate::llvm_util::to_llvm_features(self.sess(), rust_feature_name) + else { + return self.const_bool(false); + }; + + // Target features can't be lost by inlining, so early return if it's already present. + // SSA should have already done this, but this is most accurate for implied features. + let mut llvm_features = llvm_feature.into_iter(); + let llvm_feature_name = llvm_features.next().unwrap().to_string(); + let enabled = llvm::FunctionHasTargetFeature(self.cx.tm, self.llfn(), &llvm_feature_name); + if enabled { + return self.const_bool(true); + } + + // If we're not optimizing, we won't run the LLVM pass + if self.sess().opts.optimize == rustc_session::config::OptLevel::No { + return self.const_bool(false); + } + + // Generate the marker to be replaced by the LLVM pass + let fn_ty = self.type_func(&[], self.type_i1()); + let llfn = self.cx.declare_cfn( + "rust.target_feature_available_at_call_site", + llvm::UnnamedAddr::No, + fn_ty, + ); + let nounwind = llvm::AttributeKind::NoUnwind.create_attr(self.cx.llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[nounwind]); + let call = self.call(fn_ty, None, None, llfn, &[], None, None); + let kind = self.cx.get_md_kind_id("rust.target_feature"); + let feature = self.cx.create_metadata(llvm_feature_name.as_bytes()); + self.cx.set_metadata_node(call, kind, &[feature]); + call + } + fn codegen_llvm_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 56c90582e4642..7a8cca7f37474 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -944,7 +944,6 @@ unsafe extern "C" { pub(crate) fn LLVMFloatTypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type; - // Operations on non-IEEE real types pub(crate) fn LLVMBFloatTypeInContext(C: &Context) -> &Type; @@ -2406,6 +2405,12 @@ unsafe extern "C" { pub(crate) fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; pub(crate) fn LLVMRustTargetHasMnemonic(T: &TargetMachine, s: *const c_char) -> bool; + pub(crate) fn LLVMRustFunctionHasTargetFeature( + TM: &TargetMachine, + F: &Value, + Feature: *const c_char, + FeatureLen: size_t, + ) -> bool; pub(crate) fn LLVMRustPrintTargetCPUs(TM: &TargetMachine, OutStr: &RustString); pub(crate) fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index e9bc6ae0e80ef..c5e4e11940a63 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -51,6 +51,14 @@ pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: &str) { unsafe { LLVMRustRemoveFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } } +pub(crate) fn FunctionHasTargetFeature<'ll>( + tm: &'ll TargetMachine, + llfn: &'ll Value, + feature: &str, +) -> bool { + unsafe { LLVMRustFunctionHasTargetFeature(tm, llfn, feature.as_c_char_ptr(), feature.len()) } +} + pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 0173b84a4d9a1..29a613dff1241 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1008,13 +1008,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { span_bug!(self.mir.span, "can't directly store to unaligned value"); } - let args: Vec<_> = + let mir_args: Vec<_> = args.iter().map(|arg| arg.node.clone()).collect(); + let codegen_args: Vec<_> = args.iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect(); let intrinsic_result = self.codegen_intrinsic_call( bx, instance, - &args, + &codegen_args, + &mir_args, result_layout, result_place, source_info, diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 99546bea65959..64b2cb5229774 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -1,5 +1,5 @@ use rustc_abi::{Align, FieldIdx, WrappingRange}; -use rustc_middle::mir::SourceInfo; +use rustc_middle::mir::{self, SourceInfo}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; @@ -13,7 +13,7 @@ use crate::common::{AtomicRmwBinOp, SynchronizationScope}; use crate::errors::InvalidMonomorphization; use crate::mir::operand::OperandRefBuilder; use crate::traits::*; -use crate::{MemFlags, meth, size_of_val}; +use crate::{MemFlags, meth, size_of_val, target_features}; fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, @@ -59,6 +59,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &mut Bx, instance: ty::Instance<'tcx>, args: &[OperandRef<'tcx, Bx::Value>], + mir_args: &[mir::Operand<'tcx>], result_layout: ty::layout::TyAndLayout<'tcx>, result_place: Option>, source_info: SourceInfo, @@ -124,6 +125,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | sym::atomic_fence | sym::atomic_singlethreadfence | sym::caller_location + | sym::target_feature_available_at_call_site | sym::return_address => {} _ => { span_bug!( @@ -589,10 +591,34 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::ZeroSized } + sym::target_feature_available_at_call_site => { + match target_features::target_feature_available_at_call_site_intrinsic( + bx.tcx(), + span, + self.instance.def_id(), + fn_args, + ) { + target_features::TargetFeatureAvailableAtCallSite::Known(enabled) => { + OperandValue::Immediate(bx.const_bool(enabled)) + } + target_features::TargetFeatureAvailableAtCallSite::CheckBackend(feature) => { + OperandValue::Immediate( + bx.codegen_target_feature_available_at_call_site(feature.as_str()), + ) + } + } + } + _ => { // Need to use backend-specific things in the implementation. - let result = - bx.codegen_intrinsic_call(instance, args, result_layout, result_place, span); + let result = bx.codegen_intrinsic_call( + instance, + args, + mir_args, + result_layout, + result_place, + span, + ); if let IntrinsicResult::Operand(op) = result { op } else { diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index 3c771f0eb7ec4..128303d28334a 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -5,7 +5,8 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId}; use rustc_middle::middle::codegen_fn_attrs::{TargetFeature, TargetFeatureKind}; use rustc_middle::query::Providers; -use rustc_middle::ty::TyCtxt; +use rustc_middle::span_bug; +use rustc_middle::ty::{self, TyCtxt}; use rustc_session::Session; use rustc_session::errors::feature_err; use rustc_session::lint::builtin::AARCH64_SOFTFLOAT_NEON; @@ -130,7 +131,7 @@ pub(crate) fn from_target_feature_attr( /// Computes the set of target features used in a function for the purposes of /// inline assembly. -fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxIndexSet { +pub(crate) fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxIndexSet { let mut target_features = tcx.sess.unstable_target_features.clone(); if tcx.def_kind(did).has_codegen_attrs() { let attrs = tcx.codegen_fn_attrs(did); @@ -569,3 +570,64 @@ pub(crate) fn provide(providers: &mut Providers) { ..*providers } } + +#[derive(Copy, Clone, Debug)] +pub enum TargetFeatureAvailableAtCallSite { + /// The feature's presence is known immediately, including `false` for error conditions. + Known(bool), + /// It's not known if the feature is available at the call site. It's safe to return `false`, + /// but the backend may be able to refine the answer (e.g. the LLVM post-inline pass) + CheckBackend(Symbol), +} + +/// Implementation for the `target_feature_available_at_call_site` intrinsic. +/// +/// This function performs the backend-independent implementation: +/// * If the feature is enabled for the current function, we can immediately return true. +/// * If the feature is not enabled for the current function, defer to the backend to account +/// for inlining. +/// * If an error occurs, emit the diagnostic and return false. +/// +/// When deferred to the backend, it's always safe to return false, but optimization opportunities +/// are lost. +/// The LLVM backend, for example, emits a marker function that is replaced with true or false +/// with a post-inlining pass. +pub fn target_feature_available_at_call_site_intrinsic<'tcx>( + tcx: TyCtxt<'tcx>, + span: Span, + did: DefId, + generic_args: ty::GenericArgsRef<'tcx>, +) -> TargetFeatureAvailableAtCallSite { + let Some(feature_bytes) = + generic_args.const_at(0).try_to_value().and_then(|feature| feature.try_to_raw_bytes(tcx)) + else { + span_bug!(span, "wrong const argument for target_feature_available_at_call_site"); + }; + + let feature_len = + feature_bytes.iter().position(|&byte| byte == 0).unwrap_or(feature_bytes.len()); + let Some(feature_name) = std::str::from_utf8(&feature_bytes[..feature_len]).ok() else { + tcx.dcx().span_err( + span, + "`target_feature_available_at_call_site` requires a UTF-8 feature name", + ); + return TargetFeatureAvailableAtCallSite::Known(false); + }; + let feature = Symbol::intern(feature_name); + + let rust_target_features = tcx.rust_target_features(LOCAL_CRATE); + let Some(&stability) = rust_target_features.get(feature_name) else { + tcx.dcx().span_err(span, format!("unknown target feature `{feature}`")); + return TargetFeatureAvailableAtCallSite::Known(false); + }; + if let Err(reason) = stability.toggle_allowed() { + tcx.dcx().span_err(span, format!("cannot use target feature `{feature}`: {reason}")); + return TargetFeatureAvailableAtCallSite::Known(false); + } + + if asm_target_features(tcx, did).contains(&feature) { + TargetFeatureAvailableAtCallSite::Known(true) + } else { + TargetFeatureAvailableAtCallSite::CheckBackend(feature) + } +} diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs index 47144834b5072..7e4b17b241368 100644 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -1,4 +1,4 @@ -use rustc_middle::ty; +use rustc_middle::{mir, ty}; use rustc_span::Span; use super::BackendTypes; @@ -26,11 +26,17 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes { &mut self, instance: ty::Instance<'tcx>, args: &[OperandRef<'tcx, Self::Value>], + mir_args: &[mir::Operand<'tcx>], result_layout: ty::layout::TyAndLayout<'tcx>, result_place: Option>, span: Span, ) -> IntrinsicResult<'tcx, Self::Value>; + fn codegen_target_feature_available_at_call_site( + &mut self, + llvm_feature_name: &str, + ) -> Self::Value; + fn codegen_llvm_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 9dd7bb8058afc..5baac385145ff 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -207,6 +207,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::sqrtf64 | sym::sqrtf128 | sym::sub_with_overflow + | sym::target_feature_available_at_call_site | sym::three_way_compare | sym::truncf16 | sym::truncf32 @@ -683,6 +684,7 @@ pub(crate) fn check_intrinsic_type( sym::black_box => (1, 0, vec![param(0)], param(0)), sym::is_val_statically_known => (1, 0, vec![param(0)], tcx.types.bool), + sym::target_feature_available_at_call_site => (0, 1, Vec::new(), tcx.types.bool), sym::const_eval_select => (4, 0, vec![param(0), param(1), param(2)], param(3)), diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index fe5b8edce4a1d..4e3a4dff96745 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -12,9 +12,11 @@ #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/Bitcode/BitcodeWriterPass.h" #include "llvm/CodeGen/CommandFlags.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/AssemblyAnnotationWriter.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Verifier.h" #include "llvm/IRPrinter/IRPrintingPasses.h" @@ -118,6 +120,64 @@ extern "C" bool LLVMRustTargetHasMnemonic(LLVMTargetMachineRef TM, return false; } +/// During MIR lowering, the target_feature_available_at_call_site intrinsic +/// is lowered to a marker function call carrying the LLVM feature as a +/// metadata string argument. +/// +/// This pass replaces those markers with true or false depending on whether the +/// feature is enabled in the caller. To be useful, this pass must run after +/// inlining. +class TargetFeatureAvailableAtCallSitePass + : public PassInfoMixin { + TargetMachine *TM; + +public: + explicit TargetFeatureAvailableAtCallSitePass(TargetMachine *TM) : TM(TM) {} + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &) { + Function *MarkerDecl = + M.getFunction("rust.target_feature_available_at_call_site"); + if (MarkerDecl == nullptr) + return PreservedAnalyses::all(); + + auto FeatureKindID = M.getContext().getMDKindID("rust.target_feature"); + + SmallVector CallsToErase; + for (User *U : MarkerDecl->users()) { + auto *Call = dyn_cast(U); + if (!Call || Call->getCalledFunction() != MarkerDecl) + continue; + + auto *FeatureNode = Call->getMetadata(FeatureKindID); + auto *FeatureMetadata = + FeatureNode && FeatureNode->getNumOperands() == 1 + ? dyn_cast(FeatureNode->getOperand(0)) + : nullptr; + if (FeatureMetadata == nullptr) + continue; + + SmallString<64> EnabledFeature("+"); + EnabledFeature += FeatureMetadata->getString(); + + Function *Caller = Call->getFunction(); + const TargetSubtargetInfo *Subtarget = TM->getSubtargetImpl(*Caller); + bool Enabled = + Subtarget != nullptr && Subtarget->checkFeatures(EnabledFeature); + + Call->replaceAllUsesWith(ConstantInt::getBool(M.getContext(), Enabled)); + CallsToErase.push_back(Call); + } + + if (CallsToErase.empty()) + return PreservedAnalyses::all(); + + for (CallInst *Call : CallsToErase) + Call->eraseFromParent(); + + return PreservedAnalyses::none(); + } +}; + enum class LLVMRustCodeModel { Tiny, Small, @@ -723,10 +783,19 @@ extern "C" LLVMRustResult LLVMRustOptimize( // the PassBuilder does not create a pipeline. std::vector> PipelineStartEPCallbacks; + std::vector> + OptimizerEarlyEPCallbacks; std::vector> OptimizerLastEPCallbacks; + OptimizerEarlyEPCallbacks.push_back([TM](ModulePassManager &MPM, + OptimizationLevel Level, + ThinOrFullLTOPhase Phase) { + MPM.addPass(TargetFeatureAvailableAtCallSitePass(TM)); + }); + if (!IsLinkerPluginLTO && SanitizerOptions && SanitizerOptions->SanitizeCFI && !NoPrepopulatePasses) { PipelineStartEPCallbacks.push_back( @@ -864,6 +933,8 @@ extern "C" LLVMRustResult LLVMRustOptimize( if (!NoPrepopulatePasses) { for (const auto &C : PipelineStartEPCallbacks) PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerEarlyEPCallbacks) + PB.registerOptimizerEarlyEPCallback(C); for (const auto &C : OptimizerLastEPCallbacks) PB.registerOptimizerLastEPCallback(C); @@ -918,6 +989,8 @@ extern "C" LLVMRustResult LLVMRustOptimize( // add the verifier, instrumentation, etc passes if they were requested for (const auto &C : PipelineStartEPCallbacks) C(MPM, OptLevel); + for (const auto &C : OptimizerEarlyEPCallbacks) + C(MPM, OptLevel, ThinOrFullLTOPhase::None); for (const auto &C : OptimizerLastEPCallbacks) C(MPM, OptLevel, ThinOrFullLTOPhase::None); } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 3b8e6f6415365..50c50f7e39499 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -9,6 +9,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -36,6 +37,7 @@ #include "llvm/Support/Signals.h" #include "llvm/Support/Timer.h" #include "llvm/Support/ToolOutputFile.h" +#include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/ValueMapper.h" #include @@ -68,6 +70,9 @@ using namespace llvm; using namespace llvm::sys; using namespace llvm::object; +typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; +DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + // This opcode is an LLVM detail that could hypothetically change (?), so // verify that the hard-coded value in `dwarf_const.rs` still agrees with LLVM. static_assert(dwarf::DW_OP_LLVM_fragment == 0x1000); @@ -1096,6 +1101,21 @@ extern "C" void LLVMRustRemoveFnAttribute(LLVMValueRef Fn, const char *Name, } } +extern "C" bool LLVMRustFunctionHasTargetFeature(LLVMTargetMachineRef TMRef, + LLVMValueRef F, + const char *Feature, + size_t FeatureLen) { + if (auto *Fn = dyn_cast(unwrap(F))) { + TargetMachine *TM = unwrap(TMRef); + if (const TargetSubtargetInfo *Subtarget = TM->getSubtargetImpl(*Fn)) { + SmallString<64> EnabledFeature("+"); + EnabledFeature += StringRef(Feature, FeatureLen); + return Subtarget->checkFeatures(EnabledFeature); + } + } + return false; +} + extern "C" void LLVMRustGlobalAddMetadata(LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { unwrap(Global)->addMetadata(Kind, *unwrap(MD)); diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 71eae246ebaff..83f445923626c 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2076,6 +2076,7 @@ symbols! { target_family, target_feature, target_feature_11, + target_feature_available_at_call_site, target_feature_inline_always, target_has_atomic, target_has_atomic_load_store, diff --git a/library/core/src/intrinsics/simd/mod.rs b/library/core/src/intrinsics/simd/mod.rs index 9311dcc9bd00e..ecfa8c3239071 100644 --- a/library/core/src/intrinsics/simd/mod.rs +++ b/library/core/src/intrinsics/simd/mod.rs @@ -839,3 +839,62 @@ pub unsafe fn simd_flog2(a: T) -> T; #[rustc_intrinsic] #[rustc_nounwind] pub unsafe fn simd_flog(a: T) -> T; + +/// Returns whether the named target feature is available at the current call site. +/// This will not emit any runtime detection code; it always turns into `true` or `false` at some +/// point during compilation. +/// +/// "Available" in this context means known to be enabled in the calling function. +/// At a minimum, being enabled globally or in `#[target_features]` of the function containing +/// this intrinsic is considered available. Depending on codegen options and backend, the features +/// of the function the intrinsic call is inlined into may also be accounted for. This is intended +/// to support negligible or zero overhead branching on feature availability, for scenarios where +/// runtime detection has too much overhead and globally enabled features are not sufficient. +/// +/// This intrinsic takes a null-terminated feature name passed as a byte array. For a more +/// ergonomic interface, use the [`target_feature_available_at_call_site!`] macro, which accepts +/// a string literal. +/// +/// The LLVM backend implements this intrinsic by lowering to a marker that is resolved by a +/// post-inlining pass. +/// +/// # Safety +/// +/// The return value of this intrinsic is call-site dependent and may vary between calls to the +/// same function, depending on inlining and codegen. Callers must not rely on the result being +/// stable across calls or multiple uses of the same function containing the intrinsic. +/// +/// A return value of `true` guarantees that the feature is enabled at the call site and it is +/// safe to call other functions that require the feature. +/// A return value of `false` carries no information and does not indicate whether the feature is +/// enabled or not. +#[rustc_nounwind] +#[unstable(feature = "core_intrinsics", issue = "none")] +#[rustc_intrinsic] +pub fn target_feature_available_at_call_site() -> bool; + +/// Returns whether a target feature is enabled at the call site. +/// +/// This macro is a more convenient interface for [`target_feature_available_at_call_site()`]. +#[unstable(feature = "core_intrinsics", issue = "none")] +#[allow_internal_unstable(adt_const_params)] +#[diagnostic::on_unmatched_args( + note = "this macro expects a string literal target feature name, like `target_feature_available_at_call_site!(\"avx\")`" +)] +#[rustc_macro_transparency = "semiopaque"] +pub macro target_feature_available_at_call_site($feature:literal) {{ + ::core::intrinsics::simd::target_feature_available_at_call_site::< + { + let bytes = $feature.as_bytes(); + + assert!(bytes.len() <= 100, "feature string too long"); + + // FIXME(const-hack) can't use subslicing yet + let mut out = [0u8; 100]; + let (dst, _) = out.split_at_mut(bytes.len()); + dst.copy_from_slice(bytes); + + out + }, + >() +}} diff --git a/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs new file mode 100644 index 0000000000000..ec4501db22cb7 --- /dev/null +++ b/tests/assembly-llvm/target-feature-available-at-call-site-fma.rs @@ -0,0 +1,63 @@ +//@ assembly-output: emit-asm +//@ compile-flags: --crate-type=lib -Copt-level=3 -C llvm-args=-x86-asm-syntax=intel -Ctarget-feature=-avx,-fma +//@ only-x86_64 +//@ ignore-sgx +//@ ignore-backends: gcc + +#![feature(core_intrinsics)] + +use std::arch::x86_64::{_mm_add_ss, _mm_cvtss_f32, _mm_fmadd_ss, _mm_mul_ss, _mm_set_ss}; +use std::intrinsics::simd::target_feature_available_at_call_site; + +#[inline(always)] +fn maybe_fma(x: f32, y: f32, z: f32) -> f32 { + if target_feature_available_at_call_site!("fma") { x.mul_add(y, z) } else { x * y + z } +} + +#[inline(always)] +fn maybe_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { + unsafe { + let x = _mm_set_ss(x); + let y = _mm_set_ss(y); + let z = _mm_set_ss(z); + let result = if target_feature_available_at_call_site!("fma") { + _mm_fmadd_ss(x, y, z) + } else { + _mm_add_ss(_mm_mul_ss(x, y), z) + }; + _mm_cvtss_f32(result) + } +} + +// CHECK-LABEL: with_fma: +// CHECK: vfmadd +// CHECK-NOT: mulss +// CHECK-NOT: addss +#[no_mangle] +#[target_feature(enable = "avx,fma")] +pub fn with_fma(x: f32, y: f32, z: f32) -> f32 { + maybe_fma(x, y, z) +} + +// CHECK-LABEL: without_fma: +// CHECK: mulss +// CHECK: addss +// CHECK-NOT: vfmadd +#[no_mangle] +pub fn without_fma(x: f32, y: f32, z: f32) -> f32 { + maybe_fma(x, y, z) +} + +#[no_mangle] +#[target_feature(enable = "avx,fma")] +pub fn with_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { + maybe_fma_arch_intrinsic(x, y, z) +} + +#[no_mangle] +pub fn without_fma_arch_intrinsic(x: f32, y: f32, z: f32) -> f32 { + maybe_fma_arch_intrinsic(x, y, z) +} + +// CHECK: with_fma_arch_intrinsic = with_fma +// CHECK: without_fma_arch_intrinsic = without_fma diff --git a/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs b/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs new file mode 100644 index 0000000000000..ec08047bbe62e --- /dev/null +++ b/tests/codegen-llvm/target-feature-available-at-call-site-o0.rs @@ -0,0 +1,42 @@ +//@ revisions: NEG_ONLY NEG_POS POS_NEG +//@ compile-flags: --crate-type=lib -Copt-level=0 +//@ [NEG_ONLY] compile-flags: -Ctarget-feature=-fma +//@ [NEG_POS] compile-flags: -Ctarget-feature=-fma,+fma +//@ [POS_NEG] compile-flags: -Ctarget-feature=+fma,-fma +//@ only-x86_64 +//@ ignore-backends: gcc + +// opt-level=0 has different inlining properties, so ensure that the pass still works. + +#![feature(core_intrinsics)] + +use std::intrinsics::simd::target_feature_available_at_call_site; + +// NEG_ONLY-LABEL: @with_fma( +// NEG_ONLY-NOT: rust.target_feature_available_at_call_site +// NEG_ONLY: ret i1 true +// NEG_POS-LABEL: @with_fma( +// NEG_POS-NOT: rust.target_feature_available_at_call_site +// NEG_POS: ret i1 true +// POS_NEG-LABEL: @with_fma( +// POS_NEG-NOT: rust.target_feature_available_at_call_site +// POS_NEG: ret i1 true +#[no_mangle] +#[target_feature(enable = "fma")] +pub fn with_fma() -> bool { + target_feature_available_at_call_site!("fma") +} + +// NEG_ONLY-LABEL: @without_fma( +// NEG_ONLY-NOT: rust.target_feature_available_at_call_site +// NEG_ONLY: ret i1 false +// NEG_POS-LABEL: @without_fma( +// NEG_POS-NOT: rust.target_feature_available_at_call_site +// NEG_POS: ret i1 true +// POS_NEG-LABEL: @without_fma( +// POS_NEG-NOT: rust.target_feature_available_at_call_site +// POS_NEG: ret i1 false +#[no_mangle] +pub fn without_fma() -> bool { + target_feature_available_at_call_site!("fma") +} diff --git a/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs b/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs new file mode 100644 index 0000000000000..e804224b86353 --- /dev/null +++ b/tests/codegen-llvm/target-feature-available-at-call-site-rust-only-feature.rs @@ -0,0 +1,28 @@ +//@ compile-flags: --crate-type=lib -Copt-level=3 +//@ only-aarch64 +//@ ignore-backends: gcc + +// Test intrinsic on a feature that doesn't map to an LLVM feature. +// Can't resolve after inlining, but can get the current function's +// feature presence. + +#![feature(core_intrinsics)] + +use std::intrinsics::simd::target_feature_available_at_call_site; + +// CHECK-LABEL: @with_tme( +// CHECK-NOT: rust.target_feature_available_at_call_site +// CHECK: ret i1 true +#[no_mangle] +#[target_feature(enable = "tme")] +pub fn with_tme() -> bool { + target_feature_available_at_call_site!("tme") +} + +// CHECK-LABEL: @without_tme( +// CHECK-NOT: rust.target_feature_available_at_call_site +// CHECK: ret i1 false +#[no_mangle] +pub fn without_tme() -> bool { + target_feature_available_at_call_site!("tme") +} diff --git a/tests/codegen-llvm/target-feature-available-at-call-site.rs b/tests/codegen-llvm/target-feature-available-at-call-site.rs new file mode 100644 index 0000000000000..6a685e430132e --- /dev/null +++ b/tests/codegen-llvm/target-feature-available-at-call-site.rs @@ -0,0 +1,32 @@ +//@ compile-flags: --crate-type=lib -Copt-level=3 -Ctarget-feature=-avx +//@ only-x86_64 +//@ ignore-backends: gcc + +#![feature(core_intrinsics)] + +use std::intrinsics::simd::target_feature_available_at_call_site; + +#[inline] +pub fn avx_branch_value() -> i32 { + if target_feature_available_at_call_site!("avx") { 1 } else { 0 } +} + +// CHECK-LABEL: @with_avx( +// CHECK-NOT: rust.target_feature_available_at_call_site +// CHECK: ret i32 1 +#[no_mangle] +#[target_feature(enable = "avx")] +pub fn with_avx() -> i32 { + avx_branch_value() +} + +// CHECK-LABEL: @without_avx( +// CHECK-NOT: rust.target_feature_available_at_call_site +// CHECK: ret i32 0 +#[no_mangle] +pub fn without_avx() -> i32 { + avx_branch_value() +} + +// CHECK: attributes #0 = {{.*}}"target-features"="{{[^"]*}}+avx{{.*}}" +// CHECK: attributes #1 = {{.*}}"target-features"="{{[^"]*}}-avx{{.*}}" diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs new file mode 100644 index 0000000000000..3d7625709d59e --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-const.rs @@ -0,0 +1,8 @@ +#![feature(core_intrinsics)] + +const HAS_FMA: bool = std::intrinsics::simd::target_feature_available_at_call_site!("fma"); +//~^ ERROR cannot call non-const intrinsic `target_feature_available_at_call_site` in constants + +fn main() { + let _ = HAS_FMA; +} diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr new file mode 100644 index 0000000000000..a88d842c879da --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-const.stderr @@ -0,0 +1,10 @@ +error: cannot call non-const intrinsic `target_feature_available_at_call_site` in constants + --> $DIR/target-feature-available-at-call-site-const.rs:3:23 + | +LL | const HAS_FMA: bool = std::intrinsics::simd::target_feature_available_at_call_site!("fma"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `std::intrinsics::simd::target_feature_available_at_call_site` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: aborting due to 1 previous error + diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs new file mode 100644 index 0000000000000..7e10c69ccf4c1 --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.rs @@ -0,0 +1,14 @@ +//@ compile-flags: --crate-type=lib --emit=llvm-ir +//@ ignore-backends: gcc + +#![feature(core_intrinsics)] + +use std::hint::black_box; +use std::intrinsics::simd::target_feature_available_at_call_site; + +#[no_mangle] +pub fn check() { + let feature = black_box("fma"); + let _ = target_feature_available_at_call_site!(feature); + //~^ ERROR no rules expected `feature` +} diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr new file mode 100644 index 0000000000000..61328c6ed8a5d --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-nonconst.stderr @@ -0,0 +1,12 @@ +error: no rules expected `feature` + --> $DIR/target-feature-available-at-call-site-nonconst.rs:12:52 + | +LL | let _ = target_feature_available_at_call_site!(feature); + | ^^^^^^^ no rules expected this token in macro call + | +note: while trying to match meta-variable `$feature:literal` + --> $SRC_DIR/core/src/intrinsics/simd/mod.rs:LL:COL + = note: this macro expects a string literal target feature name, like `target_feature_available_at_call_site!("avx")` + +error: aborting due to 1 previous error + diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs new file mode 100644 index 0000000000000..595285c971b7a --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.rs @@ -0,0 +1,13 @@ +//@ compile-flags: --crate-type=lib --emit=llvm-ir +//@ ignore-backends: gcc + +#![feature(core_intrinsics)] + +use std::hint::black_box; +use std::intrinsics::simd::target_feature_available_at_call_site; + +#[no_mangle] +pub fn check() { + let _ = target_feature_available_at_call_site!("definitely-not-a-feature"); + //~^ ERROR unknown target feature `definitely-not-a-feature` +} diff --git a/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr new file mode 100644 index 0000000000000..ec86dc4d01b69 --- /dev/null +++ b/tests/ui/intrinsics/target-feature-available-at-call-site-unknown.stderr @@ -0,0 +1,10 @@ +error: unknown target feature `definitely-not-a-feature` + --> $DIR/target-feature-available-at-call-site-unknown.rs:11:13 + | +LL | let _ = target_feature_available_at_call_site!("definitely-not-a-feature"); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `target_feature_available_at_call_site` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: aborting due to 1 previous error +