diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index ad745913399d4..1169c94ac49a6 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1879,12 +1879,25 @@ impl BackendRepr { } } - /// Returns `true` if this is a scalar type + /// Returns `true` if this is specifically a [`Self::Scalar`] type. + /// + /// This excludes SIMD types. #[inline] pub fn is_scalar(&self) -> bool { matches!(*self, BackendRepr::Scalar(_)) } + /// Returns `true` if this is a scalar type or SIMD type. + #[inline] + pub fn is_scalar_or_simd(&self) -> bool { + matches!( + *self, + BackendRepr::Scalar(_) + | BackendRepr::SimdVector { .. } + | BackendRepr::SimdScalableVector { .. } + ) + } + /// Returns `true` if this is a bool #[inline] pub fn is_bool(&self) -> bool { @@ -2330,6 +2343,25 @@ impl LayoutData { } } + /// In the backend, a value with this type and layout is fully represented by + /// its SSA value(s), independent of the contents of memory. + /// + /// For example, you can swap by reading both then writing both without using + /// an alloca because the store of one cannot affect the value of the other. + /// + /// Any projection into a standalone type must also yield a standalone type, + /// since it might not be in memory at all. + #[inline] + pub fn is_ssa_standalone(&self) -> bool { + match self.backend_repr { + BackendRepr::Memory { .. } => self.is_zst(), + BackendRepr::Scalar(..) + | BackendRepr::ScalarPair { .. } + | BackendRepr::SimdVector { .. } + | BackendRepr::SimdScalableVector { .. } => true, + } + } + /// Checks if these two `Layout` are equal enough to be considered "the same for all function /// call ABIs". Note however that real ABIs depend on more details that are not reflected in the /// `Layout`; the `PassMode` need to be compared as well. Also note that we assume diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index da17f05bb8a32..7671d2e026b03 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -1054,7 +1054,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let val = if place.val.llextra.is_some() { // FIXME: Merge with the `else` below? OperandValue::Ref(place.val) - } else if place.layout.is_gcc_immediate() { + } else if place.layout.backend_repr.is_scalar_or_simd() { let load = self.load(place.layout.gcc_type(self), place.val.llval, place.val.align); OperandValue::Immediate( if let abi::BackendRepr::Scalar(ref scalar) = place.layout.backend_repr { diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs index 227b513c0ff30..f2ce7bca1e338 100644 --- a/compiler/rustc_codegen_gcc/src/type_of.rs +++ b/compiler/rustc_codegen_gcc/src/type_of.rs @@ -154,8 +154,6 @@ fn uncached_gcc_type<'gcc, 'tcx>( } pub trait LayoutGccExt<'tcx> { - fn is_gcc_immediate(&self) -> bool; - fn is_gcc_scalar_pair(&self) -> bool; fn gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>; fn immediate_gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>; fn scalar_gcc_type_at<'gcc>( @@ -177,25 +175,6 @@ pub trait LayoutGccExt<'tcx> { } impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> { - fn is_gcc_immediate(&self) -> bool { - match self.backend_repr { - BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true, - // FIXME(rustc_scalable_vector): Not yet implemented in rustc_codegen_gcc. - BackendRepr::SimdScalableVector { .. } => todo!(), - BackendRepr::ScalarPair { .. } | BackendRepr::Memory { .. } => false, - } - } - - fn is_gcc_scalar_pair(&self) -> bool { - match self.backend_repr { - BackendRepr::ScalarPair { .. } => true, - BackendRepr::Scalar(_) - | BackendRepr::SimdVector { .. } - | BackendRepr::SimdScalableVector { .. } - | BackendRepr::Memory { .. } => false, - } - } - /// Gets the GCC type corresponding to a Rust type, i.e., `rustc_middle::ty::Ty`. /// The pointee type of the pointer in `PlaceRef` is always this type. /// For sized types, it is also the right LLVM type for an `alloca` @@ -350,14 +329,6 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { layout.immediate_gcc_type(self) } - fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool { - layout.is_gcc_immediate() - } - - fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool { - layout.is_gcc_scalar_pair() - } - fn scalar_pair_element_backend_type( &self, layout: TyAndLayout<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index af6c24018028a..55bb6f1abeb96 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -770,7 +770,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let val = if let Some(_) = place.val.llextra { // FIXME: Merge with the `else` below? OperandValue::Ref(place.val) - } else if place.layout.is_llvm_immediate() { + } else if place.layout.backend_repr.is_scalar_or_simd() { let mut const_llval = None; let llty = place.layout.llvm_type(self); if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index c0593bb467796..1dd460c409737 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -324,9 +324,33 @@ impl CodegenBackend for LlvmCodegenBackend { target_config(sess) } + /// Intrinsics whose fallback body will not be used by the LLVM backend. fn replaced_intrinsics(&self) -> Vec { - let mut will_not_use_fallback = - vec![sym::unchecked_funnel_shl, sym::unchecked_funnel_shr, sym::carrying_mul_add]; + #[rustfmt::skip] + let mut will_not_use_fallback = vec![ + // These are mapped to LLVM intrinsics instead. + sym::unchecked_funnel_shl, + sym::unchecked_funnel_shr, + sym::carrying_mul_add, + + // Fallback via libm, but the LLVM intrinsic is used instead. + sym::sinf16, sym::sinf32, sym::sinf64, + sym::cosf16, sym::cosf32, sym::cosf64, + sym::powf16, sym::powf32, sym::powf64, + sym::expf16, sym::expf32, sym::expf64, + sym::exp2f16, sym::exp2f32, sym::exp2f64, + sym::logf16, sym::logf32, sym::logf64, + sym::log10f16, sym::log10f32, sym::log10f64, + sym::log2f16, sym::log2f32, sym::log2f64, + + // Fallback via f32 or f64, but the LLVM intrinsic is used instead. + sym::floorf16, sym::ceilf16, sym::truncf16, + sym::round_ties_even_f16, sym::roundf16, + sym::sqrtf16, sym::powif16, + sym::fmaf16, + + sym::copysignf16, sym::copysignf32, sym::copysignf64, sym::copysignf128, + ]; if llvm_util::get_version() >= (22, 0, 0) { will_not_use_fallback.push(sym::carryless_mul); diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 0d49971f52533..c422515c1a64b 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -296,12 +296,6 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { layout.immediate_llvm_type(self) } - fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool { - layout.is_llvm_immediate() - } - fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool { - layout.is_llvm_scalar_pair() - } fn scalar_pair_element_backend_type( &self, layout: TyAndLayout<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index d1c2e1d567e40..0caf7e761bdb1 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -209,8 +209,6 @@ impl<'a, 'tcx> CodegenCx<'a, 'tcx> { } pub(crate) trait LayoutLlvmExt<'tcx> { - fn is_llvm_immediate(&self) -> bool; - fn is_llvm_scalar_pair(&self) -> bool; fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type; @@ -223,25 +221,6 @@ pub(crate) trait LayoutLlvmExt<'tcx> { } impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { - fn is_llvm_immediate(&self) -> bool { - match self.backend_repr { - BackendRepr::Scalar(_) - | BackendRepr::SimdVector { .. } - | BackendRepr::SimdScalableVector { .. } => true, - BackendRepr::ScalarPair { .. } | BackendRepr::Memory { .. } => false, - } - } - - fn is_llvm_scalar_pair(&self) -> bool { - match self.backend_repr { - BackendRepr::ScalarPair { .. } => true, - BackendRepr::Scalar(_) - | BackendRepr::SimdVector { .. } - | BackendRepr::SimdScalableVector { .. } - | BackendRepr::Memory { .. } => false, - } - } - /// Gets the LLVM type corresponding to a Rust type, i.e., `rustc_middle::ty::Ty`. /// The pointee type of the pointer in `PlaceRef` is always this type. /// For sized types, it is also the right LLVM type for an `alloca` diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index fb5734ba087c6..1074a14d4ee6a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -83,12 +83,11 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> LocalKind::Unused => { let ty = fx.monomorphize(decl.ty); let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span); - *kind = - if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) { - LocalKind::SSA(location) - } else { - LocalKind::Memory - }; + *kind = if let abi::BackendRepr::Memory { .. } = layout.backend_repr { + LocalKind::Memory + } else { + LocalKind::SSA(location) + }; } LocalKind::SSA(_) => *kind = LocalKind::Memory, } @@ -157,7 +156,7 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> } } debug_assert!( - !self.fx.cx.is_backend_ref(layout), + layout.is_ssa_standalone(), "Post-projection {place_ref:?} layout should be non-Ref, but it's {layout:?}", ); } diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 284b7d3f14fed..33ec40c5790db 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -85,7 +85,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if let sym::typed_swap_nonoverlapping = name { let pointee_ty = fn_args.type_at(0); let pointee_layout = bx.layout_of(pointee_ty); - if !bx.is_backend_ref(pointee_layout) + if pointee_layout.is_ssa_standalone() // But if we're not going to optimize, trying to use the fallback // body just makes things worse, so don't bother. || bx.sess().opts.optimize == OptLevel::No @@ -609,7 +609,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; debug_assert!( - op_val.is_expected_variant_for_type(bx.cx(), result_layout), + op_val.is_expected_variant_for_type(result_layout), "[{name:?}] Value {op_val:?} is wrong for type {result_layout:?}", ); diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 2fa283a6c2eb4..0fd8a091e3ee3 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -501,7 +501,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( PassMode::Direct(_) => { let llarg = bx.get_param(llarg_idx); llarg_idx += 1; - debug_assert!(bx.is_backend_immediate(arg.layout)); + debug_assert!(arg.layout.backend_repr.is_scalar_or_simd()); return local(OperandRef { val: OperandValue::Immediate(llarg), layout: arg.layout, diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 22b8272684d36..1dbbc3fd28fb9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -28,25 +28,43 @@ use crate::traits::*; pub enum OperandValue { /// A reference to the actual operand. The data is guaranteed /// to be valid for the operand's lifetime. - /// The second value, if any, is the extra data (vtable or length) + /// The [`PlaceValue::llextra`], if any, is the extra data (vtable or length) /// which indicates that it refers to an unsized rvalue. /// /// An `OperandValue` *must* be this variant for any type for which - /// [`LayoutTypeCodegenMethods::is_backend_ref`] returns `true`. + /// [`rustc_abi::LayoutData::is_ssa_standalone`] returns `false`. /// (That basically amounts to "isn't one of the other variants".) /// /// This holds a [`PlaceValue`] (like a [`PlaceRef`] does) with a pointer /// to the location holding the value. The type behind that pointer is the /// one returned by [`LayoutTypeCodegenMethods::backend_type`]. + /// + /// Note that a [`load_operand`] which produces this variant didn't actually + /// *load* anything; it just put the pointer-to-place into this variant. + /// + /// [`load_operand`]: BuilderMethods::load_operand Ref(PlaceValue), /// A single LLVM immediate value. /// - /// An `OperandValue` *must* be this variant for any type for which - /// [`LayoutTypeCodegenMethods::is_backend_immediate`] returns `true`. + /// An `OperandValue` *must* be this variant for any type that's + /// [`BackendRepr::Scalar`], [`BackendRepr::SimdVector`], or + /// [`BackendRepr::SimdScalableVector`]. + /// /// The backend value in this variant must be the *immediate* backend type, /// as returned by [`LayoutTypeCodegenMethods::immediate_backend_type`]. + /// + /// Notably, that means that in LLVM a `bool` is `i1` here, even though we + /// load and store `bool`s as LLVM's `i8` type. Methods such as + /// [`BuilderMethods::load_operand`] and [`OperandRef::store_with_annotation`] + /// will handle that correctly, but if you're using the value directly or + /// implementing such methods, be sure to convert using + /// [`BuilderMethods::from_immediate`] and [`BuilderMethods::to_immediate_scalar`] + /// in the appropriate places. Immediate(V), - /// A pair of immediate LLVM values. Used by wide pointers too. + /// A pair of immediate LLVM values. + /// + /// Notably this includes wide pointers, where the two values are the pointer + /// and the metadata (slice length, vtable pointer, etc). /// /// # Invariants /// - For `Pair(a, b)`, `a` is always at offset 0, but may have `FieldIdx(1..)` @@ -54,11 +72,10 @@ pub enum OperandValue { /// - `a` and `b` will have a different FieldIdx, but otherwise `b`'s may be lower /// or they may not be adjacent, due to arbitrary numbers of 1ZST fields that /// will not affect the shape of the data which determines if `Pair` will be used. - /// - An `OperandValue` *must* be this variant for any type for which - /// [`LayoutTypeCodegenMethods::is_backend_scalar_pair`] returns `true`. + /// - An `OperandValue` *must* be this variant for any type that's [`BackendRepr::ScalarPair`]. /// - The backend values in this variant must be the *immediate* backend types, /// as returned by [`LayoutTypeCodegenMethods::scalar_pair_element_backend_type`] - /// with `immediate: true`. + /// with `immediate: true`. See the note in [`Self::Immediate`]. Pair(V, V), /// A value taking no bytes, and which therefore needs no LLVM value at all. /// @@ -104,16 +121,18 @@ impl OperandValue { } #[must_use] - pub(crate) fn is_expected_variant_for_type<'tcx, Cx: LayoutTypeCodegenMethods<'tcx>>( - &self, - cx: &Cx, - ty: TyAndLayout<'tcx>, - ) -> bool { - match self { - OperandValue::ZeroSized => ty.is_zst(), - OperandValue::Immediate(_) => cx.is_backend_immediate(ty), - OperandValue::Pair(_, _) => cx.is_backend_scalar_pair(ty), - OperandValue::Ref(_) => cx.is_backend_ref(ty), + pub(crate) fn is_expected_variant_for_type<'tcx>(&self, ty: TyAndLayout<'tcx>) -> bool { + match (self, ty.backend_repr) { + (OperandValue::ZeroSized, BackendRepr::Memory { .. }) => ty.is_zst(), + (OperandValue::Ref(_), BackendRepr::Memory { .. }) => !ty.is_zst(), + ( + OperandValue::Immediate(_), + BackendRepr::Scalar(..) + | BackendRepr::SimdVector { .. } + | BackendRepr::SimdScalableVector { .. }, + ) => true, + (OperandValue::Pair(_, _), BackendRepr::ScalarPair { .. }) => true, + _ => false, } } } @@ -370,11 +389,11 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let field = self.layout.field(bx.cx(), i); let offset = self.layout.fields.offset(i); - if !bx.is_backend_ref(self.layout) && bx.is_backend_ref(field) { + if self.layout.is_ssa_standalone() && !field.is_ssa_standalone() { // Part of https://github.com/rust-lang/compiler-team/issues/838 span_bug!( fx.mir.span, - "Non-ref type {self:?} cannot project to ref field type {field:?}", + "Standalone type {self:?} cannot project to memory-dependent field type {field:?}", ); } @@ -944,18 +963,23 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { layout: TyAndLayout<'tcx>, ) -> OperandValue { assert!(layout.is_sized()); - if layout.is_zst() { - OperandValue::ZeroSized - } else if bx.cx().is_backend_immediate(layout) { - let ibty = bx.cx().immediate_backend_type(layout); - OperandValue::Immediate(bx.const_poison(ibty)) - } else if bx.cx().is_backend_scalar_pair(layout) { - let ibty0 = bx.cx().scalar_pair_element_backend_type(layout, 0, true); - let ibty1 = bx.cx().scalar_pair_element_backend_type(layout, 1, true); - OperandValue::Pair(bx.const_poison(ibty0), bx.const_poison(ibty1)) - } else { - let ptr = bx.cx().type_ptr(); - OperandValue::Ref(PlaceValue::new_sized(bx.const_poison(ptr), layout.align.abi)) + match layout.backend_repr { + _ if layout.is_zst() => OperandValue::ZeroSized, + BackendRepr::Scalar(_) + | BackendRepr::SimdVector { .. } + | BackendRepr::SimdScalableVector { .. } => { + let ibty = bx.cx().immediate_backend_type(layout); + OperandValue::Immediate(bx.const_poison(ibty)) + } + BackendRepr::ScalarPair { .. } => { + let ibty0 = bx.cx().scalar_pair_element_backend_type(layout, 0, true); + let ibty1 = bx.cx().scalar_pair_element_backend_type(layout, 1, true); + OperandValue::Pair(bx.const_poison(ibty0), bx.const_poison(ibty1)) + } + BackendRepr::Memory { .. } => { + let ptr = bx.cx().type_ptr(); + OperandValue::Ref(PlaceValue::new_sized(bx.const_poison(ptr), layout.align.abi)) + } } } diff --git a/compiler/rustc_codegen_ssa/src/mir/retag.rs b/compiler/rustc_codegen_ssa/src/mir/retag.rs index 3f839d7945afd..680f0b44d70dd 100644 --- a/compiler/rustc_codegen_ssa/src/mir/retag.rs +++ b/compiler/rustc_codegen_ssa/src/mir/retag.rs @@ -14,9 +14,7 @@ use rustc_middle::ty::layout::TyAndLayout; use crate::mir::FunctionCx; use crate::mir::operand::{OperandRef, OperandRefBuilder, OperandValue}; use crate::mir::place::PlaceRef; -use crate::traits::{ - BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, LayoutTypeCodegenMethods, -}; +use crate::traits::{BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods}; use crate::{RetagFlags, RetagInfo}; pub(crate) fn rvalue_needs_retag(rvalue: &Rvalue<'_>) -> bool { @@ -255,8 +253,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let field_layout = curr_operand.layout.field(bx, ix.index()); // Part of https://github.com/rust-lang/compiler-team/issues/838 - if !bx.is_backend_ref(curr_operand.layout) && bx.is_backend_ref(field_layout) { + if curr_operand.layout.is_ssa_standalone() && !field_layout.is_ssa_standalone() + { // FIXME: support vector types, requires insert_element as part of cg-ssa + // FIXME: Nothing should be looking at the *array* inside a `repr(simd)` type, + // as that array doesn't really exist. Perhaps this should be a `bug!`, + // with simd types handled before getting here? } else { let field_operand = curr_operand.extract_field(self, bx, ix.as_usize()); self.retag_operand(bx, &plan, field_operand, builder, field_offset); diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 077d724997441..6786ee9a18e4f 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -1,3 +1,5 @@ +use std::assert_matches; + use itertools::Itertools as _; use rustc_abi::{self as abi, BackendRepr, FIRST_VARIANT}; use rustc_index::IndexVec; @@ -137,7 +139,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ) => { // The destination necessarily contains a wide pointer, so if // it's a scalar pair, it's a wide pointer or newtype thereof. - if bx.cx().is_backend_scalar_pair(dest.layout) { + if let BackendRepr::ScalarPair { .. } = dest.layout.backend_repr { // Into-coerce of a thin pointer to a wide pointer -- just // use the operand path. let temp = self.codegen_rvalue_operand(bx, rvalue); @@ -499,7 +501,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let val = match *kind { mir::CastKind::PointerExposeProvenance => { - assert!(bx.cx().is_backend_immediate(cast)); + assert!(cast.backend_repr.is_scalar_or_simd()); let llptr = operand.immediate(); let llcast_ty = bx.cx().immediate_backend_type(cast); let lladdr = bx.ptrtoint(llptr, llcast_ty); @@ -549,7 +551,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { operand.val } mir::CastKind::PointerCoercion(PointerCoercion::Unsize, _) => { - assert!(bx.cx().is_backend_scalar_pair(cast)); + assert_matches!(cast.backend_repr, BackendRepr::ScalarPair { .. }); let (lldata, llextra) = operand.val.pointer_parts(); let (lldata, llextra) = base::unsize_ptr(bx, lldata, operand.layout.ty, cast.ty, llextra); @@ -561,9 +563,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ) => { bug!("{kind:?} is for borrowck, and should never appear in codegen"); } - mir::CastKind::PtrToPtr if bx.cx().is_backend_scalar_pair(operand.layout) => { + mir::CastKind::PtrToPtr if let BackendRepr::ScalarPair { .. } = operand.layout.backend_repr => { if let OperandValue::Pair(data_ptr, meta) = operand.val { - if bx.cx().is_backend_scalar_pair(cast) { + if let BackendRepr::ScalarPair { .. } = cast.layout.backend_repr { OperandValue::Pair(data_ptr, meta) } else { // Cast of wide-ptr to thin-ptr is an extraction of data-ptr. @@ -590,7 +592,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let from_backend_ty = bx.cx().immediate_backend_type(operand.layout); - assert!(bx.cx().is_backend_immediate(cast)); + assert!(cast.backend_repr.is_scalar_or_simd()); let to_backend_ty = bx.cx().immediate_backend_type(cast); if operand.layout.is_uninhabited() { let val = OperandValue::Immediate(bx.cx().const_poison(to_backend_ty)); @@ -732,7 +734,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } }; assert!( - val.is_expected_variant_for_type(self.cx, layout), + val.is_expected_variant_for_type(layout), "Made wrong variant {val:?} for type {layout:?}", ); OperandRef { val, layout, move_annotation: None } diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 0d27ff90991b3..caf63abeef3ad 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -513,7 +513,9 @@ pub trait BuilderMethods<'a, 'tcx>: let ty = self.backend_type(layout); let val = self.load_from_place(ty, src); self.store_to_place_with_flags(val, dst, flags); - } else if self.sess().opts.optimize == OptLevel::No && self.is_backend_immediate(layout) { + } else if self.sess().opts.optimize == OptLevel::No + && layout.backend_repr.is_scalar_or_simd() + { // If we're not optimizing, the aliasing information from `memcpy` // isn't useful, so just load-store the value for smaller code. let temp = self.load_operand(src.with_type(layout)); diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 45ea8384b2d46..707eb3a6ee85d 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -127,28 +127,12 @@ pub trait LayoutTypeCodegenMethods<'tcx>: BackendTypes { /// [`from_immediate`](super::BuilderMethods::from_immediate) and /// [`to_immediate_scalar`](super::BuilderMethods::to_immediate_scalar). fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; - fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool; - fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool; fn scalar_pair_element_backend_type( &self, layout: TyAndLayout<'tcx>, index: usize, immediate: bool, ) -> Self::Type; - - /// A type that produces an [`OperandValue::Ref`] when loaded. - /// - /// AKA one that's not a ZST, not `is_backend_immediate`, and - /// not `is_backend_scalar_pair`. For such a type, a - /// [`load_operand`] doesn't actually `load` anything. - /// - /// [`OperandValue::Ref`]: crate::mir::operand::OperandValue::Ref - /// [`load_operand`]: super::BuilderMethods::load_operand - fn is_backend_ref(&self, layout: TyAndLayout<'tcx>) -> bool { - !(layout.is_zst() - || self.is_backend_immediate(layout) - || self.is_backend_scalar_pair(layout)) - } } // For backends that support CFI using type membership (i.e., testing whether a given pointer is diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index c7ea358f08170..1a7dc06d97c90 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use rustc_ast::{LitKind, MetaItemKind, token}; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_data_structures::jobserver::{self, Proxy}; +use rustc_data_structures::jobserver; use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed}; use rustc_lint::LintStore; use rustc_middle::ty; @@ -41,9 +41,6 @@ pub struct Compiler { /// A reference to the current `GlobalCtxt` which we pass on to `GlobalCtxt`. pub(crate) current_gcx: CurrentGcx, - - /// A jobserver reference which we pass on to `GlobalCtxt`. - pub(crate) jobserver_proxy: Arc, } /// Converts strings provided as `--cfg [cfgspec]` into a `Cfg`. @@ -407,7 +404,7 @@ pub fn run_compiler(config: Config, f: impl FnOnce(&Compiler) -> R + Se config.opts.unstable_opts.threads.unwrap_or(1), &config.extra_symbols, SourceMapInputs { file_loader, path_mapping, hash_kind, checksum_hash_kind }, - |current_gcx, jobserver_proxy| { + |current_gcx| { // The previous `early_dcx` can't be reused here because it doesn't // impl `Send`. Creating a new one is fine. let early_dcx = EarlyDiagCtxt::new(config.opts.error_format); @@ -480,7 +477,6 @@ pub fn run_compiler(config: Config, f: impl FnOnce(&Compiler) -> R + Se codegen_backend, override_queries: config.override_queries, current_gcx, - jobserver_proxy, }; // There are two paths out of `f`. diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index a40c477b3a441..04c89a5dd8c15 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -1012,7 +1012,6 @@ pub fn create_and_enter_global_ctxt FnOnce(TyCtxt<'tcx>) -> T>( ), providers.hooks, compiler.current_gcx.clone(), - Arc::clone(&compiler.jobserver_proxy), |tcx| { let feed = tcx.create_crate_num(stable_crate_id).unwrap(); assert_eq!(feed.key(), LOCAL_CRATE); diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index c78e419033d78..39c5ee8193256 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -133,7 +133,7 @@ fn init_stack_size(early_dcx: &EarlyDiagCtxt) -> usize { }) } -fn run_in_thread_with_globals) -> R + Send, R: Send>( +fn run_in_thread_with_globals R + Send, R: Send>( thread_stack_size: usize, edition: Edition, sm_inputs: SourceMapInputs, @@ -159,7 +159,7 @@ fn run_in_thread_with_globals) -> R + Send, R: edition, extra_symbols, Some(sm_inputs), - || f(CurrentGcx::new(), Proxy::new()), + || f(CurrentGcx::new()), ) }) .unwrap() @@ -172,10 +172,7 @@ fn run_in_thread_with_globals) -> R + Send, R: }) } -pub(crate) fn run_in_thread_pool_with_globals< - F: FnOnce(CurrentGcx, Arc) -> R + Send, - R: Send, ->( +pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( thread_builder_diag: &EarlyDiagCtxt, edition: Edition, threads: usize, @@ -199,11 +196,11 @@ pub(crate) fn run_in_thread_pool_with_globals< edition, sm_inputs, extra_symbols, - |current_gcx, jobserver_proxy| { + |current_gcx| { // Register the thread for use with the `WorkerLocal` type. registry.register(); - f(current_gcx, jobserver_proxy) + f(current_gcx) }, ); }; @@ -212,13 +209,12 @@ pub(crate) fn run_in_thread_pool_with_globals< let current_gcx2 = current_gcx.clone(); let proxy = Proxy::new(); - let proxy_ = Arc::clone(&proxy); - let proxy__ = Arc::clone(&proxy); + let builder = rustc_thread_pool::ThreadPoolBuilder::new() .thread_name(|_| "rustc".to_string()) - .acquire_thread_handler(move || proxy_.acquire_thread()) - .release_thread_handler(move || proxy__.release_thread()) + .acquire_thread_handler(move || proxy.acquire_thread()) + .release_thread_handler(move || proxy_.release_thread()) .num_threads(threads) .deadlock_handler(move || { // On deadlock, creates a new thread and forwards information in thread @@ -294,7 +290,7 @@ internal compiler error: query cycle handler thread panicked, aborting process"; }, // Run `f` on the first thread in the thread pool. move |pool: &rustc_thread_pool::ThreadPool| { - pool.install(|| f(current_gcx.into_inner(), proxy)) + pool.install(|| f(current_gcx.into_inner())) }, ) .unwrap_or_else(|err| { diff --git a/compiler/rustc_middle/src/query/job.rs b/compiler/rustc_middle/src/query/job.rs index 8c78bf24287e0..7f48b2dbdbb22 100644 --- a/compiler/rustc_middle/src/query/job.rs +++ b/compiler/rustc_middle/src/query/job.rs @@ -7,7 +7,6 @@ use parking_lot::{Condvar, Mutex}; use rustc_span::Span; use crate::query::Cycle; -use crate::ty::TyCtxt; /// A value uniquely identifying an active query job. #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] @@ -71,12 +70,7 @@ impl<'tcx> QueryLatch<'tcx> { } /// Awaits for the query job to complete. - pub fn wait_on( - &self, - tcx: TyCtxt<'tcx>, - query: Option, - span: Span, - ) -> Result<(), Cycle<'tcx>> { + pub fn wait_on(&self, query: Option, span: Span) -> Result<(), Cycle<'tcx>> { let mut waiters_guard = self.waiters.lock(); let Some(waiters) = &mut *waiters_guard else { return Ok(()); // already complete @@ -99,12 +93,11 @@ impl<'tcx> QueryLatch<'tcx> { // If this detects a deadlock and the deadlock handler wants to resume this thread // we have to be in the `wait` call. This is ensured by the deadlock handler // getting the self.info lock. - rustc_thread_pool::mark_blocked(); - tcx.jobserver_proxy.release_thread(); - waiter.condvar.wait(&mut waiters_guard); - // Release the lock before we potentially block in `acquire_thread` - drop(waiters_guard); - tcx.jobserver_proxy.acquire_thread(); + rustc_thread_pool::mark_blocked_and_wait(|| { + waiter.condvar.wait(&mut waiters_guard); + // Release the lock before we potentially block when acquiring jobserver token. + drop(waiters_guard); + }); // FIXME: Get rid of this lock. We have ownership of the QueryWaiter // although another thread may still have a Arc reference so we cannot diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 146662676acc0..633466102817b 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -20,7 +20,6 @@ use rustc_ast as ast; use rustc_data_structures::defer; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::intern::Interned; -use rustc_data_structures::jobserver::Proxy; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap}; use rustc_data_structures::stable_hash::StableHash; @@ -767,9 +766,6 @@ pub struct GlobalCtxt<'tcx> { pub(crate) alloc_map: interpret::AllocMap<'tcx>, current_gcx: CurrentGcx, - - /// A jobserver reference used to release then acquire a token while waiting on a query. - pub jobserver_proxy: Arc, } impl<'tcx> GlobalCtxt<'tcx> { @@ -944,7 +940,6 @@ impl<'tcx> TyCtxt<'tcx> { query_system: QuerySystem<'tcx>, hooks: crate::hooks::Providers, current_gcx: CurrentGcx, - jobserver_proxy: Arc, f: impl FnOnce(TyCtxt<'tcx>) -> T, ) -> T { let data_layout = sess.target.parse_data_layout().unwrap_or_else(|err| { @@ -982,7 +977,6 @@ impl<'tcx> TyCtxt<'tcx> { data_layout, alloc_map: interpret::AllocMap::new(), current_gcx, - jobserver_proxy, }); // This is a separate function to work around a crash with parallel rustc (#135870) diff --git a/compiler/rustc_query_impl/src/execution.rs b/compiler/rustc_query_impl/src/execution.rs index 5b59f005a4908..22cdc30a4ae86 100644 --- a/compiler/rustc_query_impl/src/execution.rs +++ b/compiler/rustc_query_impl/src/execution.rs @@ -252,7 +252,7 @@ fn wait_for_query<'tcx, C: QueryCache>( let query_blocked_prof_timer = tcx.prof.query_blocked(); // With parallel queries we might just have to wait on some other thread. - let result = latch.wait_on(tcx, current, span); + let result = latch.wait_on(current, span); match result { Ok(()) => { diff --git a/compiler/rustc_thread_pool/src/lib.rs b/compiler/rustc_thread_pool/src/lib.rs index 7ce7fbc27eabe..56d3bc1184454 100644 --- a/compiler/rustc_thread_pool/src/lib.rs +++ b/compiler/rustc_thread_pool/src/lib.rs @@ -95,7 +95,7 @@ pub use worker_local::WorkerLocal; pub use self::broadcast::{BroadcastContext, broadcast, spawn_broadcast}; pub use self::join::{join, join_context}; use self::registry::{CustomSpawn, DefaultSpawn, ThreadSpawn}; -pub use self::registry::{Registry, ThreadBuilder, mark_blocked, mark_unblocked}; +pub use self::registry::{Registry, ThreadBuilder, mark_blocked_and_wait, mark_unblocked}; pub use self::scope::{Scope, ScopeFifo, in_place_scope, in_place_scope_fifo, scope, scope_fifo}; pub use self::spawn::{spawn, spawn_fifo}; pub use self::thread_pool::{ diff --git a/compiler/rustc_thread_pool/src/registry.rs b/compiler/rustc_thread_pool/src/registry.rs index 9510c1842f86a..71b2e11f7a8cd 100644 --- a/compiler/rustc_thread_pool/src/registry.rs +++ b/compiler/rustc_thread_pool/src/registry.rs @@ -615,14 +615,17 @@ impl Registry { } /// Mark a Rayon worker thread as blocked. This triggers the deadlock handler -/// if no other worker thread is active +/// if no other worker thread is active. Then wait for the user-specified condition. #[inline] -pub fn mark_blocked() { +pub fn mark_blocked_and_wait(wait: impl FnOnce()) { let worker_thread = WorkerThread::current(); assert!(!worker_thread.is_null()); unsafe { let registry = &(*worker_thread).registry; - registry.sleep.mark_blocked(®istry.deadlock_handler) + registry.sleep.mark_blocked(®istry.deadlock_handler); + registry.release_thread(); + wait(); + registry.acquire_thread(); } } diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index c5da5055e16d9..db825a1c970db 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -1026,9 +1026,12 @@ pub unsafe fn unaligned_volatile_store(dst: *mut T, val: T); /// /// The stabilized version of this intrinsic is /// [`f16::sqrt`](../../std/primitive.f16.html#method.sqrt) +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub fn sqrtf16(x: f16) -> f16; +pub fn sqrtf16(x: f16) -> f16 { + sqrtf32(x as f32) as f16 +} /// Returns the square root of an `f32` /// /// The stabilized version of this intrinsic is @@ -1055,9 +1058,12 @@ pub fn sqrtf128(x: f128) -> f128; /// /// The stabilized version of this intrinsic is /// [`f16::powi`](../../std/primitive.f16.html#method.powi) +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub fn powif16(a: f16, x: i32) -> f16; +pub fn powif16(a: f16, x: i32) -> f16 { + powif32(a as f32, x) as f16 +} /// Raises an `f32` to an integer power. /// /// The stabilized version of this intrinsic is @@ -1437,9 +1443,14 @@ pub fn log2f128(x: f128) -> f128 { /// The stabilized version of this intrinsic is /// [`f16::mul_add`](../../std/primitive.f16.html#method.mul_add) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn fmaf16(a: f16, b: f16, c: f16) -> f16; +pub const fn fmaf16(a: f16, b: f16, c: f16) -> f16 { + // NOTE: f32 does not have sufficient precision, so use f64 instead. + // see also https://github.com/llvm/llvm-project/issues/128450#issuecomment-2727540179. + fmaf64(a as f64, b as f64, c as f64) as f16 +} /// Returns `a * b + c` without rounding the intermediate result for `f32` values. /// /// The stabilized version of this intrinsic is @@ -1535,9 +1546,12 @@ pub const fn fmuladdf128(a: f128, b: f128, c: f128) -> f128 { /// The stabilized version of this intrinsic is /// [`f16::floor`](../../std/primitive.f16.html#method.floor) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn floorf16(x: f16) -> f16; +pub const fn floorf16(x: f16) -> f16 { + floorf32(x as f32) as f16 +} /// Returns the largest integer less than or equal to an `f32`. /// /// The stabilized version of this intrinsic is @@ -1568,9 +1582,12 @@ pub const fn floorf128(x: f128) -> f128; /// The stabilized version of this intrinsic is /// [`f16::ceil`](../../std/primitive.f16.html#method.ceil) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn ceilf16(x: f16) -> f16; +pub const fn ceilf16(x: f16) -> f16 { + ceilf32(x as f32) as f16 +} /// Returns the smallest integer greater than or equal to an `f32`. /// /// The stabilized version of this intrinsic is @@ -1601,9 +1618,12 @@ pub const fn ceilf128(x: f128) -> f128; /// The stabilized version of this intrinsic is /// [`f16::trunc`](../../std/primitive.f16.html#method.trunc) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn truncf16(x: f16) -> f16; +pub const fn truncf16(x: f16) -> f16 { + truncf32(x as f32) as f16 +} /// Returns the integer part of an `f32`. /// /// The stabilized version of this intrinsic is @@ -1635,9 +1655,12 @@ pub const fn truncf128(x: f128) -> f128; /// The stabilized version of this intrinsic is /// [`f16::round_ties_even`](../../std/primitive.f16.html#method.round_ties_even) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn round_ties_even_f16(x: f16) -> f16; +pub const fn round_ties_even_f16(x: f16) -> f16 { + round_ties_even_f32(x as f32) as f16 +} /// Returns the nearest integer to an `f32`. Rounds half-way cases to the number with an even /// least significant digit. @@ -1674,9 +1697,12 @@ pub const fn round_ties_even_f128(x: f128) -> f128; /// The stabilized version of this intrinsic is /// [`f16::round`](../../std/primitive.f16.html#method.round) #[rustc_intrinsic_const_stable_indirect] +#[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn roundf16(x: f16) -> f16; +pub const fn roundf16(x: f16) -> f16 { + roundf32(x as f32) as f16 +} /// Returns the nearest integer to an `f32`. Rounds half-way cases away from zero. /// /// The stabilized version of this intrinsic is @@ -3610,34 +3636,46 @@ pub const fn fabs(x: T) -> T; /// /// The stabilized version of this intrinsic is /// [`f16::copysign`](../../std/primitive.f16.html#method.copysign) +#[inline] #[rustc_nounwind] #[rustc_intrinsic] -pub const fn copysignf16(x: f16, y: f16) -> f16; +pub const fn copysignf16(x: f16, y: f16) -> f16 { + f16::from_bits((x.to_bits() & !f16::SIGN_MASK) | (y.to_bits() & f16::SIGN_MASK)) +} /// Copies the sign from `y` to `x` for `f32` values. /// /// The stabilized version of this intrinsic is /// [`f32::copysign`](../../std/primitive.f32.html#method.copysign) +#[inline] #[rustc_nounwind] #[rustc_intrinsic_const_stable_indirect] #[rustc_intrinsic] -pub const fn copysignf32(x: f32, y: f32) -> f32; +pub const fn copysignf32(x: f32, y: f32) -> f32 { + f32::from_bits((x.to_bits() & !f32::SIGN_MASK) | (y.to_bits() & f32::SIGN_MASK)) +} /// Copies the sign from `y` to `x` for `f64` values. /// /// The stabilized version of this intrinsic is /// [`f64::copysign`](../../std/primitive.f64.html#method.copysign) +#[inline] #[rustc_nounwind] #[rustc_intrinsic_const_stable_indirect] #[rustc_intrinsic] -pub const fn copysignf64(x: f64, y: f64) -> f64; +pub const fn copysignf64(x: f64, y: f64) -> f64 { + f64::from_bits((x.to_bits() & !f64::SIGN_MASK) | (y.to_bits() & f64::SIGN_MASK)) +} /// Copies the sign from `y` to `x` for `f128` values. /// /// The stabilized version of this intrinsic is /// [`f128::copysign`](../../std/primitive.f128.html#method.copysign) +#[inline] #[rustc_nounwind] #[rustc_intrinsic] -pub const fn copysignf128(x: f128, y: f128) -> f128; +pub const fn copysignf128(x: f128, y: f128) -> f128 { + f128::from_bits((x.to_bits() & !f128::SIGN_MASK) | (y.to_bits() & f128::SIGN_MASK)) +} /// Generates the LLVM body for the automatic differentiation of `f` using Enzyme, /// with `df` as the derivative function and `args` as its arguments. diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 6813c3bfe5859..c758d5f4b89d6 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -122,6 +122,11 @@ //! Allocations must behave like "normal" memory: in particular, reads must not have //! side-effects, and writes must become visible to other threads using the usual synchronization //! primitives. +//! Allocations must support all atomic operations that are available for the target (as +//! determined by the `target_has_atomic*` set of cfg flags). +//! The precise instructions used for atomic operations are generally not guaranteed, so portable +//! software should place all Rust allocations in memory regions that support all atomic +//! instructions. //! //! For any allocation with `base` address, `size`, and a set of //! `addresses`, the following are guaranteed: diff --git a/src/doc/rustc/src/platform-support/android.md b/src/doc/rustc/src/platform-support/android.md index dcf7659130a38..570379e157248 100644 --- a/src/doc/rustc/src/platform-support/android.md +++ b/src/doc/rustc/src/platform-support/android.md @@ -24,9 +24,12 @@ Android targets support std. Generated binaries use the ELF file format. ## NDK/API Update Policy -Rust will support the most recent Long Term Support (LTS) Android Native -Development Kit (NDK). By default Rust will support all API levels supported -by the NDK, but a higher minimum API level may be required if deemed necessary. +Rust will support the most recent Long Term Support (LTS) [Android Native +Development Kit (NDK)][NDK]. By default Rust will support all [API levels supported +by the NDK][API-levels], but a higher minimum API level may be required if deemed necessary. + +[NDK]: https://github.com/android/ndk/wiki +[API-levels]: https://github.com/android/ndk/wiki/Compatibility ## Building the target