Rollup of 7 pull requests

bootstrap.example.toml

@@ -973,6 +973,24 @@
 #dist.vendor = if "is a tarball source" || "is a git repository" { true } else { false }
 
 
+# =============================================================================
+# Profile-guided optimization options
+#
+# Configure the PGO profile to be used for compilation, or a path where the
+# profile will be written by an instrumented binary, for individual components
+# =============================================================================
+
+# Use the following profile to PGO optimize the Rust compiler.
+#pgo.rustc.use = "/tmp/profiles/foo.profraw"
+# Instrument the Rust compiler, so that when executed, it will gather profiles
+# to this path.
+#pgo.rustc.generate = "/tmp/profiles/foo.profraw"
+# Use the following profile to PGO optimize LLVM.
+#pgo.llvm.use = "/tmp/profiles/foo.profraw"
+# Instrument LLVM, so that when executed, it will gather profiles
+# Note: for LLVM specifically, this should be a directory, not a file path.
+#pgo.llvm.generate = "/tmp/profiles"
+
 # =============================================================================
 # Options for specific targets
 #

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1155,7 +1155,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         // NOTE: libgccjit does not support specifying the alignment on the assignment, so we cast
         // to type so it gets the proper alignment.
         let destination_type = destination.to_rvalue().get_type().unqualified();
-        let align = if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() };
+        let align = align.bytes();
         let mut modified_destination_type = destination_type.get_aligned(align);
         if flags.contains(MemFlags::VOLATILE) {
             modified_destination_type = modified_destination_type.make_volatile();

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -377,16 +377,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
                     load
                 }
             }
-            sym::volatile_store => {
-                let dst = args[0].deref(self.cx());
-                args[1].val.volatile_store(self, dst);
-                return IntrinsicResult::WroteIntoPlace;
-            }
-            sym::unaligned_volatile_store => {
-                let dst = args[0].deref(self.cx());
-                args[1].val.unaligned_volatile_store(self, dst);
-                return IntrinsicResult::WroteIntoPlace;
-            }
             sym::prefetch_read_data
             | sym::prefetch_write_data
             | sym::prefetch_read_instruction

compiler/rustc_codegen_llvm/src/builder.rs

@@ -871,8 +871,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unsafe {
             let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
             let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
-            let align =
-                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
+            let align = align.bytes() as c_uint;
             llvm::LLVMSetAlignment(store, align);
             if flags.contains(MemFlags::VOLATILE) {
                 llvm::LLVMSetVolatile(store, llvm::TRUE);

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -3,8 +3,8 @@ use std::ffi::c_uint;
 use std::{assert_matches, iter, ptr};
 
 use rustc_abi::{
-    AddressSpace, Align, BackendRepr, CVariadicStatus, Float, HasDataLayout, Integer,
-    NumScalableVectors, Primitive, Size, WrappingRange,
+    AddressSpace, Align, BackendRepr, CVariadicStatus, Float, HasDataLayout, NumScalableVectors,
+    Primitive, Size, WrappingRange,
 };
 use rustc_codegen_ssa::RetagInfo;
 use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
@@ -315,11 +315,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     Primitive::Pointer(_) => {
                         // Pointers are always OK.
                     }
-                    Primitive::Int(Integer::I128, _) => {
-                        // FIXME: maybe we should support these? At least on 32-bit powerpc
-                        // the logic in LLVM does not handle i128 correctly though.
-                        bug!("the va_arg intrinsic does not support `i128`/`u128`")
-                    }
                     Primitive::Int(..) => {
                         let int_width = self.cx().size_of(result_layout.ty).bits();
                         let target_c_int_width = self.cx().sess().target.options.c_int_width;
@@ -389,16 +384,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     };
                 }
             }
-            sym::volatile_store => {
-                let dst = args[0].deref(self.cx());
-                args[1].val.volatile_store(self, dst);
-                return IntrinsicResult::Operand(OperandValue::ZeroSized);
-            }
-            sym::unaligned_volatile_store => {
-                let dst = args[0].deref(self.cx());
-                args[1].val.unaligned_volatile_store(self, dst);
-                return IntrinsicResult::Operand(OperandValue::ZeroSized);
-            }
             sym::prefetch_read_data
             | sym::prefetch_write_data
             | sym::prefetch_read_instruction

compiler/rustc_codegen_llvm/src/va_arg.rs

@@ -1,5 +1,6 @@
-use rustc_abi::{Align, BackendRepr, CVariadicStatus, Endian, HasDataLayout, Primitive, Size};
-use rustc_codegen_ssa::MemFlags;
+use rustc_abi::{
+    Align, BackendRepr, CVariadicStatus, Endian, Float, HasDataLayout, Integer, Primitive, Size,
+};
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::OperandRef;
 use rustc_codegen_ssa::traits::{
@@ -77,6 +78,35 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     }
 }
 
+/// Some backends apply special alignment rules to c-variadic arguments.
+fn get_param_type_alignment<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    layout: TyAndLayout<'tcx>,
+) -> Align {
+    let BackendRepr::Scalar(scalar) = layout.backend_repr else {
+        bug!("unexpected backend repr {:?}", layout.backend_repr);
+    };
+
+    match bx.cx.tcx.sess.target.arch {
+        Arch::PowerPC64 => match scalar.primitive() {
+            Primitive::Int(integer, _) => match integer {
+                Integer::I8 | Integer::I16 => unreachable!(),
+                Integer::I32 | Integer::I64 => { /* fall through */ }
+                Integer::I128 => return Align::EIGHT,
+            },
+            Primitive::Float(float) => match float {
+                Float::F16 | Float::F32 => unreachable!(),
+                Float::F64 => { /* fall through */ }
+                Float::F128 => return Align::from_bytes(16).unwrap(),
+            },
+            Primitive::Pointer(_) => { /* fall through */ }
+        },
+        _ => { /* fall through */ }
+    }
+
+    layout.align.abi
+}
+
 enum PassMode {
     Direct,
     Indirect,
@@ -136,23 +166,23 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
         (
             bx.cx.layout_of(Ty::new_imm_ptr(bx.cx.tcx, target_ty)).llvm_type(bx.cx),
             bx.cx.data_layout().pointer_size(),
-            bx.cx.data_layout().pointer_align(),
+            bx.cx.data_layout().pointer_align().abi,
         )
     } else {
-        (layout.llvm_type(bx.cx), layout.size, layout.align)
+        (layout.llvm_type(bx.cx), layout.size, get_param_type_alignment(bx, layout))
     };
     let (addr, addr_align) = emit_direct_ptr_va_arg(
         bx,
         list,
         size,
-        align.abi,
+        align,
         slot_size,
         allow_higher_align,
         force_right_adjust,
     );
     if indirect {
         let tmp_ret = bx.load(llty, addr, addr_align);
-        bx.load(layout.llvm_type(bx.cx), tmp_ret, align.abi)
+        bx.load(layout.llvm_type(bx.cx), tmp_ret, align)
     } else {
         bx.load(llty, addr, addr_align)
     }
@@ -585,8 +615,10 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
     // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
     // l->fp_offset > 176 - num_fp * 16 go to step 7.
 
+    // We support x86_64-unknown-linux-gnux32 which uses 4-byte pointers.
     let unsigned_int_offset = 4;
-    let ptr_offset = 8;
+    let ptr_offset = bx.tcx().data_layout.pointer_size().bytes();
+
     let gp_offset_ptr = va_list_addr;
     let fp_offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(unsigned_int_offset));
 
@@ -660,7 +692,7 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
                     let reg_hi_addr = bx.inbounds_ptradd(reg_lo_addr, bx.const_i32(16));
 
                     let align = layout.layout.align().abi;
-                    let tmp = bx.alloca(layout.layout.size(), align);
+                    let tmp = bx.alloca(layout.size, layout.align.abi);
 
                     let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
                     let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
@@ -682,7 +714,7 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
                         Primitive::Int(_, _) | Primitive::Pointer(_) => (gp_addr, fp_addr),
                     };
 
-                    let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
+                    let tmp = bx.alloca(layout.size, layout.align.abi);
 
                     let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
                     let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
@@ -749,16 +781,12 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>(
     src_align: Align,
 ) -> &'ll Value {
     if layout.layout.align.abi > src_align {
-        let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
-        bx.memcpy(
-            tmp,
-            layout.layout.align.abi,
-            reg_addr,
-            src_align,
-            bx.const_u32(layout.layout.size().bytes() as u32),
-            MemFlags::empty(),
-            None,
-        );
+        assert!(layout.ty.is_integral());
+
+        // A memcpy below optimizes poorly for 128-bit integers.
+        let tmp = bx.alloca(layout.size, layout.align.abi);
+        let val = bx.load(layout.llvm_type(bx), reg_addr, src_align);
+        bx.store(val, tmp, layout.align.abi);
         tmp
     } else {
         reg_addr
@@ -780,9 +808,14 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
     // byte boundary if alignment needed by type exceeds 8 byte boundary.
     // It isn't stated explicitly in the standard, but in practice we use
     // alignment greater than 16 where necessary.
-    if layout.layout.align.bytes() > 8 {
-        unreachable!("all instances of VaArgSafe have an alignment <= 8");
-    }
+    // The AMD64 psABI leaves unspecified what to do for alignments above 16, but
+    // this behavior for 32+ alignment matches clang.
+    // It currently (2026 July) can only occur for 16-byte-aligned types.
+    let overflow_arg_area_v = if layout.layout.align.bytes() > 8 {
+        round_pointer_up_to_alignment(bx, overflow_arg_area_v, layout.layout.align.abi)
+    } else {
+        overflow_arg_area_v
+    };
 
     // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
     let mem_addr = overflow_arg_area_v;
@@ -1052,9 +1085,15 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             bx,
             addr,
             target_ty,
-            PassMode::Direct,
+            // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
+            // not 1, 2, 4, or 8 bytes, must be passed by reference."
+            if target_ty_size > 8 || !target_ty_size.is_power_of_two() {
+                PassMode::Indirect
+            } else {
+                PassMode::Direct
+            },
             SlotSize::Bytes8,
-            if target.is_like_windows { AllowHigherAlign::No } else { AllowHigherAlign::Yes },
+            AllowHigherAlign::No,
             ForceRightAdjust::No,
         ),
         Arch::AArch64 if target.is_like_windows || target.is_like_darwin => emit_ptr_va_arg(
@@ -1063,13 +1102,14 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             target_ty,
             PassMode::Direct,
             SlotSize::Bytes8,
-            if target.is_like_windows { AllowHigherAlign::No } else { AllowHigherAlign::Yes },
+            AllowHigherAlign::Yes,
             ForceRightAdjust::No,
         ),
         Arch::AArch64 => emit_aapcs_va_arg(bx, addr, target_ty),
         Arch::Arm => {
             // Types wider than 16 bytes are not currently supported. Clang has special logic for
-            // such types, but `VaArgSafe` is not implemented for any type that is this large.
+            // such types, but `VaArgSafe` is not implemented for any type that is this large on
+            // arm (i.e. 32-bit) targets.
             assert!(bx.cx.size_of(target_ty).bytes() <= 16);
 
             emit_ptr_va_arg(
@@ -1091,6 +1131,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             PassMode::Direct,
             SlotSize::Bytes8,
             AllowHigherAlign::Yes,
+            // ForceRightAdjust only takes effect on big-endian architectures.
             ForceRightAdjust::Yes,
         ),
         Arch::RiscV32 if target.llvm_abiname == LlvmAbi::Ilp32e => {

compiler/rustc_codegen_ssa/src/lib.rs

@@ -163,11 +163,12 @@ pub enum ModuleKind {
 }
 
 bitflags::bitflags! {
+    /// This previously had an `UNALIGNED` variant, but that should never be done via flags.
+    /// If you want something to be unaligned, see [`mir::place::PlaceRef::unaligned`].
     #[derive(Debug, Clone, Copy, PartialEq, Eq)]
     pub struct MemFlags: u8 {
         const VOLATILE = 1 << 0;
         const NONTEMPORAL = 1 << 1;
-        const UNALIGNED = 1 << 2;
         /// Indicates that writing through the stored pointer is undefined behavior.
         /// Only valid on stores of pointers, or pairs where the first element is a pointer.
         /// In the latter case, the flag only applies to the first element of the pair.

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -273,16 +273,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 );
                 OperandValue::ZeroSized
             }
-            sym::volatile_store => {
+            sym::volatile_store | sym::unaligned_volatile_store => {
                 let dst = args[0].deref(bx.cx());
+                let dst = if name == sym::volatile_store { dst } else { dst.unaligned() };
                 args[1].val.volatile_store(bx, dst);
                 OperandValue::ZeroSized
             }
-            sym::unaligned_volatile_store => {
-                let dst = args[0].deref(bx.cx());
-                args[1].val.unaligned_volatile_store(bx, dst);
-                OperandValue::ZeroSized
-            }
             sym::disjoint_bitor => {
                 let a = args[0].immediate();
                 let b = args[1].immediate();

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -974,14 +974,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
         self.store_with_flags(bx, dest, MemFlags::VOLATILE);
     }
 
-    pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
-        self,
-        bx: &mut Bx,
-        dest: PlaceRef<'tcx, V>,
-    ) {
-        self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
-    }
-
     pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
         self,
         bx: &mut Bx,

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -318,6 +318,13 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
     pub fn storage_dead<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) {
         bx.lifetime_end(self.val.llval, self.layout.size);
     }
+
+    /// The same place, but with [`PlaceValue::align`] lowered to [`Align::ONE`].
+    pub fn unaligned(self) -> Self {
+        let Self { val, layout } = self;
+        let val = PlaceValue { align: Align::ONE, ..val };
+        Self { val, layout }
+    }
 }
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {

compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs

@@ -482,16 +482,18 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     ) -> Result<(), ErrorGuaranteed> {
         let tcx = self.tcx();
         let parent_def_id = self.item_def_id();
+        // In this path, `Some(context)` should be `ConstArgument`: enum
+        // discriminants are handled earlier by resolve. We still use the helper so
+        // nested inline consts are checked in the outer const-argument context.
         if let Res::Def(DefKind::ConstParam, _) = res
-            && matches!(tcx.def_kind(parent_def_id), DefKind::AnonConst | DefKind::InlineConst)
-            && let ty::AnonConstKind::MCG = tcx.anon_const_kind(parent_def_id)
+            && let Some(context) = self.anon_const_forbids_generic_params()
         {
             let folder = ForbidParamUsesFolder {
                 tcx,
                 anon_const_def_id: parent_def_id,
                 span,
                 is_self_alias: false,
-                context: ForbidParamContext::ConstArgument,
+                context,
             };
             return Err(folder.error());
         }