From 14dbd5d4df5c456634e429f8da4656bb7f902144 Mon Sep 17 00:00:00 2001
From: bit-aloo <sshourya17@gmail.com>
Date: Wed, 1 Jul 2026 05:02:21 +0000
Subject: [PATCH 01/16] reject const param in nested const argument

---
 compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs |  8 +++++---
 .../mgca/const-param-in-nested-inline-const-arg.rs     |  9 +++++++++
 .../mgca/const-param-in-nested-inline-const-arg.stderr | 10 ++++++++++
 3 files changed, 24 insertions(+), 3 deletions(-)
 create mode 100644 tests/ui/const-generics/mgca/const-param-in-nested-inline-const-arg.rs
 create mode 100644 tests/ui/const-generics/mgca/const-param-in-nested-inline-const-arg.stderr

diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
index b29e469348f81..823cd07458a7e 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
@@ -482,16 +482,18 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     ) -> Result<(), ErrorGuaranteed> {
         let tcx = self.tcx();
         let parent_def_id = self.item_def_id();
+        // In this path, `Some(context)` should be `ConstArgument`: enum
+        // discriminants are handled earlier by resolve. We still use the helper so
+        // nested inline consts are checked in the outer const-argument context.
         if let Res::Def(DefKind::ConstParam, _) = res
-            && matches!(tcx.def_kind(parent_def_id), DefKind::AnonConst | DefKind::InlineConst)
-            && let ty::AnonConstKind::MCG = tcx.anon_const_kind(parent_def_id)
+            && let Some(context) = self.anon_const_forbids_generic_params()
         {
             let folder = ForbidParamUsesFolder {
                 tcx,
                 anon_const_def_id: parent_def_id,
                 span,
                 is_self_alias: false,
-                context: ForbidParamContext::ConstArgument,
+                context,
             };
             return Err(folder.error());
         }
diff --git a/tests/ui/const-generics/mgca/const-param-in-nested-inline-const-arg.rs b/tests/ui/const-generics/mgca/const-param-in-nested-inline-const-arg.rs
new file mode 100644
index 0000000000000..e8af7e00b470a
--- /dev/null
+++ b/tests/ui/const-generics/mgca/const-param-in-nested-inline-const-arg.rs
@@ -0,0 +1,9 @@
+//@compile-flags: -Znext-solver=globally
+#![feature(generic_const_args, min_generic_const_args)]
+
+struct S<const N: usize>;
+fn foo<const N: usize>(_: S<{ const { const { N } } }>) {}
+//~^ ERROR: generic parameters in const blocks are not allowed; use a named `const` item instead
+
+
+fn main() {}
diff --git a/tests/ui/const-generics/mgca/const-param-in-nested-inline-const-arg.stderr b/tests/ui/const-generics/mgca/const-param-in-nested-inline-const-arg.stderr
new file mode 100644
index 0000000000000..fcf13e39e3981
--- /dev/null
+++ b/tests/ui/const-generics/mgca/const-param-in-nested-inline-const-arg.stderr
@@ -0,0 +1,10 @@
+error: generic parameters in const blocks are not allowed; use a named `const` item instead
+  --> $DIR/const-param-in-nested-inline-const-arg.rs:5:47
+   |
+LL | fn foo<const N: usize>(_: S<{ const { const { N } } }>) {}
+   |                                               ^
+   |
+   = help: consider factoring the expression into a `type const` item and use it as the const argument instead
+
+error: aborting due to 1 previous error
+

From 22ab4de92ac855a02a2e8517148dfc69b2f3c49c Mon Sep 17 00:00:00 2001
From: Predrag Gruevski <obi1kenobi82@gmail.com>
Date: Mon, 6 Jul 2026 11:21:25 -0400
Subject: [PATCH 02/16] [perf] Add explicit `Iterator::count` impl for
 `ChunkBy`.

---
 library/core/src/slice/iter.rs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/library/core/src/slice/iter.rs b/library/core/src/slice/iter.rs
index 18abdbed5af6f..6f75808015e71 100644
--- a/library/core/src/slice/iter.rs
+++ b/library/core/src/slice/iter.rs
@@ -3050,6 +3050,20 @@ where
         if self.slice.is_empty() { (0, Some(0)) } else { (1, Some(self.slice.len())) }
     }
 
+    #[inline]
+    fn count(mut self) -> usize {
+        let Some((mut previous, rest)) = self.slice.split_first() else {
+            return 0;
+        };
+
+        let mut count = 1;
+        for current in rest {
+            count += usize::from(!(self.predicate)(previous, current));
+            previous = current;
+        }
+        count
+    }
+
     #[inline]
     fn last(mut self) -> Option<Self::Item> {
         self.next_back()

From 9116f7db7b984960bdf1730e6b5df168440db300 Mon Sep 17 00:00:00 2001
From: Scott McMurray <scottmcm@users.noreply.github.com>
Date: Tue, 7 Jul 2026 00:41:13 -0700
Subject: [PATCH 03/16] Fix `unaligned_volatile_store` by removing
 `MemFlags::UNALIGNED`

---
 compiler/rustc_codegen_gcc/src/builder.rs     |  2 +-
 .../rustc_codegen_gcc/src/intrinsic/mod.rs    | 10 --------
 compiler/rustc_codegen_llvm/src/builder.rs    |  3 +--
 compiler/rustc_codegen_llvm/src/intrinsic.rs  | 10 --------
 compiler/rustc_codegen_ssa/src/lib.rs         |  3 ++-
 .../rustc_codegen_ssa/src/mir/intrinsic.rs    |  8 ++----
 compiler/rustc_codegen_ssa/src/mir/operand.rs |  8 ------
 compiler/rustc_codegen_ssa/src/mir/place.rs   |  7 ++++++
 tests/codegen-llvm/intrinsics/volatile.rs     | 25 +++++++++++++++++--
 9 files changed, 36 insertions(+), 40 deletions(-)

diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs
index f9358872299b7..425d9d6226f83 100644
--- a/compiler/rustc_codegen_gcc/src/builder.rs
+++ b/compiler/rustc_codegen_gcc/src/builder.rs
@@ -1155,7 +1155,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         // NOTE: libgccjit does not support specifying the alignment on the assignment, so we cast
         // to type so it gets the proper alignment.
         let destination_type = destination.to_rvalue().get_type().unqualified();
-        let align = if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() };
+        let align = align.bytes();
         let mut modified_destination_type = destination_type.get_aligned(align);
         if flags.contains(MemFlags::VOLATILE) {
             modified_destination_type = modified_destination_type.make_volatile();
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
index 78a4c7e88c895..80d4a5cf11ad8 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
@@ -377,16 +377,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
                     load
                 }
             }
-            sym::volatile_store => {
-                let dst = args[0].deref(self.cx());
-                args[1].val.volatile_store(self, dst);
-                return IntrinsicResult::WroteIntoPlace;
-            }
-            sym::unaligned_volatile_store => {
-                let dst = args[0].deref(self.cx());
-                args[1].val.unaligned_volatile_store(self, dst);
-                return IntrinsicResult::WroteIntoPlace;
-            }
             sym::prefetch_read_data
             | sym::prefetch_write_data
             | sym::prefetch_read_instruction
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 804547745c5d5..844a5028f7184 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -873,8 +873,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unsafe {
             let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
             let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
-            let align =
-                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
+            let align = align.bytes() as c_uint;
             llvm::LLVMSetAlignment(store, align);
             if flags.contains(MemFlags::VOLATILE) {
                 llvm::LLVMSetVolatile(store, llvm::TRUE);
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 7bd604bdbbd76..8fb0dd3e2ef8b 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -389,16 +389,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     };
                 }
             }
-            sym::volatile_store => {
-                let dst = args[0].deref(self.cx());
-                args[1].val.volatile_store(self, dst);
-                return IntrinsicResult::Operand(OperandValue::ZeroSized);
-            }
-            sym::unaligned_volatile_store => {
-                let dst = args[0].deref(self.cx());
-                args[1].val.unaligned_volatile_store(self, dst);
-                return IntrinsicResult::Operand(OperandValue::ZeroSized);
-            }
             sym::prefetch_read_data
             | sym::prefetch_write_data
             | sym::prefetch_read_instruction
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 200f6aac12502..994c761a5901e 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -163,11 +163,12 @@ pub enum ModuleKind {
 }
 
 bitflags::bitflags! {
+    /// This previously had an `UNALIGNED` variant, but that should never be done via flags.
+    /// If you want something to be unaligned, see [`mir::place::PlaceRef::unaligned`].
     #[derive(Debug, Clone, Copy, PartialEq, Eq)]
     pub struct MemFlags: u8 {
         const VOLATILE = 1 << 0;
         const NONTEMPORAL = 1 << 1;
-        const UNALIGNED = 1 << 2;
         /// Indicates that writing through the stored pointer is undefined behavior.
         /// Only valid on stores of pointers, or pairs where the first element is a pointer.
         /// In the latter case, the flag only applies to the first element of the pair.
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 413cb5f2c860a..6116c0fa9eb98 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -273,16 +273,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 );
                 OperandValue::ZeroSized
             }
-            sym::volatile_store => {
+            sym::volatile_store | sym::unaligned_volatile_store => {
                 let dst = args[0].deref(bx.cx());
+                let dst = if name == sym::volatile_store { dst } else { dst.unaligned() };
                 args[1].val.volatile_store(bx, dst);
                 OperandValue::ZeroSized
             }
-            sym::unaligned_volatile_store => {
-                let dst = args[0].deref(bx.cx());
-                args[1].val.unaligned_volatile_store(bx, dst);
-                OperandValue::ZeroSized
-            }
             sym::disjoint_bitor => {
                 let a = args[0].immediate();
                 let b = args[1].immediate();
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index 0246da3f27829..6459208ea0aa2 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -968,14 +968,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
         self.store_with_flags(bx, dest, MemFlags::VOLATILE);
     }
 
-    pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
-        self,
-        bx: &mut Bx,
-        dest: PlaceRef<'tcx, V>,
-    ) {
-        self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
-    }
-
     pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
         self,
         bx: &mut Bx,
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index 50166609670b6..b592e4a339346 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -318,6 +318,13 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
     pub fn storage_dead<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) {
         bx.lifetime_end(self.val.llval, self.layout.size);
     }
+
+    /// The same place, but with [`PlaceValue::align`] lowered to [`Align::ONE`].
+    pub fn unaligned(self) -> Self {
+        let Self { val, layout } = self;
+        let val = PlaceValue { align: Align::ONE, ..val };
+        Self { val, layout }
+    }
 }
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
diff --git a/tests/codegen-llvm/intrinsics/volatile.rs b/tests/codegen-llvm/intrinsics/volatile.rs
index 4ba4c5edd2df6..42ffd196f3d86 100644
--- a/tests/codegen-llvm/intrinsics/volatile.rs
+++ b/tests/codegen-llvm/intrinsics/volatile.rs
@@ -162,7 +162,28 @@ pub unsafe fn unaligned_volatile_load_fat(a: *const UninitFatPointer) -> UninitF
 
 // CHECK-LABEL: @unaligned_volatile_store
 #[no_mangle]
-pub unsafe fn unaligned_volatile_store(a: *mut u8, b: u8) {
-    // CHECK: store volatile
+pub unsafe fn unaligned_volatile_store(a: *mut u16, b: u16) {
+    // CHECK: store volatile i16 %b, ptr %a, align 1
+    intrinsics::unaligned_volatile_store(a, b)
+}
+
+// CHECK-LABEL: @unaligned_volatile_store_pair
+#[no_mangle]
+pub unsafe fn unaligned_volatile_store_pair(a: *mut (u16, u16), b: (u16, u16)) {
+    // CHECK: store volatile i16 %b.0, ptr %a, align 1
+    // CHECK: [[TEMP:%.+]] = getelementptr inbounds i8, ptr %a, {{i16|i32|i64}} 2
+    // CHECK: store volatile i16 %b.1, ptr [[TEMP]], align 1
+    intrinsics::unaligned_volatile_store(a, b)
+}
+
+// CHECK-LABEL: @unaligned_volatile_store_array
+#[no_mangle]
+pub unsafe fn unaligned_volatile_store_array(a: *mut [u16; 7], b: [u16; 7]) {
+    // Note that only the store side is unaligned; the load from the argument is aligned.
+
+    // CHECK-NOT: memcpy
+    // CHECK: call void @llvm.memcpy{{.+}}(ptr align 1 %a, ptr align 2 %b, {{i16|i32|i64}} 14, i1 true)
+    // CHECK-NOT: memcpy
+    // CHECK: ret void
     intrinsics::unaligned_volatile_store(a, b)
 }

From 017255b2e7f2763a613bac8a56a8bbecb07f36bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Tue, 7 Jul 2026 15:51:51 +0200
Subject: [PATCH 04/16] Add new PGO config section to bootstrap

---
 bootstrap.example.toml                        | 14 +++++
 src/bootstrap/src/core/build_steps/compile.rs | 14 ++---
 src/bootstrap/src/core/build_steps/dist.rs    |  2 +-
 src/bootstrap/src/core/config/config.rs       | 60 +++++++++++++++----
 src/bootstrap/src/core/config/flags.rs        |  8 ++-
 src/bootstrap/src/core/config/toml/mod.rs     | 18 +++++-
 src/bootstrap/src/core/config/toml/pgo.rs     | 30 ++++++++++
 src/bootstrap/src/core/config/toml/rust.rs    |  6 +-
 src/bootstrap/src/utils/change_tracker.rs     |  5 ++
 9 files changed, 131 insertions(+), 26 deletions(-)
 create mode 100644 src/bootstrap/src/core/config/toml/pgo.rs

diff --git a/bootstrap.example.toml b/bootstrap.example.toml
index 1ca4bc9159e7c..caa342e360d34 100644
--- a/bootstrap.example.toml
+++ b/bootstrap.example.toml
@@ -973,6 +973,20 @@
 #dist.vendor = if "is a tarball source" || "is a git repository" { true } else { false }
 
 
+# =============================================================================
+# Profile-guided optimization options
+#
+# Configure the PGO profile to be used for compilation, or a path where the
+# profile will be written by an instrumented binary, for individual components
+# =============================================================================
+[pgo]
+# Use the following profile to PGO optimize the Rust compiler.
+#rustc.use = "/tmp/profiles/foo.profraw"
+# Instrument the Rust compiler, so that when executed, it will gather profiles
+# to this path.
+#rustc.generate = "/tmp/profiles/foo.profraw"
+
+
 # =============================================================================
 # Options for specific targets
 #
diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
index 3216fd671c3f8..5a762867320c1 100644
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@@ -1293,12 +1293,12 @@ pub fn rustc_cargo(
         cargo.rustflag("-Clink-args=-Wl,--icf=all");
     }
 
-    if builder.config.rust_profile_use.is_some() && builder.config.rust_profile_generate.is_some() {
-        panic!("Cannot use and generate PGO profiles at the same time");
-    }
-    let is_collecting = if let Some(path) = &builder.config.rust_profile_generate {
+    eprintln!("Profile: {:?}", builder.config.rust_pgo.generate_profile);
+
+    let is_collecting = if let Some(path) = &builder.config.rust_pgo.generate_profile {
         if build_compiler.stage == 1 {
-            cargo.rustflag(&format!("-Cprofile-generate={path}"));
+            cargo
+                .rustflag(&format!("-Cprofile-generate={}", path.to_str().expect("non-UTF8 path")));
             // Apparently necessary to avoid overflowing the counters during
             // a Cargo build profile
             cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4");
@@ -1306,9 +1306,9 @@ pub fn rustc_cargo(
         } else {
             false
         }
-    } else if let Some(path) = &builder.config.rust_profile_use {
+    } else if let Some(path) = &builder.config.rust_pgo.use_profile {
         if build_compiler.stage == 1 {
-            cargo.rustflag(&format!("-Cprofile-use={path}"));
+            cargo.rustflag(&format!("-Cprofile-use={}", path.to_str().expect("non-UTF8 path")));
             if builder.is_verbose() {
                 cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function");
             }
diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index 114387d963a6d..d9aa6f7c18815 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -3035,7 +3035,7 @@ impl Step for ReproducibleArtifacts {
     fn run(self, builder: &Builder<'_>) -> Self::Output {
         let mut added_anything = false;
         let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple);
-        if let Some(path) = builder.config.rust_profile_use.as_ref() {
+        if let Some(path) = builder.config.rust_pgo.use_profile.as_ref() {
             tarball.add_file(path, ".", FileType::Regular);
             added_anything = true;
         }
diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index 267e21c599728..ddd2442802d10 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -41,6 +41,7 @@ use crate::core::config::toml::dist::Dist;
 use crate::core::config::toml::gcc::Gcc;
 use crate::core::config::toml::install::Install;
 use crate::core::config::toml::llvm::Llvm;
+use crate::core::config::toml::pgo::{Pgo, PgoConfig};
 use crate::core::config::toml::rust::{
     BootstrapOverrideLld, Rust, RustOptimize, check_incompatible_options_for_ci_rustc,
     parse_codegen_backends,
@@ -225,14 +226,13 @@ pub struct Config {
     pub rust_remap_debuginfo: bool,
     pub rust_new_symbol_mangling: Option<bool>,
     pub rust_annotate_moves_size_limit: Option<u64>,
-    pub rust_profile_use: Option<String>,
-    pub rust_profile_generate: Option<String>,
     pub rust_lto: RustcLto,
     pub rust_validate_mir_opts: Option<u32>,
     pub rust_std_features: BTreeSet<String>,
     pub rust_break_on_ice: bool,
     pub rust_parallel_frontend_threads: Option<u32>,
     pub rust_rustflags: Vec<String>,
+    pub rust_pgo: PgoConfig,
 
     pub llvm_profile_use: Option<String>,
     pub llvm_profile_generate: bool,
@@ -456,8 +456,20 @@ impl Config {
 
         // Now load the TOML config, as soon as possible
         let (mut toml, toml_path) = load_toml_config(&src, flags_config, &get_toml);
-
         postprocess_toml(&mut toml, &src, toml_path.clone(), &exec_ctx, &flags_set, &get_toml);
+        let TomlConfig {
+            change_id: toml_change_id,
+            build: toml_build,
+            install: toml_install,
+            llvm: toml_llvm,
+            gcc: toml_gcc,
+            rust: toml_rust,
+            target: toml_target,
+            dist: toml_dist,
+            pgo: toml_pgo,
+            profile: _,
+            include: _,
+        } = toml;
 
         // Now override TOML values with flags, to make sure that we won't later override flags with
         // TOML values by accident instead, because flags have higher priority.
@@ -523,7 +535,7 @@ impl Config {
             ccache: build_ccache,
             exclude: build_exclude,
             compiletest_allow_stage0: build_compiletest_allow_stage0,
-        } = toml.build.unwrap_or_default();
+        } = toml_build.unwrap_or_default();
 
         let Install {
             prefix: install_prefix,
@@ -533,7 +545,7 @@ impl Config {
             libdir: install_libdir,
             mandir: install_mandir,
             datadir: install_datadir,
-        } = toml.install.unwrap_or_default();
+        } = toml_install.unwrap_or_default();
 
         let Rust {
             optimize: rust_optimize,
@@ -595,7 +607,7 @@ impl Config {
             std_features: rust_std_features,
             break_on_ice: rust_break_on_ice,
             rustflags: rust_rustflags,
-        } = toml.rust.unwrap_or_default();
+        } = toml_rust.unwrap_or_default();
 
         let Llvm {
             optimize: llvm_optimize,
@@ -627,7 +639,7 @@ impl Config {
             enable_warnings: llvm_enable_warnings,
             download_ci_llvm: llvm_download_ci_llvm,
             build_config: llvm_build_config,
-        } = toml.llvm.unwrap_or_default();
+        } = toml_llvm.unwrap_or_default();
 
         let Dist {
             sign_folder: dist_sign_folder,
@@ -637,12 +649,35 @@ impl Config {
             compression_profile: dist_compression_profile,
             include_mingw_linker: dist_include_mingw_linker,
             vendor: dist_vendor,
-        } = toml.dist.unwrap_or_default();
+        } = toml_dist.unwrap_or_default();
 
         let Gcc {
             download_ci_gcc: gcc_download_ci_gcc,
             libgccjit_libs_dir: gcc_libgccjit_libs_dir,
-        } = toml.gcc.unwrap_or_default();
+        } = toml_gcc.unwrap_or_default();
+
+        let Pgo { rustc: pgo_rustc } = toml_pgo.unwrap_or_default();
+        let mut pgo_rustc = pgo_rustc.unwrap_or_default();
+
+        // Backcompat: flags have priority over config
+        if flags_rust_profile_use.is_some() || flags_rust_profile_generate.is_some() {
+            eprintln!(
+                "WARNING: the `--rust-profile-generate` and `--rust-profile-use` flags have been deprecated. Configure PGO through the config file instead, in the [pgo.rustc] section."
+            );
+        }
+        if rust_profile_use.is_some() || rust_profile_generate.is_some() {
+            eprintln!(
+                "WARNING: the `rust.profile-generate` and `rust.profile-use` config options have been deprecated. Configure PGO through the config file instead, in the [pgo.rustc] section."
+            );
+        }
+
+        pgo_rustc.use_profile =
+            flags_rust_profile_use.or(pgo_rustc.use_profile).or(rust_profile_use);
+        pgo_rustc.generate_profile =
+            flags_rust_profile_generate.or(pgo_rustc.generate_profile).or(rust_profile_generate);
+        if pgo_rustc.use_profile.is_some() && pgo_rustc.generate_profile.is_some() {
+            panic!("Cannot use and generate rust PGO profiles at the same time");
+        }
 
         if rust_bootstrap_override_lld.is_some() && rust_bootstrap_override_lld_legacy.is_some() {
             panic!(
@@ -877,7 +912,7 @@ impl Config {
         // Linux targets for which the user explicitly overrode the used linker
         let mut targets_with_user_linker_override = HashSet::new();
 
-        if let Some(t) = toml.target {
+        if let Some(t) = toml_target {
             for (triple, cfg) in t {
                 let TomlTarget {
                     cc: target_cc,
@@ -1337,7 +1372,7 @@ NOTE: Please add `--stage 2` to your command line, or if you're sure you want to
             cargo_info,
             cargo_native_static: build_cargo_native_static.unwrap_or(false),
             ccache,
-            change_id: toml.change_id.inner,
+            change_id: toml_change_id.inner,
             channel,
             ci_env,
             clippy_info,
@@ -1496,8 +1531,7 @@ NOTE: Please add `--stage 2` to your command line, or if you're sure you want to
                 .or(rust_overflow_checks)
                 .unwrap_or(rust_debug == Some(true)),
             rust_parallel_frontend_threads: rust_parallel_frontend_threads.map(threads_from_config),
-            rust_profile_generate: flags_rust_profile_generate.or(rust_profile_generate),
-            rust_profile_use: flags_rust_profile_use.or(rust_profile_use),
+            rust_pgo: pgo_rustc,
             rust_randomize_layout: rust_randomize_layout.unwrap_or(false),
             rust_remap_debuginfo: rust_remap_debuginfo.unwrap_or(false),
             rust_rpath: rust_rpath.unwrap_or(true),
diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs
index bf171f1de34e0..d6020262e0579 100644
--- a/src/bootstrap/src/core/config/flags.rs
+++ b/src/bootstrap/src/core/config/flags.rs
@@ -153,11 +153,14 @@ pub struct Flags {
 
     /// generate PGO profile with rustc build
     #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")]
-    pub rust_profile_generate: Option<String>,
+    // FIXME: Remove this option at the end of 2026
+    pub rust_profile_generate: Option<PathBuf>,
     /// use PGO profile for rustc build
+    // FIXME: Remove this option at the end of 2026
     #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")]
-    pub rust_profile_use: Option<String>,
+    pub rust_profile_use: Option<PathBuf>,
     /// use PGO profile for LLVM build
+    // FIXME: Remove this option at the end of 2026
     #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")]
     pub llvm_profile_use: Option<String>,
     // LLVM doesn't support a custom location for generating profile
@@ -165,6 +168,7 @@ pub struct Flags {
     //
     // llvm_out/build/profiles/ is the location this writes to.
     /// generate PGO profile with llvm built for rustc
+    // FIXME: Remove this option at the end of 2026
     #[arg(global = true, long)]
     pub llvm_profile_generate: bool,
     /// Enable BOLT link flags
diff --git a/src/bootstrap/src/core/config/toml/mod.rs b/src/bootstrap/src/core/config/toml/mod.rs
index f6dc5b67e1010..74c21617746bf 100644
--- a/src/bootstrap/src/core/config/toml/mod.rs
+++ b/src/bootstrap/src/core/config/toml/mod.rs
@@ -15,6 +15,7 @@ pub mod dist;
 pub mod gcc;
 pub mod install;
 pub mod llvm;
+pub mod pgo;
 pub mod rust;
 pub mod target;
 
@@ -27,6 +28,7 @@ use llvm::Llvm;
 use rust::Rust;
 use target::TomlTarget;
 
+use crate::core::config::toml::pgo::Pgo;
 use crate::core::config::{Merge, ReplaceOpt};
 use crate::{Config, HashMap, HashSet, Path, PathBuf, exit, fs, t};
 
@@ -47,6 +49,7 @@ pub(crate) struct TomlConfig {
     pub(super) rust: Option<Rust>,
     pub(super) target: Option<HashMap<String, TomlTarget>>,
     pub(super) dist: Option<Dist>,
+    pub(super) pgo: Option<Pgo>,
     pub(super) profile: Option<String>,
     pub(super) include: Option<Vec<PathBuf>>,
 }
@@ -56,7 +59,19 @@ impl Merge for TomlConfig {
         &mut self,
         parent_config_path: Option<PathBuf>,
         included_extensions: &mut HashSet<PathBuf>,
-        TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id, include }: Self,
+        TomlConfig {
+            build,
+            install,
+            llvm,
+            gcc,
+            rust,
+            dist,
+            target,
+            pgo,
+            profile,
+            change_id,
+            include,
+        }: Self,
         replace: ReplaceOpt,
     ) {
         fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) {
@@ -78,6 +93,7 @@ impl Merge for TomlConfig {
         do_merge(&mut self.gcc, gcc, replace);
         do_merge(&mut self.rust, rust, replace);
         do_merge(&mut self.dist, dist, replace);
+        do_merge(&mut self.pgo, pgo, replace);
 
         match (self.target.as_mut(), target) {
             (_, None) => {}
diff --git a/src/bootstrap/src/core/config/toml/pgo.rs b/src/bootstrap/src/core/config/toml/pgo.rs
new file mode 100644
index 0000000000000..dde76f078fc39
--- /dev/null
+++ b/src/bootstrap/src/core/config/toml/pgo.rs
@@ -0,0 +1,30 @@
+//! This module defines the `Pgo` struct, which represents the `[pgo]` table
+//! in the `bootstrap.toml` configuration file.
+//!
+//! The `[pgo]` table contains options related PGO (Profile-Guided Optimization) of various
+//! components built by bootstrap.
+
+use serde::{Deserialize, Deserializer};
+
+use crate::core::config::Merge;
+use crate::core::config::toml::ReplaceOpt;
+use crate::{HashSet, PathBuf, define_config, exit};
+
+#[derive(Clone, Default, Debug, serde_derive::Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct PgoConfig {
+    /// Use the given PGO profile to optimize a component.
+    #[serde(default, rename = "use")]
+    pub use_profile: Option<PathBuf>,
+    /// Build a component with PGO instrumentation. Once executed, the profiles will be stored
+    /// into this path.
+    #[serde(default, rename = "generate")]
+    pub generate_profile: Option<PathBuf>,
+}
+
+define_config! {
+    #[derive(Default)]
+    struct Pgo {
+        rustc: Option<PgoConfig> = "rustc",
+    }
+}
diff --git a/src/bootstrap/src/core/config/toml/rust.rs b/src/bootstrap/src/core/config/toml/rust.rs
index 2facf839521d3..5c0fd37e46c2e 100644
--- a/src/bootstrap/src/core/config/toml/rust.rs
+++ b/src/bootstrap/src/core/config/toml/rust.rs
@@ -64,8 +64,10 @@ define_config! {
         ehcont_guard: Option<bool> = "ehcont-guard",
         new_symbol_mangling: Option<bool> = "new-symbol-mangling",
         annotate_moves_size_limit: Option<u64> = "annotate-moves-size-limit",
-        profile_generate: Option<String> = "profile-generate",
-        profile_use: Option<String> = "profile-use",
+        // FIXME: Remove this option at the end of 2026
+        profile_generate: Option<PathBuf> = "profile-generate",
+        // FIXME: Remove this option at the end of 2026
+        profile_use: Option<PathBuf> = "profile-use",
         // ignored; this is set from an env var set by bootstrap.py
         download_rustc: Option<StringOrBool> = "download-rustc",
         lto: Option<String> = "lto",
diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs
index b34c1b55700f8..94a7acf30d4e9 100644
--- a/src/bootstrap/src/utils/change_tracker.rs
+++ b/src/bootstrap/src/utils/change_tracker.rs
@@ -636,4 +636,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Info,
         summary: "New option `rust.compress-debuginfo` allows configuring whether Rust and C/C++ debuginfo should be compressed.",
     },
+    ChangeInfo {
+        change_id: 999999,
+        severity: ChangeSeverity::Info,
+        summary: "New config section `pgo` was introduced, to configure PGO profiling options. The `--rust-profile-use`/`--rust-profile-generate`/`--llvm-profile-use`/`--llvm-profile-generate` flags and the `rust.profile-use`/`rust.profile-generate` config options have been deprecated.",
+    },
 ];

From 33da2c897239a35693804f91e3697cf481d170d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Tue, 7 Jul 2026 16:04:32 +0200
Subject: [PATCH 05/16] Port LLVM PGO to the new PGO config section

---
 bootstrap.example.toml                        |  6 ++-
 src/bootstrap/src/core/build_steps/compile.rs |  2 +-
 src/bootstrap/src/core/build_steps/dist.rs    |  2 +-
 src/bootstrap/src/core/build_steps/llvm.rs    | 15 +++---
 src/bootstrap/src/core/config/config.rs       | 46 ++++++++++++++++---
 src/bootstrap/src/core/config/flags.rs        |  2 +-
 src/bootstrap/src/core/config/toml/pgo.rs     |  1 +
 7 files changed, 58 insertions(+), 16 deletions(-)

diff --git a/bootstrap.example.toml b/bootstrap.example.toml
index caa342e360d34..b48f7ad7da742 100644
--- a/bootstrap.example.toml
+++ b/bootstrap.example.toml
@@ -985,7 +985,11 @@
 # Instrument the Rust compiler, so that when executed, it will gather profiles
 # to this path.
 #rustc.generate = "/tmp/profiles/foo.profraw"
-
+# Use the following profile to PGO optimize LLVM.
+#llvm.use = "/tmp/profiles/foo.profraw"
+# Instrument LLVM, so that when executed, it will gather profiles
+# Note: for LLVM specifically, this should be a directory, not a file path.
+#llvm.generate = "/tmp/profiles"
 
 # =============================================================================
 # Options for specific targets
diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
index 5a762867320c1..3ef6428b2f746 100644
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@@ -1464,7 +1464,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect
     // found. This is to avoid the linker errors about undefined references to
     // `__llvm_profile_instrument_memop` when linking `rustc_driver`.
     let mut llvm_linker_flags = String::new();
-    if builder.config.llvm_profile_generate
+    if builder.config.llvm_pgo.generate_profile.is_some()
         && target.is_msvc()
         && let Some(ref clang_cl_path) = builder.config.llvm_clang_cl
     {
diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index d9aa6f7c18815..fdb13d3ca6a34 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -3039,7 +3039,7 @@ impl Step for ReproducibleArtifacts {
             tarball.add_file(path, ".", FileType::Regular);
             added_anything = true;
         }
-        if let Some(path) = builder.config.llvm_profile_use.as_ref() {
+        if let Some(path) = builder.config.llvm_pgo.use_profile.as_ref() {
             tarball.add_file(path, ".", FileType::Regular);
             added_anything = true;
         }
diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs
index 6f1b8532f031b..f40b384a589ba 100644
--- a/src/bootstrap/src/core/build_steps/llvm.rs
+++ b/src/bootstrap/src/core/build_steps/llvm.rs
@@ -19,7 +19,7 @@ use build_helper::git::PathFreshness;
 
 use crate::core::build_steps::llvm;
 use crate::core::builder::{Builder, RunConfig, ShouldRun, Step, StepMetadata};
-use crate::core::config::{Config, TargetSelection};
+use crate::core::config::{Config, LlvmPgoGenerationMode, TargetSelection};
 use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
 use crate::utils::exec::command;
 use crate::utils::helpers::{
@@ -369,14 +369,17 @@ impl Step for Llvm {
         // This flag makes sure `FileCheck` is copied in the final binaries directory.
         cfg.define("LLVM_INSTALL_UTILS", "ON");
 
-        if builder.config.llvm_profile_generate {
+        if let Some(mode) = builder.config.llvm_pgo.generate_profile.as_ref() {
             cfg.define("LLVM_BUILD_INSTRUMENTED", "IR");
-            if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") {
-                cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir);
+            match mode {
+                LlvmPgoGenerationMode::Implicit => {}
+                LlvmPgoGenerationMode::Directory(llvm_profile_dir) => {
+                    cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir);
+                }
             }
             cfg.define("LLVM_BUILD_RUNTIME", "No");
         }
-        if let Some(path) = builder.config.llvm_profile_use.as_ref() {
+        if let Some(path) = builder.config.llvm_pgo.use_profile.as_ref() {
             cfg.define("LLVM_PROFDATA_FILE", path);
         }
 
@@ -1326,7 +1329,7 @@ impl Step for Lld {
         // when doing PGO on CI, cmake or clang-cl don't automatically link clang's
         // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid
         // linking errors, much like LLVM's cmake setup does in that situation.
-        if builder.config.llvm_profile_generate
+        if builder.config.llvm_pgo.generate_profile.is_some()
             && target.is_msvc()
             && let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref()
         {
diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index ddd2442802d10..a76792a58252b 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -191,6 +191,7 @@ pub struct Config {
     pub llvm_cxxflags: Option<String>,
     pub llvm_ldflags: Option<String>,
     pub llvm_use_libcxx: bool,
+    pub llvm_pgo: LlvmPgoConfig,
 
     // gcc codegen options
     pub gcc_ci_mode: GccCiMode,
@@ -234,8 +235,6 @@ pub struct Config {
     pub rust_rustflags: Vec<String>,
     pub rust_pgo: PgoConfig,
 
-    pub llvm_profile_use: Option<String>,
-    pub llvm_profile_generate: bool,
     pub llvm_libunwind_default: Option<LlvmLibunwind>,
     pub enable_bolt_settings: bool,
 
@@ -656,8 +655,7 @@ impl Config {
             libgccjit_libs_dir: gcc_libgccjit_libs_dir,
         } = toml_gcc.unwrap_or_default();
 
-        let Pgo { rustc: pgo_rustc } = toml_pgo.unwrap_or_default();
-        let mut pgo_rustc = pgo_rustc.unwrap_or_default();
+        let Pgo { rustc: pgo_rustc, llvm: pgo_llvm } = toml_pgo.unwrap_or_default();
 
         // Backcompat: flags have priority over config
         if flags_rust_profile_use.is_some() || flags_rust_profile_generate.is_some() {
@@ -670,7 +668,13 @@ impl Config {
                 "WARNING: the `rust.profile-generate` and `rust.profile-use` config options have been deprecated. Configure PGO through the config file instead, in the [pgo.rustc] section."
             );
         }
+        if flags_llvm_profile_use.is_some() || flags_llvm_profile_generate {
+            eprintln!(
+                "WARNING: the `--llvm-profile-generate` and `--llvm-profile-use` flags have been deprecated. Configure PGO through the config file instead, in the [pgo.llvm] section."
+            );
+        }
 
+        let mut pgo_rustc = pgo_rustc.unwrap_or_default();
         pgo_rustc.use_profile =
             flags_rust_profile_use.or(pgo_rustc.use_profile).or(rust_profile_use);
         pgo_rustc.generate_profile =
@@ -679,6 +683,23 @@ impl Config {
             panic!("Cannot use and generate rust PGO profiles at the same time");
         }
 
+        let pgo_llvm = pgo_llvm.unwrap_or_default();
+        let pgo_llvm = LlvmPgoConfig {
+            use_profile: flags_llvm_profile_use.or(pgo_llvm.use_profile),
+            generate_profile: if flags_llvm_profile_generate {
+                Some(if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") {
+                    LlvmPgoGenerationMode::Directory(PathBuf::from(llvm_profile_dir))
+                } else {
+                    LlvmPgoGenerationMode::Implicit
+                })
+            } else {
+                pgo_llvm.generate_profile.map(LlvmPgoGenerationMode::Directory)
+            },
+        };
+        if pgo_llvm.use_profile.is_some() && pgo_llvm.generate_profile.is_some() {
+            panic!("Cannot use and generate LLVM PGO profiles at the same time");
+        }
+
         if rust_bootstrap_override_lld.is_some() && rust_bootstrap_override_lld_legacy.is_some() {
             panic!(
                 "Cannot use both `rust.use-lld` and `rust.bootstrap-override-lld`. Please use only `rust.bootstrap-override-lld`"
@@ -1463,10 +1484,9 @@ NOTE: Please add `--stage 2` to your command line, or if you're sure you want to
             ),
             llvm_offload: llvm_offload.unwrap_or(false),
             llvm_optimize: llvm_optimize.unwrap_or(true),
+            llvm_pgo: pgo_llvm,
             llvm_plugins: llvm_plugin.unwrap_or(false),
             llvm_polly: llvm_polly.unwrap_or(false),
-            llvm_profile_generate: flags_llvm_profile_generate,
-            llvm_profile_use: flags_llvm_profile_use,
             llvm_release_debuginfo: llvm_release_debuginfo.unwrap_or(false),
             llvm_static_stdcpp: llvm_static_libstdcpp.unwrap_or(false),
             llvm_targets,
@@ -2064,6 +2084,20 @@ fn compute_src_directory(src_dir: Option<PathBuf>, exec_ctx: &ExecutionContext)
     None
 }
 
+#[derive(Clone)]
+pub enum LlvmPgoGenerationMode {
+    /// Enable PGO instrumentation that will write profiles into a default path.
+    Implicit,
+    /// Enable PGO instrumentation that will write profiles into the specified directory.
+    Directory(PathBuf),
+}
+
+#[derive(Clone)]
+pub struct LlvmPgoConfig {
+    pub use_profile: Option<PathBuf>,
+    pub generate_profile: Option<LlvmPgoGenerationMode>,
+}
+
 /// Loads bootstrap TOML config and returns the config together with a path from where
 /// it was loaded.
 /// `src` is the source root directory, and `config_path` is an optionally provided path to the
diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs
index d6020262e0579..775de587b3e54 100644
--- a/src/bootstrap/src/core/config/flags.rs
+++ b/src/bootstrap/src/core/config/flags.rs
@@ -162,7 +162,7 @@ pub struct Flags {
     /// use PGO profile for LLVM build
     // FIXME: Remove this option at the end of 2026
     #[arg(global = true, value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")]
-    pub llvm_profile_use: Option<String>,
+    pub llvm_profile_use: Option<PathBuf>,
     // LLVM doesn't support a custom location for generating profile
     // information.
     //
diff --git a/src/bootstrap/src/core/config/toml/pgo.rs b/src/bootstrap/src/core/config/toml/pgo.rs
index dde76f078fc39..b845f3c182b53 100644
--- a/src/bootstrap/src/core/config/toml/pgo.rs
+++ b/src/bootstrap/src/core/config/toml/pgo.rs
@@ -26,5 +26,6 @@ define_config! {
     #[derive(Default)]
     struct Pgo {
         rustc: Option<PgoConfig> = "rustc",
+        llvm: Option<PgoConfig> = "llvm",
     }
 }

From b576701d43c6ac88af39c9bd31e6ae136957af54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Tue, 7 Jul 2026 16:13:00 +0200
Subject: [PATCH 06/16] Use the new bootstrap PGO options in `opt-dist`

---
 src/tools/opt-dist/src/exec.rs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/tools/opt-dist/src/exec.rs b/src/tools/opt-dist/src/exec.rs
index a3935f9835956..7c6b07b8c8bb0 100644
--- a/src/tools/opt-dist/src/exec.rs
+++ b/src/tools/opt-dist/src/exec.rs
@@ -134,20 +134,21 @@ impl Bootstrap {
     pub fn llvm_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
         self.cmd = self
             .cmd
-            .arg("--llvm-profile-generate")
-            .env("LLVM_PROFILE_DIR", profile_dir.join("prof-%p").as_str());
+            .arg("--set")
+            .arg(format!(r#"pgo.llvm.generate="{}""#, profile_dir.join("prof-%p").as_str()));
         self
     }
 
     pub fn llvm_pgo_optimize(mut self, profile: Option<&LlvmPGOProfile>) -> Self {
         if let Some(prof) = profile {
-            self.cmd = self.cmd.arg("--llvm-profile-use").arg(prof.0.as_str());
+            self.cmd = self.cmd.arg("--set").arg(format!(r#"pgo.llvm.use="{}""#, prof.0.as_str()));
         }
         self
     }
 
     pub fn rustc_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
-        self.cmd = self.cmd.arg("--rust-profile-generate").arg(profile_dir.as_str());
+        self.cmd =
+            self.cmd.arg("--set").arg(format!(r#"pgo.rustc.generate="{}""#, profile_dir.as_str()));
         self
     }
 
@@ -162,7 +163,7 @@ impl Bootstrap {
     }
 
     pub fn rustc_pgo_optimize(mut self, profile: &RustcPGOProfile) -> Self {
-        self.cmd = self.cmd.arg("--rust-profile-use").arg(profile.0.as_str());
+        self.cmd = self.cmd.arg("--set").arg(format!(r#"pgo.rustc.use="{}""#, profile.0.as_str()));
         self
     }
 

From 1f6b1d5683f47618b29c0bc5514c6e8b571dc8eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Tue, 7 Jul 2026 16:34:10 +0200
Subject: [PATCH 07/16] Fix bootstrap example TOML

---
 bootstrap.example.toml                        | 10 +++++-----
 src/bootstrap/src/core/build_steps/compile.rs |  2 --
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/bootstrap.example.toml b/bootstrap.example.toml
index b48f7ad7da742..2c9a5ab1a152c 100644
--- a/bootstrap.example.toml
+++ b/bootstrap.example.toml
@@ -979,17 +979,17 @@
 # Configure the PGO profile to be used for compilation, or a path where the
 # profile will be written by an instrumented binary, for individual components
 # =============================================================================
-[pgo]
+
 # Use the following profile to PGO optimize the Rust compiler.
-#rustc.use = "/tmp/profiles/foo.profraw"
+#pgo.rustc.use = "/tmp/profiles/foo.profraw"
 # Instrument the Rust compiler, so that when executed, it will gather profiles
 # to this path.
-#rustc.generate = "/tmp/profiles/foo.profraw"
+#pgo.rustc.generate = "/tmp/profiles/foo.profraw"
 # Use the following profile to PGO optimize LLVM.
-#llvm.use = "/tmp/profiles/foo.profraw"
+#pgo.llvm.use = "/tmp/profiles/foo.profraw"
 # Instrument LLVM, so that when executed, it will gather profiles
 # Note: for LLVM specifically, this should be a directory, not a file path.
-#llvm.generate = "/tmp/profiles"
+#pgo.llvm.generate = "/tmp/profiles"
 
 # =============================================================================
 # Options for specific targets
diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
index 3ef6428b2f746..bc6d1f94d698b 100644
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@@ -1293,8 +1293,6 @@ pub fn rustc_cargo(
         cargo.rustflag("-Clink-args=-Wl,--icf=all");
     }
 
-    eprintln!("Profile: {:?}", builder.config.rust_pgo.generate_profile);
-
     let is_collecting = if let Some(path) = &builder.config.rust_pgo.generate_profile {
         if build_compiler.stage == 1 {
             cargo

From f25731fb5b85d5388297c7977c5c4aa765212b0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Thu, 9 Jul 2026 20:17:18 +0200
Subject: [PATCH 08/16] Normalize paths on Windows

---
 src/tools/opt-dist/src/exec.rs     | 25 +++++++++++++++++--------
 src/tools/opt-dist/src/tests.rs    |  8 ++++----
 src/tools/opt-dist/src/utils/io.rs |  6 ++++++
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/tools/opt-dist/src/exec.rs b/src/tools/opt-dist/src/exec.rs
index 7c6b07b8c8bb0..d53ec45f9f49c 100644
--- a/src/tools/opt-dist/src/exec.rs
+++ b/src/tools/opt-dist/src/exec.rs
@@ -8,6 +8,7 @@ use crate::environment::Environment;
 use crate::metrics::{load_metrics, record_metrics};
 use crate::timer::TimerSection;
 use crate::training::{BoltProfile, LlvmPGOProfile, RustcPGOProfile};
+use crate::utils::io::normalize_path;
 
 #[derive(Default)]
 pub struct CmdBuilder {
@@ -132,23 +133,28 @@ impl Bootstrap {
     }
 
     pub fn llvm_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
-        self.cmd = self
-            .cmd
-            .arg("--set")
-            .arg(format!(r#"pgo.llvm.generate="{}""#, profile_dir.join("prof-%p").as_str()));
+        self.cmd = self.cmd.arg("--set").arg(format!(
+            r#"pgo.llvm.generate="{}""#,
+            normalize_path(&profile_dir.join("prof-%p")).as_str()
+        ));
         self
     }
 
     pub fn llvm_pgo_optimize(mut self, profile: Option<&LlvmPGOProfile>) -> Self {
         if let Some(prof) = profile {
-            self.cmd = self.cmd.arg("--set").arg(format!(r#"pgo.llvm.use="{}""#, prof.0.as_str()));
+            self.cmd = self
+                .cmd
+                .arg("--set")
+                .arg(format!(r#"pgo.llvm.use="{}""#, normalize_path(&prof.0).as_str()));
         }
         self
     }
 
     pub fn rustc_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
-        self.cmd =
-            self.cmd.arg("--set").arg(format!(r#"pgo.rustc.generate="{}""#, profile_dir.as_str()));
+        self.cmd = self
+            .cmd
+            .arg("--set")
+            .arg(format!(r#"pgo.rustc.generate="{}""#, normalize_path(profile_dir).as_str()));
         self
     }
 
@@ -163,7 +169,10 @@ impl Bootstrap {
     }
 
     pub fn rustc_pgo_optimize(mut self, profile: &RustcPGOProfile) -> Self {
-        self.cmd = self.cmd.arg("--set").arg(format!(r#"pgo.rustc.use="{}""#, profile.0.as_str()));
+        self.cmd = self
+            .cmd
+            .arg("--set")
+            .arg(format!(r#"pgo.rustc.use="{}""#, normalize_path(&profile.0).as_str()));
         self
     }
 
diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs
index 1aacfeaf538e1..e0e2c487db4c5 100644
--- a/src/tools/opt-dist/src/tests.rs
+++ b/src/tools/opt-dist/src/tests.rs
@@ -5,7 +5,7 @@ use camino::{Utf8Path, Utf8PathBuf};
 
 use crate::environment::{Environment, executable_extension};
 use crate::exec::cmd;
-use crate::utils::io::{copy_directory, find_file_in_dir, unpack_archive};
+use crate::utils::io::{copy_directory, find_file_in_dir, normalize_path, unpack_archive};
 
 /// Run tests on optimized dist artifacts.
 pub fn run_tests(env: &Environment) -> anyhow::Result<()> {
@@ -94,9 +94,9 @@ compiletest-allow-stage0=true
 [target.{host_triple}]
 llvm-config = "{llvm_config}"
 "#,
-        rustc = rustc_path.to_string().replace('\\', "/"),
-        cargo = cargo_path.to_string().replace('\\', "/"),
-        llvm_config = llvm_config.to_string().replace('\\', "/")
+        rustc = normalize_path(&rustc_path),
+        cargo = normalize_path(&cargo_path),
+        llvm_config = normalize_path(&llvm_config)
     );
     log::info!("Using following `bootstrap.toml` for running tests:\n{config_content}");
 
diff --git a/src/tools/opt-dist/src/utils/io.rs b/src/tools/opt-dist/src/utils/io.rs
index ce425bdfece2c..3e3ad36cb0e90 100644
--- a/src/tools/opt-dist/src/utils/io.rs
+++ b/src/tools/opt-dist/src/utils/io.rs
@@ -85,3 +85,9 @@ pub fn find_file_in_dir(
         )),
     }
 }
+
+/// Normalizes a file path so that it can be used on the CLI or in bootstrap config file.
+/// Replaces backslashes with forward slashes on Windows.
+pub fn normalize_path(path: &Utf8Path) -> Utf8PathBuf {
+    path.to_string().replace('\\', "/").into()
+}

From ab1907725ecc26033d5a3a6fd1377e37ed3ef262 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert@folkertdev.nl>
Date: Fri, 17 Apr 2026 11:25:05 +0200
Subject: [PATCH 09/16] Support `u128`/`i128` c-variadic arguments

On platforms where `clang` defines `__int128`.
---
 compiler/rustc_codegen_llvm/src/intrinsic.rs  |  9 +--
 compiler/rustc_codegen_llvm/src/va_arg.rs     | 81 +++++++++++++------
 library/core/src/ffi/va_list.rs               | 52 +++++++++++-
 tests/assembly-llvm/c-variadic/sparc.rs       | 18 +++++
 .../c-link-to-rust-va-list-fn/checkrust.rs    | 45 ++++++++++-
 .../run-make/c-link-to-rust-va-list-fn/test.c |  5 ++
 tests/ui/c-variadic/roundtrip.rs              | 45 ++++++++++-
 7 files changed, 220 insertions(+), 35 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 54456ddea620b..20dca864cff6b 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -3,8 +3,8 @@ use std::ffi::c_uint;
 use std::{assert_matches, iter, ptr};
 
 use rustc_abi::{
-    AddressSpace, Align, BackendRepr, CVariadicStatus, Float, HasDataLayout, Integer,
-    NumScalableVectors, Primitive, Size, WrappingRange,
+    AddressSpace, Align, BackendRepr, CVariadicStatus, Float, HasDataLayout, NumScalableVectors,
+    Primitive, Size, WrappingRange,
 };
 use rustc_codegen_ssa::RetagInfo;
 use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
@@ -315,11 +315,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     Primitive::Pointer(_) => {
                         // Pointers are always OK.
                     }
-                    Primitive::Int(Integer::I128, _) => {
-                        // FIXME: maybe we should support these? At least on 32-bit powerpc
-                        // the logic in LLVM does not handle i128 correctly though.
-                        bug!("the va_arg intrinsic does not support `i128`/`u128`")
-                    }
                     Primitive::Int(..) => {
                         let int_width = self.cx().size_of(result_layout.ty).bits();
                         let target_c_int_width = self.cx().sess().target.options.c_int_width;
diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs
index d2583f94cf80c..92b0709506526 100644
--- a/compiler/rustc_codegen_llvm/src/va_arg.rs
+++ b/compiler/rustc_codegen_llvm/src/va_arg.rs
@@ -1,5 +1,6 @@
-use rustc_abi::{Align, BackendRepr, CVariadicStatus, Endian, HasDataLayout, Primitive, Size};
-use rustc_codegen_ssa::MemFlags;
+use rustc_abi::{
+    Align, BackendRepr, CVariadicStatus, Endian, Float, HasDataLayout, Integer, Primitive, Size,
+};
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::OperandRef;
 use rustc_codegen_ssa::traits::{
@@ -77,6 +78,35 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     }
 }
 
+/// Some backends apply special alignment rules to c-variadic arguments.
+fn get_param_type_alignment<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    layout: TyAndLayout<'tcx>,
+) -> Align {
+    let BackendRepr::Scalar(scalar) = layout.backend_repr else {
+        bug!("unexpected backend repr {:?}", layout.backend_repr);
+    };
+
+    match bx.cx.tcx.sess.target.arch {
+        Arch::PowerPC64 => match scalar.primitive() {
+            Primitive::Int(integer, _) => match integer {
+                Integer::I8 | Integer::I16 => unreachable!(),
+                Integer::I32 | Integer::I64 => { /* fall through */ }
+                Integer::I128 => return Align::EIGHT,
+            },
+            Primitive::Float(float) => match float {
+                Float::F16 | Float::F32 => unreachable!(),
+                Float::F64 => { /* fall through */ }
+                Float::F128 => return Align::from_bytes(16).unwrap(),
+            },
+            Primitive::Pointer(_) => { /* fall through */ }
+        },
+        _ => { /* fall through */ }
+    }
+
+    layout.align.abi
+}
+
 enum PassMode {
     Direct,
     Indirect,
@@ -136,23 +166,23 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
         (
             bx.cx.layout_of(Ty::new_imm_ptr(bx.cx.tcx, target_ty)).llvm_type(bx.cx),
             bx.cx.data_layout().pointer_size(),
-            bx.cx.data_layout().pointer_align(),
+            bx.cx.data_layout().pointer_align().abi,
         )
     } else {
-        (layout.llvm_type(bx.cx), layout.size, layout.align)
+        (layout.llvm_type(bx.cx), layout.size, get_param_type_alignment(bx, layout))
     };
     let (addr, addr_align) = emit_direct_ptr_va_arg(
         bx,
         list,
         size,
-        align.abi,
+        align,
         slot_size,
         allow_higher_align,
         force_right_adjust,
     );
     if indirect {
         let tmp_ret = bx.load(llty, addr, addr_align);
-        bx.load(layout.llvm_type(bx.cx), tmp_ret, align.abi)
+        bx.load(layout.llvm_type(bx.cx), tmp_ret, align)
     } else {
         bx.load(llty, addr, addr_align)
     }
@@ -585,8 +615,10 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
     // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
     // l->fp_offset > 176 - num_fp * 16 go to step 7.
 
+    // We support x86_64-unknown-linux-gnux32 which uses 4-byte pointers.
     let unsigned_int_offset = 4;
-    let ptr_offset = 8;
+    let ptr_offset = bx.tcx().data_layout.pointer_size().bytes();
+
     let gp_offset_ptr = va_list_addr;
     let fp_offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(unsigned_int_offset));
 
@@ -660,7 +692,7 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
                     let reg_hi_addr = bx.inbounds_ptradd(reg_lo_addr, bx.const_i32(16));
 
                     let align = layout.layout.align().abi;
-                    let tmp = bx.alloca(layout.layout.size(), align);
+                    let tmp = bx.alloca(layout.size, layout.align.abi);
 
                     let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
                     let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
@@ -682,7 +714,7 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
                         Primitive::Int(_, _) | Primitive::Pointer(_) => (gp_addr, fp_addr),
                     };
 
-                    let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
+                    let tmp = bx.alloca(layout.size, layout.align.abi);
 
                     let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
                     let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
@@ -749,16 +781,12 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>(
     src_align: Align,
 ) -> &'ll Value {
     if layout.layout.align.abi > src_align {
-        let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
-        bx.memcpy(
-            tmp,
-            layout.layout.align.abi,
-            reg_addr,
-            src_align,
-            bx.const_u32(layout.layout.size().bytes() as u32),
-            MemFlags::empty(),
-            None,
-        );
+        assert!(layout.ty.is_integral());
+
+        // A memcpy below optimizes poorly for 128-bit integers.
+        let tmp = bx.alloca(layout.size, layout.align.abi);
+        let val = bx.load(layout.llvm_type(bx), reg_addr, src_align);
+        bx.store(val, tmp, layout.align.abi);
         tmp
     } else {
         reg_addr
@@ -780,9 +808,14 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
     // byte boundary if alignment needed by type exceeds 8 byte boundary.
     // It isn't stated explicitly in the standard, but in practice we use
     // alignment greater than 16 where necessary.
-    if layout.layout.align.bytes() > 8 {
-        unreachable!("all instances of VaArgSafe have an alignment <= 8");
-    }
+    // The AMD64 psABI leaves unspecified what to do for alignments above 16, but
+    // this behavior for 32+ alignment matches clang.
+    // It currently (2026 July) can only occur for 16-byte-aligned types.
+    let overflow_arg_area_v = if layout.layout.align.bytes() > 8 {
+        round_pointer_up_to_alignment(bx, overflow_arg_area_v, layout.layout.align.abi)
+    } else {
+        overflow_arg_area_v
+    };
 
     // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
     let mem_addr = overflow_arg_area_v;
@@ -1069,7 +1102,8 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
         Arch::AArch64 => emit_aapcs_va_arg(bx, addr, target_ty),
         Arch::Arm => {
             // Types wider than 16 bytes are not currently supported. Clang has special logic for
-            // such types, but `VaArgSafe` is not implemented for any type that is this large.
+            // such types, but `VaArgSafe` is not implemented for any type that is this large on
+            // arm (i.e. 32-bit) targets.
             assert!(bx.cx.size_of(target_ty).bytes() <= 16);
 
             emit_ptr_va_arg(
@@ -1091,6 +1125,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             PassMode::Direct,
             SlotSize::Bytes8,
             AllowHigherAlign::Yes,
+            // ForceRightAdjust only takes effect on big-endian architectures.
             ForceRightAdjust::Yes,
         ),
         Arch::RiscV32 if target.llvm_abiname == LlvmAbi::Ilp32e => {
diff --git a/library/core/src/ffi/va_list.rs b/library/core/src/ffi/va_list.rs
index 55d3cd22354e5..1c31e1b690d63 100644
--- a/library/core/src/ffi/va_list.rs
+++ b/library/core/src/ffi/va_list.rs
@@ -299,6 +299,9 @@ const impl<'f> Drop for VaList<'f> {
 /// and [`c_float`] is promoted to [`c_double`]. Implementing this trait for types that are
 /// subject to this promotion rule is invalid.
 ///
+/// This trait is only implemented for 128-bit integers when the platform defines the `__int128`
+/// type.
+///
 /// [`c_int`]: core::ffi::c_int
 /// [`c_long`]: core::ffi::c_long
 /// [`c_longlong`]: core::ffi::c_longlong
@@ -310,8 +313,8 @@ const impl<'f> Drop for VaList<'f> {
 /// [`c_float`]: core::ffi::c_float
 /// [`c_double`]: core::ffi::c_double
 // We may unseal this trait in the future, but currently our `va_arg` implementations don't support
-// types with an alignment larger than 8, or with a non-scalar layout. Inline assembly can be used
-// to accept unsupported types in the meantime.
+// types with a non-scalar layout. Inline assembly can be used to accept unsupported types in the
+// meantime.
 #[lang = "va_arg_safe"]
 pub impl(self) unsafe trait VaArgSafe: Copy {}
 
@@ -352,6 +355,51 @@ unsafe impl VaArgSafe for u32 {}
 unsafe impl VaArgSafe for u64 {}
 unsafe impl VaArgSafe for usize {}
 
+// Implement `VaArgSafe` for 128-bit integers on targets where clang provides `__int128`.
+//
+// GCC does not implement `__int128` for any 16-bit/32-bit target:
+//
+// https://gcc.gnu.org/onlinedocs/gcc-15.2.0/gcc/_005f_005fint128.html
+//
+// > There is no support in GCC for expressing an integer constant of type __int128 for targets
+// > with long long integer less than 128 bits wide.
+//
+// Per https://learn.microsoft.com/en-us/cpp/cpp/data-type-ranges?view=msvc-170, MSVC does not
+// define `__int128`.
+//
+// Clang is slightly more permissive: it defines `__int128` on wasm32 (a 32-bit target) and also
+// does provide `__int128` on 64-bit `*-pc-windows-msvc`, and we follow suit.
+cfg_select! {
+    any(
+        target_arch = "aarch64",
+        target_arch = "amdgpu",
+        target_arch = "arm64ec",
+        target_arch = "bpf",
+        target_arch = "loongarch64",
+        target_arch = "mips64",
+        target_arch = "mips64r6",
+        target_arch = "nvptx64",
+        target_arch = "powerpc64",
+        target_arch = "riscv64",
+        target_arch = "s390x",
+        target_arch = "sparc64",
+        target_arch = "wasm32",
+        target_arch = "wasm64",
+        target_arch = "x86_64",
+    ) => {
+        #[cfg(not(any(target_arch = "wasm32", target_abi = "x32", target_pointer_width = "64")))]
+        compile_error!("unexpected target architecture for 128-bit c-variadic");
+
+        #[unstable_feature_bound(c_variadic_int128)]
+        #[unstable(feature = "c_variadic_int128", issue = "155752")]
+        unsafe impl VaArgSafe for i128 {}
+        #[unstable_feature_bound(c_variadic_int128)]
+        #[unstable(feature = "c_variadic_int128", issue = "155752")]
+        unsafe impl VaArgSafe for u128 {}
+    }
+    _ => {}
+}
+
 unsafe impl VaArgSafe for f64 {}
 
 unsafe impl<T> VaArgSafe for *mut T {}
diff --git a/tests/assembly-llvm/c-variadic/sparc.rs b/tests/assembly-llvm/c-variadic/sparc.rs
index 6ce25119a135c..81e3313c244cc 100644
--- a/tests/assembly-llvm/c-variadic/sparc.rs
+++ b/tests/assembly-llvm/c-variadic/sparc.rs
@@ -21,6 +21,8 @@ pub unsafe trait VaArgSafe {}
 
 unsafe impl VaArgSafe for i32 {}
 unsafe impl VaArgSafe for i64 {}
+#[cfg(target_pointer_width = "64")]
+unsafe impl VaArgSafe for i128 {}
 unsafe impl VaArgSafe for f64 {}
 unsafe impl<T> VaArgSafe for *const T {}
 
@@ -105,6 +107,22 @@ unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
     va_arg(ap)
 }
 
+#[unsafe(no_mangle)]
+#[cfg(target_pointer_width = "64")]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // SPARC64-LABEL: read_i128
+    // SPARC64: ldx [%o0], %o1
+    // SPARC64-NEXT: add %o1, 15, %o1
+    // SPARC64-NEXT: and %o1, -16, %o1
+    // SPARC64-NEXT: add %o1, 16, %o2
+    // SPARC64-NEXT: stx %o2, [%o0]
+    // SPARC64-NEXT: ldx [%o1], %o0
+    // SPARC64-NEXT: or %o1, 8, %o1
+    // SPARC64-NEXT: retl
+    // SPARC64-NEXT: ldx [%o1], %o1
+    va_arg(ap)
+}
+
 #[unsafe(no_mangle)]
 unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
     // SPARC: read_ptr = read_i32
diff --git a/tests/run-make/c-link-to-rust-va-list-fn/checkrust.rs b/tests/run-make/c-link-to-rust-va-list-fn/checkrust.rs
index f3d82474e94aa..9fc7523ecc7f3 100644
--- a/tests/run-make/c-link-to-rust-va-list-fn/checkrust.rs
+++ b/tests/run-make/c-link-to-rust-va-list-fn/checkrust.rs
@@ -1,5 +1,5 @@
 #![crate_type = "staticlib"]
-#![feature(c_variadic)]
+#![feature(c_variadic, c_variadic_int128)]
 
 use core::ffi::{CStr, VaList, c_char, c_double, c_int, c_long, c_longlong};
 
@@ -57,6 +57,49 @@ pub unsafe extern "C" fn check_list_copy_0(mut ap: VaList) -> usize {
     if compare_c_str(ap.next_arg::<*const c_char>(), c"Correct") { 0 } else { 0xff }
 }
 
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn check_list_i128(mut ap: VaList) -> usize {
+    cfg_select! {
+        any(
+            target_arch = "aarch64",
+            target_arch = "amdgpu",
+            target_arch = "arm64ec",
+            target_arch = "bpf",
+            target_arch = "loongarch64",
+            target_arch = "mips64",
+            target_arch = "mips64r6",
+            target_arch = "nvptx64",
+            target_arch = "powerpc64",
+            target_arch = "riscv64",
+            target_arch = "s390x",
+            target_arch = "sparc64",
+            target_arch = "wasm32",
+            target_arch = "wasm64",
+            target_arch = "x86_64",
+        ) => {
+            #[cfg(not(any(
+                target_arch = "wasm32",
+                target_abi = "x32",
+                target_pointer_width = "64",
+            )))]
+            compile_error!("unexpected target architecture for 128-bit c-variadic");
+
+            continue_if!(ap.next_arg::<i128>() == -42);
+            // use a 32-bit value here to test the alignment logic.
+            continue_if!(ap.next_arg::<c_int>() == 0xAAAA_AAAAu32.cast_signed());
+            continue_if!(ap.next_arg::<u128>() == u128::MAX);
+
+            return 0;
+        }
+        _ => {
+            // This function was called a platform where rustc does not implement
+            // VaArgSafe for i128 but clang does define __int128. Rustc should add
+            // the implementation if this comes up.
+            0xFF
+        }
+    }
+}
+
 #[unsafe(no_mangle)]
 pub unsafe extern "C" fn check_varargs_0(_: c_int, mut ap: ...) -> usize {
     continue_if!(ap.next_arg::<c_int>() == 42);
diff --git a/tests/run-make/c-link-to-rust-va-list-fn/test.c b/tests/run-make/c-link-to-rust-va-list-fn/test.c
index b368302326c71..c7510a29445a5 100644
--- a/tests/run-make/c-link-to-rust-va-list-fn/test.c
+++ b/tests/run-make/c-link-to-rust-va-list-fn/test.c
@@ -8,6 +8,7 @@ extern size_t check_list_0(va_list ap);
 extern size_t check_list_1(va_list ap);
 extern size_t check_list_2(va_list ap);
 extern size_t check_list_copy_0(va_list ap);
+extern size_t check_list_i128(va_list ap);
 extern size_t check_varargs_0(int fixed, ...);
 extern size_t check_varargs_1(int fixed, ...);
 extern size_t check_varargs_2(int fixed, ...);
@@ -38,6 +39,10 @@ int main(int argc, char* argv[]) {
 
     assert(test_rust(check_list_copy_0, 6.28, 16, 'A', "Skip Me!", "Correct") == 0);
 
+#if defined(__SIZEOF_INT128__)
+    assert(test_rust(check_list_i128, (__int128)-42, 0xAAAAAAAA, (unsigned __int128)-1) == 0);
+#endif
+
     assert(check_varargs_0(0, 42, "Hello, World!") == 0);
 
     assert(check_varargs_1(0, 3.14, 12l, 'A', 0x1LL) == 0);
diff --git a/tests/ui/c-variadic/roundtrip.rs b/tests/ui/c-variadic/roundtrip.rs
index 22a545fec5a88..ab6d61a5b8250 100644
--- a/tests/ui/c-variadic/roundtrip.rs
+++ b/tests/ui/c-variadic/roundtrip.rs
@@ -1,6 +1,13 @@
 //@ run-pass
 //@ ignore-backends: gcc
-#![feature(c_variadic, const_c_variadic, const_destruct, const_raw_ptr_comparison)]
+#![feature(
+    c_variadic,
+    const_c_variadic,
+    c_variadic_int128,
+    const_destruct,
+    const_raw_ptr_comparison
+)]
+#![allow(unused_features)] // c_variadic_int128 is only used on 64-bit targets.
 
 use std::ffi::*;
 
@@ -12,7 +19,8 @@ use std::ffi::*;
 const unsafe extern "C" fn variadic<T: VaArgSafe>(mut ap: ...) -> (T, T) {
     let x = ap.next_arg::<T>();
     // Intersperse a small type to test alignment logic. A `u32` (i.e. `c_uint`) is the smallest
-    // type that implements `VaArgSafe`: smaller types would automatically be promoted.
+    // type that implements `VaArgSafe` (except on some 16-bit targets): smaller types would
+    // automatically be promoted.
     assert!(ap.next_arg::<u32>() == 0xAAAA_AAAA);
     let y = ap.next_arg::<T>();
 
@@ -74,5 +82,38 @@ fn main() {
         static mut B: u32 = 2u32;
         roundtrip_ptr!(*const u32, &raw const A, &raw const B);
         roundtrip_ptr!(*mut u32, &raw mut A, &raw mut B);
+
+        // The 128-bit integers only implement VaArgSafe on some targets, a subset of those that
+        // define `__int128`. We test some of those targets here.
+        cfg_select! {
+            any(
+                target_arch = "aarch64",
+                target_arch = "amdgpu",
+                target_arch = "arm64ec",
+                target_arch = "bpf",
+                target_arch = "loongarch64",
+                target_arch = "mips64",
+                target_arch = "mips64r6",
+                target_arch = "nvptx64",
+                target_arch = "powerpc64",
+                target_arch = "riscv64",
+                target_arch = "s390x",
+                target_arch = "sparc64",
+                target_arch = "wasm32",
+                target_arch = "wasm64",
+                target_arch = "x86_64",
+            ) => {
+                #[cfg(not(any(
+                    target_arch = "wasm32",
+                    target_abi = "x32",
+                    target_pointer_width = "64",
+                )))]
+                compile_error!("unexpected target architecture for 128-bit c-variadic");
+
+                roundtrip!(i128, -1, -2);
+                roundtrip!(u128, 1, 2);
+            }
+            _ => {}
+        }
     }
 }

From 92d4098ccc492c3d240202493b1dbac1977a5533 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert@folkertdev.nl>
Date: Thu, 9 Jul 2026 21:51:01 +0200
Subject: [PATCH 10/16] c-variadic: add assembly test files

---
 library/core/src/ffi/va_list.rs               |  12 +-
 tests/assembly-llvm/c-variadic/aarch64.rs     | 294 ++++++++++++++++++
 tests/assembly-llvm/c-variadic/gpu.rs         | 179 +++++++++++
 tests/assembly-llvm/c-variadic/powerpc.rs     | 198 ++++++++++++
 tests/assembly-llvm/c-variadic/riscv.rs       | 127 ++++++++
 tests/assembly-llvm/c-variadic/s390x.rs       | 133 ++++++++
 tests/assembly-llvm/c-variadic/wasm.rs        | 224 +++++++++++++
 tests/assembly-llvm/c-variadic/x86-linux.rs   | 237 ++++++++++++++
 .../c-variadic/x86_64-windows.rs              | 100 ++++++
 9 files changed, 1503 insertions(+), 1 deletion(-)
 create mode 100644 tests/assembly-llvm/c-variadic/aarch64.rs
 create mode 100644 tests/assembly-llvm/c-variadic/gpu.rs
 create mode 100644 tests/assembly-llvm/c-variadic/powerpc.rs
 create mode 100644 tests/assembly-llvm/c-variadic/riscv.rs
 create mode 100644 tests/assembly-llvm/c-variadic/s390x.rs
 create mode 100644 tests/assembly-llvm/c-variadic/wasm.rs
 create mode 100644 tests/assembly-llvm/c-variadic/x86-linux.rs
 create mode 100644 tests/assembly-llvm/c-variadic/x86_64-windows.rs

diff --git a/library/core/src/ffi/va_list.rs b/library/core/src/ffi/va_list.rs
index 1c31e1b690d63..e27f581c04ca2 100644
--- a/library/core/src/ffi/va_list.rs
+++ b/library/core/src/ffi/va_list.rs
@@ -397,7 +397,17 @@ cfg_select! {
         #[unstable(feature = "c_variadic_int128", issue = "155752")]
         unsafe impl VaArgSafe for u128 {}
     }
-    _ => {}
+    _ => {
+        #[repr(transparent)]
+        #[derive(Clone, Copy)]
+        struct S(i32);
+
+        // When there are no actual implementations on i128, declare the c_variadic_int128 feature
+        // on a private type so that the feature is defined on all targets.
+        #[unstable(feature = "c_variadic_int128", issue = "155752")]
+        unsafe impl VaArgSafe for S {}
+
+    }
 }
 
 unsafe impl VaArgSafe for f64 {}
diff --git a/tests/assembly-llvm/c-variadic/aarch64.rs b/tests/assembly-llvm/c-variadic/aarch64.rs
new file mode 100644
index 0000000000000..eb242a1a41e21
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic/aarch64.rs
@@ -0,0 +1,294 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ revisions: AARCH64_LINUX AARCH64_DARWIN AARCH64_BE ARM64EC_MSVC
+//@ [AARCH64_LINUX] compile-flags: -Copt-level=3 --target aarch64-unknown-linux-gnu
+//@ [AARCH64_LINUX] needs-llvm-components: aarch64
+//@ [AARCH64_BE] compile-flags: -Copt-level=3 --target aarch64_be-unknown-linux-gnu
+//@ [AARCH64_BE] needs-llvm-components: aarch64
+//@ [AARCH64_DARWIN] compile-flags: -Copt-level=3 --target aarch64-apple-darwin
+//@ [AARCH64_DARWIN] needs-llvm-components: aarch64
+//@ [ARM64EC_MSVC] compile-flags: -Copt-level=3 --target arm64ec-pc-windows-msvc
+//@ [ARM64EC_MSVC] needs-llvm-components: aarch64
+#![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
+#![no_core]
+#![crate_type = "lib"]
+
+// Check that the assembly that rustc generates matches what clang emits.
+
+// For aarch64-unknown-linux-gnu LLVM canonicalizes a comparison, leading to slightly different
+// assembly.
+//
+// For aarch64-apple-darwin LLVM is able to optimize our output better, because we effectively
+// desugar va_arg early, hence we don't actually match Clang there.
+
+extern crate minicore;
+use minicore::*;
+
+#[lang = "va_arg_safe"]
+pub unsafe trait VaArgSafe {}
+
+unsafe impl VaArgSafe for i32 {}
+unsafe impl VaArgSafe for i64 {}
+unsafe impl VaArgSafe for i128 {}
+unsafe impl VaArgSafe for f64 {}
+unsafe impl<T> VaArgSafe for *const T {}
+
+#[repr(transparent)]
+struct VaListInner {
+    ptr: *const c_void,
+}
+
+#[repr(transparent)]
+#[lang = "va_list"]
+pub struct VaList<'a> {
+    inner: VaListInner,
+    _marker: PhantomData<&'a mut ()>,
+}
+
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub const unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
+    // AARCH64_LINUX-LABEL: read_f64:
+    // AARCH64_LINUX: ldrsw   x8, [x0, #28]
+    // AARCH64_LINUX-NEXT: tbz w8, #31, .LBB0_2
+    // AARCH64_LINUX-NEXT: add w9, w8, #16
+    // AARCH64_LINUX-NEXT: cmn w8, #16
+    // AARCH64_LINUX-NEXT: str w9, [x0, #28]
+    // AARCH64_LINUX-NEXT: b.ls    .LBB0_3
+    // AARCH64_LINUX-NEXT: .LBB0_2:
+    // AARCH64_LINUX-NEXT: ldr x8, [x0]
+    // AARCH64_LINUX-NEXT: ldr d0, [x8]
+    // AARCH64_LINUX-NEXT: add x9, x8, #8
+    // AARCH64_LINUX-NEXT: str x9, [x0]
+    // AARCH64_LINUX-NEXT: ret
+    // AARCH64_LINUX-NEXT: .LBB0_3
+    // AARCH64_LINUX-NEXT: ldr x9, [x0, #16]
+    // AARCH64_LINUX-NEXT: add x8, x9, x8
+    // AARCH64_LINUX-NEXT: ldr d0, [x8]
+    // AARCH64_LINUX-NEXT: ret
+
+    // AARCH64_BE-LABEL: read_f64:
+    // AARCH64_BE: ldrsw    x8, [x0, #28]
+    // AARCH64_BE-NEXT: tbz w8, #31, .LBB0_2
+    // AARCH64_BE-NEXT: add w9, w8, #16
+    // AARCH64_BE-NEXT: cmn w8, #16
+    // AARCH64_BE-NEXT: str w9, [x0, #28]
+    // AARCH64_BE-NEXT: b.ls    .LBB0_3
+    // AARCH64_BE-NEXT: .LBB0_2:
+    // AARCH64_BE-NEXT: ldr x8, [x0]
+    // AARCH64_BE-NEXT: ldr d0, [x8]
+    // AARCH64_BE-NEXT: add x9, x8, #8
+    // AARCH64_BE-NEXT: str x9, [x0]
+    // AARCH64_BE-NEXT: ret
+    // AARCH64_BE-NEXT: .LBB0_3:
+    // AARCH64_BE-NEXT: ldr x9, [x0, #16]
+    // AARCH64_BE-NEXT: add x8, x9, x8
+    // AARCH64_BE-NEXT: ldr d0, [x8, #8]!
+    // AARCH64_BE-NEXT: ret
+
+    // ARM64EC_MSVC-LABEL: read_f64 = "#read_f64"
+    // ARM64EC_MSVC: ldr x8, [x0]
+    // ARM64EC_MSVC-NEXT: ldr d0, [x8], #8
+    // ARM64EC_MSVC-NEXT: str x8, [x0]
+    // ARM64EC_MSVC-NEXT: ret
+
+    // AARCH64_DARWIN-LABEL: _read_f64:
+    // AARCH64_DARWIN: ldr x8, [x0]
+    // AARCH64_DARWIN-NEXT: ldr d0, [x8], #8
+    // AARCH64_DARWIN-NEXT: str x8, [x0]
+    // AARCH64_DARWIN-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
+    // AARCH64_LINUX-LABEL: read_i32:
+    // AARCH64_LINUX: ldrsw   x8, [x0, #24]
+    // AARCH64_LINUX-NEXT: tbz w8, #31, .LBB1_2
+    // AARCH64_LINUX-NEXT: add w9, w8, #8
+    // AARCH64_LINUX-NEXT: cmn w8, #8
+    // AARCH64_LINUX-NEXT: str w9, [x0, #24]
+    // AARCH64_LINUX-NEXT: b.ls    .LBB1_3
+    // AARCH64_LINUX-NEXT: .LBB1_2:
+    // AARCH64_LINUX-NEXT: ldr x8, [x0]
+    // AARCH64_LINUX-NEXT: add x9, x8, #8
+    // AARCH64_LINUX-NEXT: str x9, [x0]
+    // AARCH64_LINUX-NEXT: ldr w0, [x8]
+    // AARCH64_LINUX-NEXT: ret
+    // AARCH64_LINUX-NEXT: .LBB1_3
+    // AARCH64_LINUX-NEXT: ldr x9, [x0, #8]
+    // AARCH64_LINUX-NEXT: add x8, x9, x8
+    // AARCH64_LINUX-NEXT: ldr w0, [x8]
+    // AARCH64_LINUX-NEXT: ret
+
+    // AARCH64_BE-LABEL: read_i32:
+    // AARCH64_BE: ldrsw    x8, [x0, #24]
+    // AARCH64_BE-NEXT: tbz w8, #31, .LBB1_2
+    // AARCH64_BE-NEXT: add w9, w8, #8
+    // AARCH64_BE-NEXT: cmn w8, #8
+    // AARCH64_BE-NEXT: str w9, [x0, #24]
+    // AARCH64_BE-NEXT: b.ls    .LBB1_3
+    // AARCH64_BE-NEXT: .LBB1_2:
+    // AARCH64_BE-NEXT: ldr x8, [x0]
+    // AARCH64_BE-NEXT: add x9, x8, #8
+    // AARCH64_BE-NEXT: str x9, [x0]
+    // AARCH64_BE-NEXT: ldr w0, [x8]
+    // AARCH64_BE-NEXT: ret
+    // AARCH64_BE-NEXT: .LBB1_3:
+    // AARCH64_BE-NEXT: ldr x9, [x0, #8]
+    // AARCH64_BE-NEXT: add x8, x9, x8
+    // AARCH64_BE-NEXT: ldr w0, [x8, #4]!
+    // AARCH64_BE-NEXT: ret
+
+    // ARM64EC_MSVC-LABEL: read_i32 = "#read_i32"
+    // ARM64EC_MSVC: ldr x9, [x0]
+    // ARM64EC_MSVC-NEXT: mov x8, x0
+    // ARM64EC_MSVC-NEXT: ldr w0, [x9], #8
+    // ARM64EC_MSVC-NEXT: str x9, [x8]
+    // ARM64EC_MSVC-NEXT: ret
+
+    // AARCH64_DARWIN-LABEL: _read_i32:
+    // AARCH64_DARWIN: ldr x9, [x0]
+    // AARCH64_DARWIN-NEXT: ldr w8, [x9], #8
+    // AARCH64_DARWIN-NEXT: str x9, [x0]
+    // AARCH64_DARWIN-NEXT: mov x0, x8
+    // AARCH64_DARWIN-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
+    // AARCH64_LINUX-LABEL: read_i64:
+    // AARCH64_LINUX: ldrsw   x8, [x0, #24]
+    // AARCH64_LINUX-NEXT: tbz w8, #31, .LBB2_2
+    // AARCH64_LINUX-NEXT: add w9, w8, #8
+    // AARCH64_LINUX-NEXT: cmn w8, #8
+    // AARCH64_LINUX-NEXT: str w9, [x0, #24]
+    // AARCH64_LINUX-NEXT: b.ls    .LBB2_3
+    // AARCH64_LINUX-NEXT: .LBB2_2:
+    // AARCH64_LINUX-NEXT: ldr x8, [x0]
+    // AARCH64_LINUX-NEXT: add x9, x8, #8
+    // AARCH64_LINUX-NEXT: str x9, [x0]
+    // AARCH64_LINUX-NEXT: ldr x0, [x8]
+    // AARCH64_LINUX-NEXT: ret
+    // AARCH64_LINUX-NEXT: .LBB2_3
+    // AARCH64_LINUX-NEXT: ldr x9, [x0, #8]
+    // AARCH64_LINUX-NEXT: add x8, x9, x8
+    // AARCH64_LINUX-NEXT: ldr x0, [x8]
+    // AARCH64_LINUX-NEXT: ret
+
+    // AARCH64_BE-LABEL: read_i64:
+    // AARCH64_BE: ldrsw    x8, [x0, #24]
+    // AARCH64_BE-NEXT: tbz w8, #31, .LBB2_2
+    // AARCH64_BE-NEXT: add w9, w8, #8
+    // AARCH64_BE-NEXT: cmn w8, #8
+    // AARCH64_BE-NEXT: str w9, [x0, #24]
+    // AARCH64_BE-NEXT: b.ls    .LBB2_3
+    // AARCH64_BE-NEXT: .LBB2_2:
+    // AARCH64_BE-NEXT: ldr x8, [x0]
+    // AARCH64_BE-NEXT: add x9, x8, #8
+    // AARCH64_BE-NEXT: str x9, [x0]
+    // AARCH64_BE-NEXT: ldr x0, [x8]
+    // AARCH64_BE-NEXT: ret
+    // AARCH64_BE-NEXT: .LBB2_3:
+    // AARCH64_BE-NEXT: ldr x9, [x0, #8]
+    // AARCH64_BE-NEXT: add x8, x9, x8
+    // AARCH64_BE-NEXT: ldr x0, [x8]
+    // AARCH64_BE-NEXT: ret
+
+    // ARM64EC_MSVC-LABEL: read_ptr = "#read_ptr"
+    // ARM64EC_MSVC-LABEL: read_i64 = "#read_i64"
+    // ARM64EC_MSVC: ldr x9, [x0]
+    // ARM64EC_MSVC-NEXT: mov x8, x0
+    // ARM64EC_MSVC-NEXT: ldr x0, [x9], #8
+    // ARM64EC_MSVC-NEXT: str x9, [x8]
+    // ARM64EC_MSVC-NEXT: ret
+
+    // AARCH64_DARWIN-LABEL: _read_i64:
+    // AARCH64_DARWIN: ldr x9, [x0]
+    // AARCH64_DARWIN-NEXT: ldr x8, [x9], #8
+    // AARCH64_DARWIN-NEXT: str x9, [x0]
+    // AARCH64_DARWIN-NEXT: mov x0, x8
+    // AARCH64_DARWIN-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // AARCH64_LINUX-LABEL: read_i128:
+    // AARCH64_LINUX: ldrsw   x8, [x0, #24]
+    // AARCH64_LINUX-NEXT: tbz w8, #31, .LBB3_2
+    // AARCH64_LINUX-NEXT: add x8, x8, #15
+    // AARCH64_LINUX-NEXT: and x8, x8, #0xfffffffffffffff0
+    // AARCH64_LINUX-NEXT: add w9, w8, #16
+    // AARCH64_LINUX-NEXT: cmp w9, #0
+    // AARCH64_LINUX-NEXT: str w9, [x0, #24]
+    // AARCH64_LINUX-NEXT: b.le    .LBB3_3
+    // AARCH64_LINUX-NEXT: .LBB3_2:
+    // AARCH64_LINUX-NEXT: ldr x8, [x0]
+    // AARCH64_LINUX-NEXT: add x8, x8, #15
+    // AARCH64_LINUX-NEXT: and x8, x8, #0xfffffffffffffff0
+    // AARCH64_LINUX-NEXT: add x9, x8, #16
+    // AARCH64_LINUX-NEXT: str x9, [x0]
+    // AARCH64_LINUX-NEXT: ldp x0, x1, [x8]
+    // AARCH64_LINUX-NEXT: ret
+    // AARCH64_LINUX-NEXT: .LBB3_3
+    // AARCH64_LINUX-NEXT: ldr x9, [x0, #8]
+    // AARCH64_LINUX-NEXT: add x8, x9, x8
+    // AARCH64_LINUX-NEXT: ldp x0, x1, [x8]
+    // AARCH64_LINUX-NEXT: ret
+
+    // AARCH64_BE-LABEL: read_i128:
+    // AARCH64_BE: ldrsw   x8, [x0, #24]
+    // AARCH64_BE-NEXT: tbz     w8, #31, .LBB3_2
+    // AARCH64_BE-NEXT: add     x8, x8, #15
+    // AARCH64_BE-NEXT: and     x8, x8, #0xfffffffffffffff0
+    // AARCH64_BE-NEXT: add     w9, w8, #16
+    // AARCH64_BE-NEXT: cmp     w9, #0
+    // AARCH64_BE-NEXT: str     w9, [x0, #24]
+    // AARCH64_BE-NEXT: b.le    .LBB3_3
+    // AARCH64_BE-NEXT: .LBB3_2:
+    // AARCH64_BE-NEXT: ldr     x8, [x0]
+    // AARCH64_BE-NEXT: add     x8, x8, #15
+    // AARCH64_BE-NEXT: and     x8, x8, #0xfffffffffffffff0
+    // AARCH64_BE-NEXT: add     x9, x8, #16
+    // AARCH64_BE-NEXT: str     x9, [x0]
+    // AARCH64_BE-NEXT: ldp     x0, x1, [x8]
+    // AARCH64_BE-NEXT: ret
+    // AARCH64_BE-NEXT: .LBB3_3:
+    // AARCH64_BE-NEXT: ldr     x9, [x0, #8]
+    // AARCH64_BE-NEXT: add     x8, x9, x8
+    // AARCH64_BE-NEXT: ldp     x0, x1, [x8]
+    // AARCH64_BE-NEXT: ret
+
+    // ARM64EC_MSVC-LABEL: read_i128 = "#read_i128"
+    // ARM64EC_MSVC: ldr x9, [x0]
+    // ARM64EC_MSVC-NEXT: mov x8, x0
+    // ARM64EC_MSVC-NEXT: ldp x0, x1, [x9], #16
+    // ARM64EC_MSVC-NEXT: str x9, [x8]
+    // ARM64EC_MSVC-NEXT: ret
+
+    // AARCH64_DARWIN-LABEL: _read_i128:
+    // AARCH64_DARWIN: ldr x8, [x0]
+    // AARCH64_DARWIN-NEXT: add x8, x8, #15
+    // AARCH64_DARWIN-NEXT: and x9, x8, #0xfffffffffffffff0
+    // AARCH64_DARWIN-NEXT: ldr x1, [x9, #8]
+    // AARCH64_DARWIN-NEXT: ldr x8, [x9], #16
+    // AARCH64_DARWIN-NEXT: str x9, [x0]
+    // AARCH64_DARWIN-NEXT: mov x0, x8
+    // AARCH64_DARWIN-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
+    // AARCH64_LINUX-CHECK: read_ptr = read_i64
+    // AARCH64_BE-CHECK: read_ptr = read_i64
+    // ARM64EC_MSVC: "#read_ptr" = "#read_i64"
+    // AARCH64_DARWIN-CHECK: _read_ptr = _read_i64
+    va_arg(ap)
+}
diff --git a/tests/assembly-llvm/c-variadic/gpu.rs b/tests/assembly-llvm/c-variadic/gpu.rs
new file mode 100644
index 0000000000000..61dc4aa52c675
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic/gpu.rs
@@ -0,0 +1,179 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//@ compile-flags: -Copt-level=3
+//
+//@ revisions: AMDGPU NVPTX
+//@ [AMDGPU] compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900
+//@ [AMDGPU] needs-llvm-components: amdgpu
+//@ [NVPTX] compile-flags: --crate-type=rlib --target=nvptx64-nvidia-cuda
+//@ [NVPTX] needs-llvm-components: nvptx
+#![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+#[lang = "va_arg_safe"]
+pub unsafe trait VaArgSafe {}
+
+unsafe impl VaArgSafe for i32 {}
+unsafe impl VaArgSafe for i64 {}
+unsafe impl VaArgSafe for i128 {}
+unsafe impl VaArgSafe for f64 {}
+unsafe impl<T> VaArgSafe for *const T {}
+
+#[repr(transparent)]
+struct VaListInner {
+    ptr: *const c_void,
+}
+
+#[repr(transparent)]
+#[lang = "va_list"]
+pub struct VaList<'a> {
+    inner: VaListInner,
+    _marker: PhantomData<&'a mut ()>,
+}
+
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub const unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
+    // CHECK-LABEL: read_f64
+    //
+    // AMDGPU: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+    // AMDGPU-NEXT: v_add_co_u32_e32 v4, vcc, 8, v4
+    // AMDGPU-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+    // AMDGPU-NEXT: flat_store_dwordx2 v[0:1], v[4:5]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: v_mov_b32_e32 v0, v2
+    // AMDGPU-NEXT: v_mov_b32_e32 v1, v3
+    // AMDGPU-NEXT: s_setpc_b64 s[30:31]
+    //
+    // NVPTX: ld.param.b64 %rd1, [read_f64_param_0];
+    // NVPTX-NEXT: ld.b64 %rd2, [%rd1];
+    // NVPTX-NEXT: add.s64 %rd3, %rd2, 7;
+    // NVPTX-NEXT: and.b64 %rd4, %rd3, -8;
+    // NVPTX-NEXT: add.s64 %rd5, %rd4, 8;
+    // NVPTX-NEXT: st.b64 [%rd1], %rd5;
+    // NVPTX-NEXT: ld.b64 %rd6, [%rd4];
+    // NVPTX-NEXT: st.param.b64 [func_retval0], %rd6;
+    // NVPTX-NEXT: ret;
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
+    // CHECK-LABEL: read_i32
+    //
+    // AMDGPU: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dwordx2 v[3:4], v[0:1]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dword v2, v[3:4]
+    // AMDGPU-NEXT: v_add_co_u32_e32 v3, vcc, 4, v3
+    // AMDGPU-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+    // AMDGPU-NEXT: flat_store_dwordx2 v[0:1], v[3:4]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: v_mov_b32_e32 v0, v2
+    // AMDGPU-NEXT: s_setpc_b64 s[30:31]
+    //
+    // NVPTX: ld.param.b64 %rd1, [read_i32_param_0];
+    // NVPTX-NEXT: ld.b64 %rd2, [%rd1];
+    // NVPTX-NEXT: add.s64 %rd3, %rd2, 3;
+    // NVPTX-NEXT: and.b64 %rd4, %rd3, -4;
+    // NVPTX-NEXT: add.s64 %rd5, %rd4, 4;
+    // NVPTX-NEXT: st.b64 [%rd1], %rd5;
+    // NVPTX-NEXT: ld.b32 %r1, [%rd4];
+    // NVPTX-NEXT: st.param.b32 [func_retval0], %r1;
+    // NVPTX-NEXT: ret;
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
+    // CHECK-LABEL: read_i64
+    //
+    // AMDGPU: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+    // AMDGPU-NEXT: v_add_co_u32_e32 v4, vcc, 8, v4
+    // AMDGPU-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+    // AMDGPU-NEXT: flat_store_dwordx2 v[0:1], v[4:5]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: v_mov_b32_e32 v0, v2
+    // AMDGPU-NEXT: v_mov_b32_e32 v1, v3
+    // AMDGPU-NEXT: s_setpc_b64 s[30:31]
+    //
+    // NVPTX: ld.param.b64 %rd1, [read_i64_param_0];
+    // NVPTX-NEXT: ld.b64 %rd2, [%rd1];
+    // NVPTX-NEXT: add.s64 %rd3, %rd2, 7;
+    // NVPTX-NEXT: and.b64 %rd4, %rd3, -8;
+    // NVPTX-NEXT: add.s64 %rd5, %rd4, 8;
+    // NVPTX-NEXT: st.b64 [%rd1], %rd5;
+    // NVPTX-NEXT: ld.b64 %rd6, [%rd4];
+    // NVPTX-NEXT: st.param.b64 [func_retval0], %rd6;
+    // NVPTX-NEXT: ret;
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // CHECK-LABEL: read_i128
+    //
+    // AMDGPU: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: v_mov_b32_e32 v5, v1
+    // AMDGPU-NEXT: v_mov_b32_e32 v4, v0
+    // AMDGPU-NEXT: flat_load_dwordx2 v[6:7], v[4:5]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dwordx4 v[0:3], v[6:7]
+    // AMDGPU-NEXT: v_add_co_u32_e32 v6, vcc, 16, v6
+    // AMDGPU-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+    // AMDGPU-NEXT: flat_store_dwordx2 v[4:5], v[6:7]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: s_setpc_b64 s[30:31]
+    //
+    // NVPTX: ld.param.b64 %rd1, [read_i128_param_0];
+    // NVPTX-NEXT: ld.b64 %rd2, [%rd1];
+    // NVPTX-NEXT: add.s64 %rd3, %rd2, 15;
+    // NVPTX-NEXT: and.b64 %rd4, %rd3, -16;
+    // NVPTX-NEXT: add.s64 %rd5, %rd4, 16;
+    // NVPTX-NEXT: st.b64 [%rd1], %rd5;
+    // NVPTX-NEXT: ld.v2.b64 {%rd6, %rd7}, [%rd4];
+    // NVPTX-NEXT: st.param.v2.b64 [func_retval0], {%rd6, %rd7};
+    // NVPTX-NEXT: ret;
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
+    // CHECK-LABEL: read_ptr
+    //
+    // AMDGPU: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+    // AMDGPU-NEXT: v_add_co_u32_e32 v4, vcc, 8, v4
+    // AMDGPU-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+    // AMDGPU-NEXT: flat_store_dwordx2 v[0:1], v[4:5]
+    // AMDGPU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+    // AMDGPU-NEXT: v_mov_b32_e32 v0, v2
+    // AMDGPU-NEXT: v_mov_b32_e32 v1, v3
+    // AMDGPU-NEXT: s_setpc_b64 s[30:31]
+    //
+    // NVPTX: ld.param.b64 %rd1, [read_ptr_param_0];
+    // NVPTX-NEXT: ld.b64 %rd2, [%rd1];
+    // NVPTX-NEXT: add.s64 %rd3, %rd2, 7;
+    // NVPTX-NEXT: and.b64 %rd4, %rd3, -8;
+    // NVPTX-NEXT: add.s64 %rd5, %rd4, 8;
+    // NVPTX-NEXT: st.b64 [%rd1], %rd5;
+    // NVPTX-NEXT: ld.b64 %rd6, [%rd4];
+    // NVPTX-NEXT: st.param.b64 [func_retval0], %rd6;
+    // NVPTX-NEXT: ret;
+    va_arg(ap)
+}
diff --git a/tests/assembly-llvm/c-variadic/powerpc.rs b/tests/assembly-llvm/c-variadic/powerpc.rs
new file mode 100644
index 0000000000000..47d35e7579737
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic/powerpc.rs
@@ -0,0 +1,198 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ revisions: POWERPC POWERPC64 POWERPC64LE AIX
+//@ [POWERPC] compile-flags: -Copt-level=3 --target powerpc-unknown-linux-gnu
+//@ [POWERPC] needs-llvm-components: powerpc
+//@ [POWERPC64] compile-flags: -Copt-level=3 --target powerpc64-unknown-linux-gnu
+//@ [POWERPC64] needs-llvm-components: powerpc
+//@ [POWERPC64LE] compile-flags: -Copt-level=3 --target powerpc64le-unknown-linux-gnu
+//@ [POWERPC64LE] needs-llvm-components: powerpc
+//@ [AIX] compile-flags: -Copt-level=3 --target powerpc64-ibm-aix
+//@ [AIX] needs-llvm-components: powerpc
+#![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
+#![no_core]
+#![crate_type = "lib"]
+
+// Check that the assembly that rustc generates matches what clang emits.
+
+extern crate minicore;
+use minicore::*;
+
+#[lang = "va_arg_safe"]
+pub unsafe trait VaArgSafe {}
+
+unsafe impl VaArgSafe for i32 {}
+unsafe impl VaArgSafe for i64 {}
+#[cfg(target_pointer_width = "64")]
+unsafe impl VaArgSafe for i128 {}
+unsafe impl VaArgSafe for f64 {}
+unsafe impl<T> VaArgSafe for *const T {}
+
+#[repr(transparent)]
+struct VaListInner {
+    ptr: *const c_void,
+}
+
+#[repr(transparent)]
+#[lang = "va_list"]
+pub struct VaList<'a> {
+    inner: VaListInner,
+    _marker: PhantomData<&'a mut ()>,
+}
+
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub const unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
+    // CHECK-LABEL: read_f64
+    //
+    // POWERPC: lbz 5, 1(3)
+    // POWERPC-NEXT: cmplwi 5, 7
+    // POWERPC-NEXT: bgt 0, .LBB0_2
+    // POWERPC-NEXT: lwz 4, 8(3)
+    // POWERPC-NEXT: rlwinm 6, 5, 3, 24, 28
+    // POWERPC-NEXT: addi 5, 5, 1
+    // POWERPC-NEXT: add 4, 4, 6
+    // POWERPC-NEXT: addi 4, 4, 32
+    // POWERPC-NEXT: lfd 1, 0(4)
+    // POWERPC-NEXT: stb 5, 1(3)
+    // POWERPC-NEXT: blr
+    //
+    // POWERPC64: ld 4, 0(3)
+    // POWERPC64-NEXT: lfd 1, 0(4)
+    // POWERPC64-NEXT: addi 4, 4, 8
+    // POWERPC64-NEXT: std 4, 0(3)
+    // POWERPC64-NEXT: blr
+    //
+    // POWERPC64LE: ld 4, 0(3)
+    // POWERPC64LE-NEXT: lfd 1, 0(4)
+    // POWERPC64LE-NEXT: addi 5, 4, 8
+    // POWERPC64LE-NEXT: std 5, 0(3)
+    // POWERPC64LE-NEXT: blr
+    //
+    // AIX: ld 4, 0(3)
+    // AIX-NEXT: lfd 1, 0(4)
+    // AIX-NEXT: addi 5, 4, 8
+    // AIX-NEXT: std 5, 0(3)
+    // AIX-NEXT: blr
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
+    // CHECK-LABEL: read_i32
+    //
+    // POWERPC: lbz 5, 0(3)
+    // POWERPC-NEXT: mr 4, 3
+    // POWERPC-NEXT: cmplwi 5, 7
+    // POWERPC-NEXT: bgt 0, .LBB1_2
+    // POWERPC-NEXT: lwz 3, 8(4)
+    // POWERPC-NEXT: rlwinm 6, 5, 2, 24, 29
+    // POWERPC-NEXT: addi 5, 5, 1
+    // POWERPC-NEXT: add 3, 3, 6
+    // POWERPC-NEXT: lwz 3, 0(3)
+    // POWERPC-NEXT: stb 5, 0(4)
+    // POWERPC-NEXT: blr
+    //
+    // POWERPC64: ld 5, 0(3)
+    // POWERPC64-NEXT: mr 4, 3
+    // POWERPC64-NEXT: lwa 3, 4(5)
+    // POWERPC64-NEXT: addi 5, 5, 8
+    // POWERPC64-NEXT: std 5, 0(4)
+    // POWERPC64-NEXT: blr
+    //
+    // POWERPC64LE: ld 4, 0(3)
+    // POWERPC64LE-NEXT: addi 5, 4, 8
+    // POWERPC64LE-NEXT: std 5, 0(3)
+    // POWERPC64LE-NEXT: lwa 3, 0(4)
+    // POWERPC64LE-NEXT: blr
+    //
+    // AIX: ld 4, 0(3)
+    // AIX-NEXT: addi 5, 4, 8
+    // AIX-NEXT: std 5, 0(3)
+    // AIX-NEXT: lwa 3, 4(4)
+    // AIX-NEXT: blr
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
+    // CHECK-LABEL: read_i64
+    //
+    // POWERPC: mr 5, 3
+    // POWERPC-NEXT: lbz 3, 0(3)
+    // POWERPC-NEXT: addi 3, 3, 1
+    // POWERPC-NEXT: clrlwi 4, 3, 24
+    // POWERPC-NEXT: cmplwi 4, 7
+    // POWERPC-NEXT: bgt 0, .LBB2_2
+    // POWERPC-NEXT: lwz 4, 8(5)
+    // POWERPC-NEXT: rlwinm 6, 3, 0, 29, 30
+    // POWERPC-NEXT: rlwinm 3, 3, 2, 27, 28
+    // POWERPC-NEXT: addi 6, 6, 2
+    // POWERPC-NEXT: add 4, 4, 3
+    // POWERPC-NEXT: lwz 3, 0(4)
+    // POWERPC-NEXT: lwz 4, 4(4)
+    // POWERPC-NEXT: stb 6, 0(5)
+    // POWERPC-NEXT: blr
+    //
+    // POWERPC64: ld 5, 0(3)
+    // POWERPC64-NEXT: mr 4, 3
+    // POWERPC64-NEXT: ld 3, 0(5)
+    // POWERPC64-NEXT: addi 5, 5, 8
+    // POWERPC64-NEXT: std 5, 0(4)
+    // POWERPC64-NEXT: blr
+    //
+    // POWERPC64LE: ld 4, 0(3)
+    // POWERPC64LE-NEXT: addi 5, 4, 8
+    // POWERPC64LE-NEXT: std 5, 0(3)
+    // POWERPC64LE-NEXT: ld 3, 0(4)
+    // POWERPC64LE-NEXT: blr
+    //
+    // AIX: ld 4, 0(3)
+    // AIX-NEXT: addi 5, 4, 8
+    // AIX-NEXT: std 5, 0(3)
+    // AIX-NEXT: ld 3, 0(4)
+    // AIX-NEXT: blr
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+#[cfg(target_pointer_width = "64")]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // POWERPC64-LABEL: read_i128
+    // POWERPC64: ld 6, 0(3)
+    // POWERPC64-NEXT: mr  5, 3
+    // POWERPC64-NEXT: ld 3, 0(6)
+    // POWERPC64-NEXT: ld 4, 8(6)
+    // POWERPC64-NEXT: addi 6, 6, 16
+    // POWERPC64-NEXT: std 6, 0(5)
+    // POWERPC64-NEXT: blr
+    //
+    // POWERPC64LE-LABEL: read_i128
+    // POWERPC64LE: ld 4, 0(3)
+    // POWERPC64LE-NEXT: addi 5, 4, 16
+    // POWERPC64LE-NEXT: std 5, 0(3)
+    // POWERPC64LE-NEXT: ld 3, 0(4)
+    // POWERPC64LE-NEXT: ld 4, 8(4)
+    // POWERPC64LE-NEXT: blr
+    //
+    // AIX-LABEL: read_i128
+    // AIX: ld 4, 0(3)
+    // AIX-NEXT: addi 5, 4, 16
+    // AIX-NEXT: std 5, 0(3)
+    // AIX-NEXT: ld 3, 0(4)
+    // AIX-NEXT: ld 4, 8(4)
+    // AIX-NEXT: blr
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
+    // POWERPC: read_ptr = read_i32
+    // POWERPC64: read_ptr = read_i64
+    // POWERPC64LE: read_ptr = read_i64
+    va_arg(ap)
+}
diff --git a/tests/assembly-llvm/c-variadic/riscv.rs b/tests/assembly-llvm/c-variadic/riscv.rs
new file mode 100644
index 0000000000000..893d2a65eb37d
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic/riscv.rs
@@ -0,0 +1,127 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ revisions: RISCV32 RISCV64
+//@ [RISCV32] compile-flags: -Copt-level=3 --target riscv32gc-unknown-linux-gnu
+//@ [RISCV32] needs-llvm-components: riscv
+//@ [RISCV64] compile-flags: -Copt-level=3 --target riscv64gc-unknown-linux-gnu
+//@ [RISCV64] needs-llvm-components: riscv
+#![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+#[lang = "va_arg_safe"]
+pub unsafe trait VaArgSafe {}
+
+unsafe impl VaArgSafe for i32 {}
+unsafe impl VaArgSafe for i64 {}
+#[cfg(target_pointer_width = "64")]
+unsafe impl VaArgSafe for i128 {}
+unsafe impl VaArgSafe for f64 {}
+unsafe impl<T> VaArgSafe for *const T {}
+
+#[repr(transparent)]
+struct VaListInner {
+    ptr: *const c_void,
+}
+
+#[repr(transparent)]
+#[lang = "va_list"]
+pub struct VaList<'a> {
+    inner: VaListInner,
+    _marker: PhantomData<&'a mut ()>,
+}
+
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub const unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
+    // CHECK-LABEL: read_f64
+    //
+    // RISCV32: lw a1, 0(a0)
+    // RISCV32-NEXT: addi a1, a1, 7
+    // RISCV32-NEXT: andi a1, a1, -8
+    // RISCV32-NEXT: fld fa0, 0(a1)
+    // RISCV32-NEXT: addi a1, a1, 8
+    // RISCV32-NEXT: sw a1, 0(a0)
+    // RISCV32-NEXT: ret
+    //
+    // RISCV64: ld a1, 0(a0)
+    // RISCV64-NEXT: fld fa0, 0(a1)
+    // RISCV64-NEXT: addi a1, a1, 8
+    // RISCV64-NEXT: sd a1, 0(a0)
+    // RISCV64-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
+    // CHECK-LABEL: read_i32
+    //
+    // RISCV32: lw a2, 0(a0)
+    // RISCV32-NEXT: lw a1, 0(a2)
+    // RISCV32-NEXT: addi a2, a2, 4
+    // RISCV32-NEXT: sw a2, 0(a0)
+    // RISCV32-NEXT: mv a0, a1
+    // RISCV32-NEXT: ret
+    //
+    // RISCV64: ld a2, 0(a0)
+    // RISCV64-NEXT: lw a1, 0(a2)
+    // RISCV64-NEXT: addi a2, a2, 8
+    // RISCV64-NEXT: sd a2, 0(a0)
+    // RISCV64-NEXT: mv a0, a1
+    // RISCV64-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
+    // CHECK-LABEL: read_i64
+    //
+    // RISCV32: lw a1, 0(a0)
+    // RISCV32-NEXT: addi a1, a1, 7
+    // RISCV32-NEXT: andi a3, a1, -8
+    // RISCV32-NEXT: lw a2, 0(a3)
+    // RISCV32-NEXT: lw a1, 4(a3)
+    // RISCV32-NEXT: addi a3, a3, 8
+    // RISCV32-NEXT: sw a3, 0(a0)
+    // RISCV32-NEXT: mv a0, a2
+    // RISCV32-NEXT: ret
+    //
+    // RISCV64: ld a2, 0(a0)
+    // RISCV64-NEXT: ld a1, 0(a2)
+    // RISCV64-NEXT: addi a2, a2, 8
+    // RISCV64-NEXT: sd a2, 0(a0)
+    // RISCV64-NEXT: mv a0, a1
+    // RISCV64-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+#[cfg(target_pointer_width = "64")]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // RISCV64-LABEL: read_i128
+    //
+    // RISCV64: ld a1, 0(a0)
+    // RISCV64-NEXT: addi a1, a1, 15
+    // RISCV64-NEXT: andi a3, a1, -16
+    // RISCV64-NEXT: ld a2, 0(a3)
+    // RISCV64-NEXT: ld a1, 8(a3)
+    // RISCV64-NEXT: addi a3, a3, 16
+    // RISCV64-NEXT: sd a3, 0(a0)
+    // RISCV64-NEXT: mv a0, a2
+    // RISCV64-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
+    // RISCV32: read_ptr = read_i32
+    // RISCV64: read_ptr = read_i64
+    va_arg(ap)
+}
diff --git a/tests/assembly-llvm/c-variadic/s390x.rs b/tests/assembly-llvm/c-variadic/s390x.rs
new file mode 100644
index 0000000000000..fef239273e2c5
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic/s390x.rs
@@ -0,0 +1,133 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ compile-flags: -Copt-level=3 --target s390x-unknown-linux-gnu
+//@ needs-llvm-components: systemz
+#![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
+#![no_core]
+#![crate_type = "lib"]
+
+// Check that the assembly that rustc generates matches what clang emits.
+
+extern crate minicore;
+use minicore::*;
+
+#[lang = "va_arg_safe"]
+pub unsafe trait VaArgSafe {}
+
+unsafe impl VaArgSafe for i32 {}
+unsafe impl VaArgSafe for i64 {}
+unsafe impl VaArgSafe for i128 {}
+unsafe impl VaArgSafe for f64 {}
+unsafe impl<T> VaArgSafe for *const T {}
+
+#[repr(transparent)]
+struct VaListInner {
+    ptr: *const c_void,
+}
+
+#[repr(transparent)]
+#[lang = "va_list"]
+pub struct VaList<'a> {
+    inner: VaListInner,
+    _marker: PhantomData<&'a mut ()>,
+}
+
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub const unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
+    // CHECK-LABEL: read_f64:
+    // CHECK: lg %r3, 8(%r2)
+    // CHECK-NEXT: clgijh %r3, 3, .LBB0_2
+    // CHECK-NEXT: lg %r1, 24(%r2)
+    // CHECK-NEXT: sllg %r4, %r3, 3
+    // CHECK-NEXT: la %r1, 128(%r4,%r1)
+    // CHECK-NEXT: la %r0, 1(%r3)
+    // CHECK-NEXT: stg %r0, 8(%r2)
+    // CHECK-NEXT: ld %f0, 0(%r1)
+    // CHECK-NEXT: br %r14
+    // CHECK-NEXT: .LBB0_2:
+    // CHECK-NEXT: lg %r1, 16(%r2)
+    // CHECK-NEXT: la %r0, 8(%r1)
+    // CHECK-NEXT: stg %r0, 16(%r2)
+    // CHECK-NEXT: ld %f0, 0(%r1)
+    // CHECK-NEXT: br %r14
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
+    // CHECK-LABEL: read_i32:
+    // CHECK: lg %r3, 0(%r2)
+    // CHECK-NEXT: clgijh %r3, 4, .LBB1_2
+    // CHECK-NEXT: lg %r1, 24(%r2)
+    // CHECK-NEXT: sllg %r4, %r3, 3
+    // CHECK-NEXT: la %r1, 20(%r4,%r1)
+    // CHECK-NEXT: la %r0, 1(%r3)
+    // CHECK-NEXT: stg %r0, 0(%r2)
+    // CHECK-NEXT: lgf %r2, 0(%r1)
+    // CHECK-NEXT: br %r14
+    // CHECK-NEXT: .LBB1_2:
+    // CHECK-NEXT: lg %r3, 16(%r2)
+    // CHECK-NEXT: la %r1, 4(%r3)
+    // CHECK-NEXT: la %r0, 8(%r3)
+    // CHECK-NEXT: stg %r0, 16(%r2)
+    // CHECK-NEXT: lgf %r2, 0(%r1)
+    // CHECK-NEXT: br %r14
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
+    // CHECK-LABEL: read_i64:
+    // CHECK: lg %r3, 0(%r2)
+    // CHECK-NEXT: clgijh %r3, 4, .LBB2_2
+    // CHECK-NEXT: lg %r1, 24(%r2)
+    // CHECK-NEXT: sllg %r4, %r3, 3
+    // CHECK-NEXT: la %r1, 16(%r4,%r1)
+    // CHECK-NEXT: la %r0, 1(%r3)
+    // CHECK-NEXT: stg %r0, 0(%r2)
+    // CHECK-NEXT: lg %r2, 0(%r1)
+    // CHECK-NEXT: br %r14
+    // CHECK-NEXT: .LBB2_2:
+    // CHECK-NEXT: lg %r1, 16(%r2)
+    // CHECK-NEXT: la %r0, 8(%r1)
+    // CHECK-NEXT: stg %r0, 16(%r2)
+    // CHECK-NEXT: lg %r2, 0(%r1)
+    // CHECK-NEXT: br %r14
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // CHECK-LABEL: read_i128:
+    // CHECK: lg %r4, 0(%r3)
+    // CHECK-NEXT: clgijh %r4, 4, .LBB3_2
+    // CHECK-NEXT: lg %r1, 24(%r3)
+    // CHECK-NEXT: sllg %r5, %r4, 3
+    // CHECK-NEXT: la %r1, 16(%r5,%r1)
+    // CHECK-NEXT: la %r0, 1(%r4)
+    // CHECK-NEXT: stg %r0, 0(%r3)
+    // CHECK-NEXT: lg %r1, 0(%r1)
+    // CHECK-NEXT: mvc 8(8,%r2), 8(%r1)
+    // CHECK-NEXT: mvc 0(8,%r2), 0(%r1)
+    // CHECK-NEXT: br %r14
+    // CHECK-NEXT: .LBB3_2:
+    // CHECK-NEXT: lg %r1, 16(%r3)
+    // CHECK-NEXT: la %r0, 8(%r1)
+    // CHECK-NEXT: stg %r0, 16(%r3)
+    // CHECK-NEXT: lg %r1, 0(%r1)
+    // CHECK-NEXT: mvc 8(8,%r2), 8(%r1)
+    // CHECK-NEXT: mvc 0(8,%r2), 0(%r1)
+    // CHECK-NEXT: br %r14
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
+    // CHECK: read_ptr = read_i64
+    va_arg(ap)
+}
diff --git a/tests/assembly-llvm/c-variadic/wasm.rs b/tests/assembly-llvm/c-variadic/wasm.rs
new file mode 100644
index 0000000000000..8d05cdb1923b4
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic/wasm.rs
@@ -0,0 +1,224 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ revisions: WASM32 WASM64
+//@ [WASM32] compile-flags: -Copt-level=3 -Zmerge-functions=disabled --target wasm32-unknown-unknown
+//@ [WASM32] needs-llvm-components: webassembly
+//@ [WASM64] compile-flags: -Copt-level=3 -Zmerge-functions=disabled --target wasm64-unknown-unknown
+//@ [WASM64] needs-llvm-components: webassembly
+#![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
+#![no_core]
+#![crate_type = "lib"]
+
+// Check that the assembly that rustc generates matches what clang emits.
+
+extern crate minicore;
+use minicore::*;
+
+#[lang = "va_arg_safe"]
+pub unsafe trait VaArgSafe {}
+
+unsafe impl VaArgSafe for i32 {}
+unsafe impl VaArgSafe for i64 {}
+unsafe impl VaArgSafe for i128 {}
+unsafe impl VaArgSafe for f64 {}
+unsafe impl<T> VaArgSafe for *const T {}
+
+#[repr(transparent)]
+struct VaListInner {
+    ptr: *const c_void,
+}
+
+#[repr(transparent)]
+#[lang = "va_list"]
+pub struct VaList<'a> {
+    inner: VaListInner,
+    _marker: PhantomData<&'a mut ()>,
+}
+
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub const unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
+    // WASM32-LABEL: read_f64:
+    // WASM32: local.get 0
+    // WASM32-NEXT: local.get 0
+    // WASM32-NEXT: i32.load 0
+    // WASM32-NEXT: i32.const 7
+    // WASM32-NEXT: i32.add
+    // WASM32-NEXT: i32.const -8
+    // WASM32-NEXT: i32.and
+    // WASM32-NEXT: local.tee 1
+    // WASM32-NEXT: i32.const 8
+    // WASM32-NEXT: i32.add
+    // WASM32-NEXT: i32.store 0
+    // WASM32-NEXT: local.get 1
+    // WASM32-NEXT: f64.load 0
+    // WASM32-NEXT: end_function
+    //
+    // WASM64-LABEL: read_f64:
+    // WASM64: local.get 0
+    // WASM64-NEXT: local.get 0
+    // WASM64-NEXT: i64.load 0
+    // WASM64-NEXT: i64.const 7
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.const -8
+    // WASM64-NEXT: i64.and
+    // WASM64-NEXT: local.tee 1
+    // WASM64-NEXT: i64.const 8
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.store 0
+    // WASM64-NEXT: local.get 1
+    // WASM64-NEXT: f64.load 0
+    // WASM64-NEXT: end_function
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
+    // WASM32-LABEL: read_i32:
+    // WASM32: local.get 0
+    // WASM32-NEXT: local.get 0
+    // WASM32-NEXT: i32.load 0
+    // WASM32-NEXT: local.tee 1
+    // WASM32-NEXT: i32.const 4
+    // WASM32-NEXT: i32.add
+    // WASM32-NEXT: i32.store 0
+    // WASM32-NEXT: local.get 1
+    // WASM32-NEXT: i32.load 0
+    // WASM32-NEXT: end_function
+    //
+    // WASM64-LABEL: read_i32:
+    // WASM64: local.get 0
+    // WASM64-NEXT: local.get 0
+    // WASM64-NEXT: i64.load 0
+    // WASM64-NEXT: local.tee 1
+    // WASM64-NEXT: i64.const 4
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.store 0
+    // WASM64-NEXT: local.get 1
+    // WASM64-NEXT: i32.load 0
+    // WASM64-NEXT: end_function
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
+    // WASM32-LABEL: read_ptr:
+    // WASM32: local.get 0
+    // WASM32-NEXT: local.get 0
+    // WASM32-NEXT: i32.load 0
+    // WASM32-NEXT: local.tee 1
+    // WASM32-NEXT: i32.const 4
+    // WASM32-NEXT: i32.add
+    // WASM32-NEXT: i32.store 0
+    // WASM32-NEXT: local.get 1
+    // WASM32-NEXT: i32.load 0
+    // WASM32-NEXT: end_function
+    //
+    // WASM64-LABEL: read_ptr:
+    // WASM64: local.get 0
+    // WASM64-NEXT: local.get 0
+    // WASM64-NEXT: i64.load 0
+    // WASM64-NEXT: i64.const 7
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.const -8
+    // WASM64-NEXT: i64.and
+    // WASM64-NEXT: local.tee 1
+    // WASM64-NEXT: i64.const 8
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.store 0
+    // WASM64-NEXT: local.get 1
+    // WASM64-NEXT: i64.load 0
+    // WASM64-NEXT: end_function
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
+    // WASM32-LABEL: read_i64:
+    // WASM32: local.get 0
+    // WASM32-NEXT: local.get 0
+    // WASM32-NEXT: i32.load 0
+    // WASM32-NEXT: i32.const 7
+    // WASM32-NEXT: i32.add
+    // WASM32-NEXT: i32.const -8
+    // WASM32-NEXT: i32.and
+    // WASM32-NEXT: local.tee 1
+    // WASM32-NEXT: i32.const 8
+    // WASM32-NEXT: i32.add
+    // WASM32-NEXT: i32.store 0
+    // WASM32-NEXT: local.get 1
+    // WASM32-NEXT: i64.load 0
+    // WASM32-NEXT: end_function
+    //
+    // WASM64-LABEL: read_i64:
+    // WASM64: local.get 0
+    // WASM64-NEXT: local.get 0
+    // WASM64-NEXT: i64.load 0
+    // WASM64-NEXT: i64.const 7
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.const -8
+    // WASM64-NEXT: i64.and
+    // WASM64-NEXT: local.tee 1
+    // WASM64-NEXT: i64.const 8
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.store 0
+    // WASM64-NEXT: local.get 1
+    // WASM64-NEXT: i64.load 0
+    // WASM64-NEXT: end_function
+    va_arg(ap)
+}
+
+// Clang and Rustc use a different ABI for i128 on wasm32, and LLVM optimizes differently if we use
+// a mutable reference instead of just a pointer. With this setup we match the equivalent Clang
+// input exactly.
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i128(out: *mut i128, ap: *mut VaList<'_>) {
+    // WASM32-LABEL: read_i128:
+    // WASM32: local.get 1
+    // WASM32-NEXT: local.get 1
+    // WASM32-NEXT: i32.load 0
+    // WASM32-NEXT: i32.const 15
+    // WASM32-NEXT: i32.add
+    // WASM32-NEXT: i32.const -16
+    // WASM32-NEXT: i32.and
+    // WASM32-NEXT: local.tee 2
+    // WASM32-NEXT: i32.const 16
+    // WASM32-NEXT: i32.add
+    // WASM32-NEXT: i32.store 0
+    // WASM32-NEXT: local.get 0
+    // WASM32-NEXT: local.get 2
+    // WASM32-NEXT: i64.load 0
+    // WASM32-NEXT: i64.store 0
+    // WASM32-NEXT: local.get 0
+    // WASM32-NEXT: local.get 2
+    // WASM32-NEXT: i64.load 8
+    // WASM32-NEXT: i64.store 8
+    // WASM32-NEXT: end_function
+    //
+    // WASM64-LABEL: read_i128:
+    // WASM64: local.get 1
+    // WASM64-NEXT: local.get 1
+    // WASM64-NEXT: i64.load 0
+    // WASM64-NEXT: i64.const 15
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.const -16
+    // WASM64-NEXT: i64.and
+    // WASM64-NEXT: local.tee 2
+    // WASM64-NEXT: i64.const 16
+    // WASM64-NEXT: i64.add
+    // WASM64-NEXT: i64.store 0
+    // WASM64-NEXT: local.get 0
+    // WASM64-NEXT: local.get 2
+    // WASM64-NEXT: i64.load 0
+    // WASM64-NEXT: i64.store 0
+    // WASM64-NEXT: local.get 0
+    // WASM64-NEXT: local.get 2
+    // WASM64-NEXT: i64.load 8
+    // WASM64-NEXT: i64.store 8
+    // WASM64-NEXT: end_function
+    *out = va_arg(mem::transmute(ap));
+}
diff --git a/tests/assembly-llvm/c-variadic/x86-linux.rs b/tests/assembly-llvm/c-variadic/x86-linux.rs
new file mode 100644
index 0000000000000..8cc4731346b17
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic/x86-linux.rs
@@ -0,0 +1,237 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ revisions: X86_64 X86_64_GNUX32 I686
+//@ [X86_64] compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel
+//@ [X86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//@ [X86_64] needs-llvm-components: x86
+//@ [X86_64_GNUX32] compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel
+//@ [X86_64_GNUX32] compile-flags: --target x86_64-unknown-linux-gnux32
+//@ [X86_64_GNUX32] needs-llvm-components: x86
+//@ [I686] compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel
+//@ [I686] compile-flags: --target i686-unknown-linux-gnu
+//@ [I686] needs-llvm-components: x86
+#![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
+#![no_core]
+#![crate_type = "lib"]
+
+// Check that the assembly that rustc generates matches what clang emits.
+
+extern crate minicore;
+use minicore::*;
+
+#[lang = "va_arg_safe"]
+pub unsafe trait VaArgSafe {}
+
+unsafe impl VaArgSafe for i32 {}
+unsafe impl VaArgSafe for i64 {}
+#[cfg(target_pointer_width = "64")]
+unsafe impl VaArgSafe for i128 {}
+unsafe impl VaArgSafe for f64 {}
+unsafe impl<T> VaArgSafe for *const T {}
+
+#[repr(transparent)]
+struct VaListInner {
+    ptr: *const c_void,
+}
+
+#[repr(transparent)]
+#[lang = "va_list"]
+pub struct VaList<'a> {
+    inner: VaListInner,
+    _marker: PhantomData<&'a mut ()>,
+}
+
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub const unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
+    // CHECK-LABEL: read_f64
+
+    // X86_64: mov     ecx, dword ptr [rdi + 4]
+    // X86_64-NEXT: cmp     rcx, 160
+    // X86_64-NEXT: ja      .LBB0_2
+    // X86_64-NEXT: mov     rax, rcx
+    // X86_64-NEXT: add     rax, qword ptr [rdi + 16]
+    // X86_64-NEXT: add     ecx, 16
+    // X86_64-NEXT: mov     dword ptr [rdi + 4], ecx
+    // X86_64-NEXT: movsd   xmm0, qword ptr [rax]
+    // X86_64-NEXT: ret
+    // X86_64-NEXT: .LBB0_2:
+    // X86_64-NEXT: mov     rax, qword ptr [rdi + 8]
+    // X86_64-NEXT: lea     rcx, [rax + 8]
+    // X86_64-NEXT: mov     qword ptr [rdi + 8], rcx
+    // X86_64-NEXT: movsd   xmm0, qword ptr [rax]
+    // X86_64-NEXT: ret
+
+    // X86_64-NEXT_GNUX32: mov     ecx, dword ptr [edi + 4]
+    // X86_64-NEXT_GNUX32-NEXT: cmp     ecx, 160
+    // X86_64-NEXT_GNUX32-NEXT: ja      .LBB0_2
+    // X86_64-NEXT_GNUX32-NEXT: mov     eax, dword ptr [edi + 12]
+    // X86_64-NEXT_GNUX32-NEXT: add     eax, ecx
+    // X86_64-NEXT_GNUX32-NEXT: add     ecx, 16
+    // X86_64-NEXT_GNUX32-NEXT: mov     dword ptr [edi + 4], ecx
+    // X86_64-NEXT_GNUX32-NEXT: movsd   xmm0, qword ptr [eax]
+    // X86_64-NEXT_GNUX32-NEXT: ret
+    // X86_64-NEXT_GNUX32-NEXT: .LBB0_2:
+    // X86_64-NEXT_GNUX32-NEXT: mov     eax, dword ptr [edi + 8]
+    // X86_64-NEXT_GNUX32-NEXT: lea     ecx, [rax + 8]
+    // X86_64-NEXT_GNUX32-NEXT: mov     dword ptr [edi + 8], ecx
+    // X86_64-NEXT_GNUX32-NEXT: movsd   xmm0, qword ptr [eax]
+    // X86_64-NEXT_GNUX32-NEXT: ret
+
+    // I686: mov eax, dword ptr [esp + 4]
+    // I686-NEXT: mov ecx, dword ptr [eax]
+    // I686-NEXT: lea edx, [ecx + 8]
+    // I686-NEXT: mov dword ptr [eax], edx
+    // I686-NEXT: fld qword ptr [ecx]
+    // I686-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
+    // CHECK-LABEL: read_i32
+    //
+    // X86_64: mov     ecx, dword ptr [rdi]
+    // X86_64-NEXT: cmp     rcx, 40
+    // X86_64-NEXT: ja      .LBB1_2
+    // X86_64-NEXT: mov     rax, rcx
+    // X86_64-NEXT: add     rax, qword ptr [rdi + 16]
+    // X86_64-NEXT: add     ecx, 8
+    // X86_64-NEXT: mov     dword ptr [rdi], ecx
+    // X86_64-NEXT: mov     eax, dword ptr [rax]
+    // X86_64-NEXT: ret
+    // X86_64-NEXT: .LBB1_2:
+    // X86_64-NEXT: mov     rax, qword ptr [rdi + 8]
+    // X86_64-NEXT: lea     rcx, [rax + 8]
+    // X86_64-NEXT: mov     qword ptr [rdi + 8], rcx
+    // X86_64-NEXT: mov     eax, dword ptr [rax]
+    // X86_64-NEXT: ret
+
+    // X86_64-NEXT_GNUX32: mov     ecx, dword ptr [edi]
+    // X86_64-NEXT_GNUX32-NEXT: cmp     ecx, 40
+    // X86_64-NEXT_GNUX32-NEXT: ja      .LBB1_2
+    // X86_64-NEXT_GNUX32-NEXT: mov     eax, dword ptr [edi + 12]
+    // X86_64-NEXT_GNUX32-NEXT: add     eax, ecx
+    // X86_64-NEXT_GNUX32-NEXT: add     ecx, 8
+    // X86_64-NEXT_GNUX32-NEXT: mov     dword ptr [edi], ecx
+    // X86_64-NEXT_GNUX32-NEXT: mov     eax, dword ptr [eax]
+    // X86_64-NEXT_GNUX32-NEXT: ret
+    // X86_64-NEXT_GNUX32-NEXT: .LBB1_2:
+    // X86_64-NEXT_GNUX32-NEXT: mov     eax, dword ptr [edi + 8]
+    // X86_64-NEXT_GNUX32-NEXT: lea     ecx, [rax + 8]
+    // X86_64-NEXT_GNUX32-NEXT: mov     dword ptr [edi + 8], ecx
+    // X86_64-NEXT_GNUX32-NEXT: mov     eax, dword ptr [eax]
+    // X86_64-NEXT_GNUX32-NEXT: ret
+
+    // I686: mov eax, dword ptr [esp + 4]
+    // I686-NEXT: mov ecx, dword ptr [eax]
+    // I686-NEXT: lea edx, [ecx + 4]
+    // I686-NEXT: mov dword ptr [eax], edx
+    // I686-NEXT: mov eax, dword ptr [ecx]
+    // I686-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
+    // CHECK-LABEL: read_i64
+
+    // X86_64: mov     ecx, dword ptr [rdi]
+    // X86_64-NEXT: cmp     rcx, 40
+    // X86_64-NEXT: ja      .LBB2_2
+    // X86_64-NEXT: mov     rax, rcx
+    // X86_64-NEXT: add     rax, qword ptr [rdi + 16]
+    // X86_64-NEXT: add     ecx, 8
+    // X86_64-NEXT: mov     dword ptr [rdi], ecx
+    // X86_64-NEXT: mov     rax, qword ptr [rax]
+    // X86_64-NEXT: ret
+    // X86_64-NEXT: .LBB2_2:
+    // X86_64-NEXT: mov     rax, qword ptr [rdi + 8]
+    // X86_64-NEXT: lea     rcx, [rax + 8]
+    // X86_64-NEXT: mov     qword ptr [rdi + 8], rcx
+    // X86_64-NEXT: mov     rax, qword ptr [rax]
+    // X86_64-NEXT: ret
+
+    // X86_64-NEXT_GNUX32: mov     ecx, dword ptr [edi]
+    // X86_64-NEXT_GNUX32-NEXT: cmp     ecx, 40
+    // X86_64-NEXT_GNUX32-NEXT: ja      .LBB2_2
+    // X86_64-NEXT_GNUX32-NEXT: mov     eax, dword ptr [edi + 12]
+    // X86_64-NEXT_GNUX32-NEXT: add     eax, ecx
+    // X86_64-NEXT_GNUX32-NEXT: add     ecx, 8
+    // X86_64-NEXT_GNUX32-NEXT: mov     dword ptr [edi], ecx
+    // X86_64-NEXT_GNUX32-NEXT: mov     rax, qword ptr [eax]
+    // X86_64-NEXT_GNUX32-NEXT: ret
+    // X86_64-NEXT_GNUX32-NEXT: .LBB2_2:
+    // X86_64-NEXT_GNUX32-NEXT: mov     eax, dword ptr [edi + 8]
+    // X86_64-NEXT_GNUX32-NEXT: lea     ecx, [rax + 8]
+    // X86_64-NEXT_GNUX32-NEXT: mov     dword ptr [edi + 8], ecx
+    // X86_64-NEXT_GNUX32-NEXT: mov     rax, qword ptr [eax]
+    // X86_64-NEXT_GNUX32-NEXT: ret
+
+    // I686: mov eax, dword ptr [esp + 4]
+    // I686-NEXT: mov ecx, dword ptr [eax]
+    // I686-NEXT: lea edx, [ecx + 8]
+    // I686-NEXT: mov dword ptr [eax], edx
+    // I686-NEXT: mov eax, dword ptr [ecx]
+    // I686-NEXT: mov edx, dword ptr [ecx + 4]
+    // I686-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+#[cfg(target_pointer_width = "64")]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // X86_64-LABEL: read_i128
+    //
+    // X86_64: mov ecx, dword ptr [rdi]
+    // X86_64-NEXT: cmp rcx, 32
+    // X86_64-NEXT: ja  .LBB3_2
+    // X86_64-NEXT: mov rdx, qword ptr [rdi + 16]
+    // X86_64-NEXT: mov rax, qword ptr [rdx + rcx]
+    // X86_64-NEXT: mov rdx, qword ptr [rdx + rcx + 8]
+    // X86_64-NEXT: add ecx, 16
+    // X86_64-NEXT: mov dword ptr [rdi], ecx
+    // X86_64-NEXT: ret
+    // X86_64-NEXT: .LBB3_2:
+    // X86_64-NEXT: mov     rcx, qword ptr [rdi + 8]
+    // X86_64-NEXT: add     rcx, 15
+    // X86_64-NEXT: and     rcx, -16
+    // X86_64-NEXT: lea     rax, [rcx + 16]
+    // X86_64-NEXT: mov     qword ptr [rdi + 8], rax
+    // X86_64-NEXT: mov     rax, qword ptr [rcx]
+    // X86_64-NEXT: mov     rdx, qword ptr [rcx + 8]
+    // X86_64-NEXT: ret
+
+    // X86_64-NEXT_GNUX32: mov     ecx, dword ptr [edi]
+    // X86_64-NEXT_GNUX32-NEXT: cmp     ecx, 32
+    // X86_64-NEXT_GNUX32-NEXT: ja      .LBB3_2
+    // X86_64-NEXT_GNUX32-NEXT: mov     edx, dword ptr [edi + 12]
+    // X86_64-NEXT_GNUX32-NEXT: mov     rax, qword ptr [edx + ecx]
+    // X86_64-NEXT_GNUX32-NEXT: mov     rdx, qword ptr [edx + ecx + 8]
+    // X86_64-NEXT_GNUX32-NEXT: add     ecx, 16
+    // X86_64-NEXT_GNUX32-NEXT: mov     dword ptr [edi], ecx
+    // X86_64-NEXT_GNUX32-NEXT: ret
+    // X86_64-NEXT_GNUX32-NEXT: .LBB3_2:
+    // X86_64-NEXT_GNUX32-NEXT: mov     ecx, dword ptr [edi + 8]
+    // X86_64-NEXT_GNUX32-NEXT: add     ecx, 15
+    // X86_64-NEXT_GNUX32-NEXT: and     ecx, -16
+    // X86_64-NEXT_GNUX32-NEXT: lea     eax, [rcx + 16]
+    // X86_64-NEXT_GNUX32-NEXT: mov     dword ptr [edi + 8], eax
+    // X86_64-NEXT_GNUX32-NEXT: mov     rax, qword ptr [ecx]
+    // X86_64-NEXT_GNUX32-NEXT: mov     rdx, qword ptr [ecx + 8]
+    // X86_64-NEXT_GNUX32-NEXT: ret
+
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
+    // X86_64: read_ptr = read_i64
+    // X86_64_GNUX32: read_ptr = read_i32
+    // I686: read_ptr = read_i32
+    va_arg(ap)
+}
diff --git a/tests/assembly-llvm/c-variadic/x86_64-windows.rs b/tests/assembly-llvm/c-variadic/x86_64-windows.rs
new file mode 100644
index 0000000000000..fa40699af80b4
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic/x86_64-windows.rs
@@ -0,0 +1,100 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ revisions: WINDOWS_GNU WINDOWS_MSVC
+//@ [WINDOWS_GNU] compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel
+//@ [WINDOWS_GNU] compile-flags: --target x86_64-pc-windows-gnu
+//@ [WINDOWS_GNU] needs-llvm-components: x86
+//@ [WINDOWS_MSVC] compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel
+//@ [WINDOWS_MSVC] compile-flags: --target x86_64-pc-windows-msvc
+//@ [WINDOWS_MSVC] needs-llvm-components: x86
+#![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
+#![no_core]
+#![crate_type = "lib"]
+
+// Check that the assembly that rustc generates matches what clang emits for
+//
+// ```c
+// T function(va_list ap) {
+//     return va_arg(ap, T);
+// }
+// ```
+
+extern crate minicore;
+use minicore::*;
+
+#[lang = "va_arg_safe"]
+pub unsafe trait VaArgSafe {}
+
+unsafe impl VaArgSafe for i32 {}
+unsafe impl VaArgSafe for i64 {}
+unsafe impl VaArgSafe for i128 {}
+unsafe impl VaArgSafe for f64 {}
+unsafe impl<T> VaArgSafe for *const T {}
+
+#[repr(transparent)]
+struct VaListInner {
+    ptr: *const c_void,
+}
+
+#[repr(transparent)]
+#[lang = "va_list"]
+pub struct VaList<'a> {
+    inner: VaListInner,
+    _marker: PhantomData<&'a mut ()>,
+}
+
+#[rustc_intrinsic]
+#[rustc_nounwind]
+pub const unsafe fn va_arg<T: VaArgSafe>(ap: &mut VaList<'_>) -> T;
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
+    // CHECK-LABEL: read_f64:
+    // CHECK: mov rax, qword ptr [rcx]
+    // CHECK-NEXT: lea rdx, [rax + 8]
+    // CHECK-NEXT: mov qword ptr [rcx], rdx
+    // CHECK-NEXT: movsd xmm0, qword ptr [rax]
+    // CHECK-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
+    // CHECK-LABEL: read_i32:
+    // CHECK: mov rax, qword ptr [rcx]
+    // CHECK-NEXT: lea rdx, [rax + 8]
+    // CHECK-NEXT: mov qword ptr [rcx], rdx
+    // CHECK-NEXT: mov eax, dword ptr [rax]
+    // CHECK-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
+    // CHECK-LABEL: read_i64:
+    // CHECK: mov rax, qword ptr [rcx]
+    // CHECK-NEXT: lea rdx, [rax + 8]
+    // CHECK-NEXT: mov qword ptr [rcx], rdx
+    // CHECK-NEXT: mov rax, qword ptr [rax]
+    // CHECK-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
+    // CHECK-LABEL: read_i128:
+    // CHECK: mov rax, qword ptr [rcx]
+    // CHECK-NEXT: lea rdx, [rax + 8]
+    // CHECK-NEXT: mov qword ptr [rcx], rdx
+    // CHECK-NEXT: mov rax, qword ptr [rax]
+    // CHECK-NEXT: movups xmm0, xmmword ptr [rax]
+    // CHECK-NEXT: ret
+    va_arg(ap)
+}
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
+    // CHECK: read_ptr = read_i64
+    va_arg(ap)
+}

From acccc68b623325aaf1117a3696fe063b2b16b37c Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert@folkertdev.nl>
Date: Sun, 5 Jul 2026 13:34:55 +0200
Subject: [PATCH 11/16] c-variadic: fix i128 argument being misread on aarch64
 msvc

---
 compiler/rustc_codegen_llvm/src/va_arg.rs |  2 +-
 tests/assembly-llvm/c-variadic/aarch64.rs | 39 ++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs
index 92b0709506526..3ca920eeaa64a 100644
--- a/compiler/rustc_codegen_llvm/src/va_arg.rs
+++ b/compiler/rustc_codegen_llvm/src/va_arg.rs
@@ -1096,7 +1096,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             target_ty,
             PassMode::Direct,
             SlotSize::Bytes8,
-            if target.is_like_windows { AllowHigherAlign::No } else { AllowHigherAlign::Yes },
+            AllowHigherAlign::Yes,
             ForceRightAdjust::No,
         ),
         Arch::AArch64 => emit_aapcs_va_arg(bx, addr, target_ty),
diff --git a/tests/assembly-llvm/c-variadic/aarch64.rs b/tests/assembly-llvm/c-variadic/aarch64.rs
index eb242a1a41e21..a5dbc8a42761a 100644
--- a/tests/assembly-llvm/c-variadic/aarch64.rs
+++ b/tests/assembly-llvm/c-variadic/aarch64.rs
@@ -1,13 +1,15 @@
 //@ add-minicore
 //@ assembly-output: emit-asm
 //
-//@ revisions: AARCH64_LINUX AARCH64_DARWIN AARCH64_BE ARM64EC_MSVC
+//@ revisions: AARCH64_LINUX AARCH64_DARWIN AARCH64_BE AARCH64_MSVC ARM64EC_MSVC
 //@ [AARCH64_LINUX] compile-flags: -Copt-level=3 --target aarch64-unknown-linux-gnu
 //@ [AARCH64_LINUX] needs-llvm-components: aarch64
 //@ [AARCH64_BE] compile-flags: -Copt-level=3 --target aarch64_be-unknown-linux-gnu
 //@ [AARCH64_BE] needs-llvm-components: aarch64
 //@ [AARCH64_DARWIN] compile-flags: -Copt-level=3 --target aarch64-apple-darwin
 //@ [AARCH64_DARWIN] needs-llvm-components: aarch64
+//@ [AARCH64_MSVC] compile-flags: -Copt-level=3 --target aarch64-pc-windows-msvc
+//@ [AARCH64_MSVC] needs-llvm-components: aarch64
 //@ [ARM64EC_MSVC] compile-flags: -Copt-level=3 --target arm64ec-pc-windows-msvc
 //@ [ARM64EC_MSVC] needs-llvm-components: aarch64
 #![feature(c_variadic, no_core, lang_items, intrinsics, rustc_attrs)]
@@ -101,6 +103,12 @@ unsafe extern "C" fn read_f64(ap: &mut VaList<'_>) -> f64 {
     // AARCH64_DARWIN-NEXT: ldr d0, [x8], #8
     // AARCH64_DARWIN-NEXT: str x8, [x0]
     // AARCH64_DARWIN-NEXT: ret
+
+    // AARCH64_MSVC-LABEL: read_f64:
+    // AARCH64_MSVC: ldr x8, [x0]
+    // AARCH64_MSVC-NEXT: ldr d0, [x8], #8
+    // AARCH64_MSVC-NEXT: str x8, [x0]
+    // AARCH64_MSVC-NEXT: ret
     va_arg(ap)
 }
 
@@ -157,6 +165,13 @@ unsafe extern "C" fn read_i32(ap: &mut VaList<'_>) -> i32 {
     // AARCH64_DARWIN-NEXT: str x9, [x0]
     // AARCH64_DARWIN-NEXT: mov x0, x8
     // AARCH64_DARWIN-NEXT: ret
+
+    // AARCH64_MSVC-LABEL: read_i32:
+    // AARCH64_MSVC: ldr x9, [x0]
+    // AARCH64_MSVC-NEXT: mov x8, x0
+    // AARCH64_MSVC-NEXT: ldr w0, [x9], #8
+    // AARCH64_MSVC-NEXT: str x9, [x8]
+    // AARCH64_MSVC-NEXT: ret
     va_arg(ap)
 }
 
@@ -214,6 +229,13 @@ unsafe extern "C" fn read_i64(ap: &mut VaList<'_>) -> i64 {
     // AARCH64_DARWIN-NEXT: str x9, [x0]
     // AARCH64_DARWIN-NEXT: mov x0, x8
     // AARCH64_DARWIN-NEXT: ret
+
+    // AARCH64_MSVC-LABEL: read_i64:
+    // AARCH64_MSVC: ldr x9, [x0]
+    // AARCH64_MSVC-NEXT: mov x8, x0
+    // AARCH64_MSVC-NEXT: ldr x0, [x9], #8
+    // AARCH64_MSVC-NEXT: str x9, [x8]
+    // AARCH64_MSVC-NEXT: ret
     va_arg(ap)
 }
 
@@ -281,6 +303,20 @@ unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
     // AARCH64_DARWIN-NEXT: str x9, [x0]
     // AARCH64_DARWIN-NEXT: mov x0, x8
     // AARCH64_DARWIN-NEXT: ret
+
+    // NOTE: rustc bumps the alignment to 16, which deviates from clang's va_arg
+    // but matches MSVC and how clang passes i128 c-variadic arguments.
+    //
+    // AARCH64_MSVC-LABEL: read_i128:
+    // AARCH64_MSVC: ldr x9, [x0]
+    // AARCH64_MSVC-NEXT: mov x8, x0
+    // AARCH64_MSVC-NEXT: add x9, x9, #15
+    // AARCH64_MSVC-NEXT: and x9, x9, #0xfffffffffffffff0
+    // AARCH64_MSVC-NEXT: mov x10, x9
+    // AARCH64_MSVC-NEXT: ldr x1, [x9, #8]
+    // AARCH64_MSVC-NEXT: ldr x0, [x10], #16
+    // AARCH64_MSVC-NEXT: str x10, [x8]
+    // AARCH64_MSVC-NEXT: ret
     va_arg(ap)
 }
 
@@ -288,6 +324,7 @@ unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
 unsafe extern "C" fn read_ptr(ap: &mut VaList<'_>) -> *const u8 {
     // AARCH64_LINUX-CHECK: read_ptr = read_i64
     // AARCH64_BE-CHECK: read_ptr = read_i64
+    // AARCH64_MSVC: read_ptr = read_i64
     // ARM64EC_MSVC: "#read_ptr" = "#read_i64"
     // AARCH64_DARWIN-CHECK: _read_ptr = _read_i64
     va_arg(ap)

From 3ba37359ab4e1569df9dcef54fbe9e25e7d55366 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert@folkertdev.nl>
Date: Sun, 5 Jul 2026 15:10:44 +0200
Subject: [PATCH 12/16] c-variadic: fix i128 argument being misread on arm64ec

---
 compiler/rustc_codegen_llvm/src/va_arg.rs | 10 ++++++++--
 tests/assembly-llvm/c-variadic/aarch64.rs |  8 +++++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs
index 3ca920eeaa64a..a64452dbc5a7e 100644
--- a/compiler/rustc_codegen_llvm/src/va_arg.rs
+++ b/compiler/rustc_codegen_llvm/src/va_arg.rs
@@ -1085,9 +1085,15 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             bx,
             addr,
             target_ty,
-            PassMode::Direct,
+            // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
+            // not 1, 2, 4, or 8 bytes, must be passed by reference."
+            if target_ty_size > 8 || !target_ty_size.is_power_of_two() {
+                PassMode::Indirect
+            } else {
+                PassMode::Direct
+            },
             SlotSize::Bytes8,
-            if target.is_like_windows { AllowHigherAlign::No } else { AllowHigherAlign::Yes },
+            AllowHigherAlign::No,
             ForceRightAdjust::No,
         ),
         Arch::AArch64 if target.is_like_windows || target.is_like_darwin => emit_ptr_va_arg(
diff --git a/tests/assembly-llvm/c-variadic/aarch64.rs b/tests/assembly-llvm/c-variadic/aarch64.rs
index a5dbc8a42761a..fcb44d044abb5 100644
--- a/tests/assembly-llvm/c-variadic/aarch64.rs
+++ b/tests/assembly-llvm/c-variadic/aarch64.rs
@@ -287,11 +287,13 @@ unsafe extern "C" fn read_i128(ap: &mut VaList<'_>) -> i128 {
     // AARCH64_BE-NEXT: ldp     x0, x1, [x8]
     // AARCH64_BE-NEXT: ret
 
+    // NOTE: matches x86_64 Windows: an `i128` is passed indirectly, the slot holds a pointer.
+    //
     // ARM64EC_MSVC-LABEL: read_i128 = "#read_i128"
     // ARM64EC_MSVC: ldr x9, [x0]
-    // ARM64EC_MSVC-NEXT: mov x8, x0
-    // ARM64EC_MSVC-NEXT: ldp x0, x1, [x9], #16
-    // ARM64EC_MSVC-NEXT: str x9, [x8]
+    // ARM64EC_MSVC-NEXT: ldr x10, [x9], #8
+    // ARM64EC_MSVC-NEXT: str x9, [x0]
+    // ARM64EC_MSVC-NEXT: ldp x0, x1, [x10]
     // ARM64EC_MSVC-NEXT: ret
 
     // AARCH64_DARWIN-LABEL: _read_i128:

From 06da2d2081a48cd54c785a92f7c52ea1d929c92b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Thu, 9 Jul 2026 23:17:16 +0200
Subject: [PATCH 13/16] Print step stack trace when bootstrap panics

---
 src/bootstrap/src/bin/main.rs                | 13 +++++-
 src/bootstrap/src/core/builder/mod.rs        | 17 ++++++-
 src/bootstrap/src/core/builder/step_stack.rs | 48 ++++++++++++++++++++
 src/bootstrap/src/lib.rs                     |  2 +-
 src/bootstrap/src/utils/tracing.rs           |  1 -
 5 files changed, 77 insertions(+), 4 deletions(-)
 create mode 100644 src/bootstrap/src/core/builder/step_stack.rs

diff --git a/src/bootstrap/src/bin/main.rs b/src/bootstrap/src/bin/main.rs
index 44eab9b87783b..a86dac84b1756 100644
--- a/src/bootstrap/src/bin/main.rs
+++ b/src/bootstrap/src/bin/main.rs
@@ -13,7 +13,7 @@ use std::time::Instant;
 use std::{env, process};
 
 use bootstrap::{
-    Build, CONFIG_CHANGE_HISTORY, ChangeId, Config, Flags, Subcommand, debug,
+    Build, CONFIG_CHANGE_HISTORY, ChangeId, Config, Flags, StepStack, Subcommand, debug,
     find_recent_config_change_ids, human_readable_changes, t,
 };
 
@@ -27,6 +27,17 @@ fn main() {
 
     let _start_time = Instant::now();
 
+    let default_panic_hook = std::panic::take_hook();
+    std::panic::set_hook(Box::new(move |info| {
+        default_panic_hook(info);
+        StepStack::with_current(|stack| {
+            eprintln!("\nBootstrap has panicked, currently active steps:");
+            for step in stack.get_active_steps() {
+                eprintln!("{} at {}", step.info, step.location);
+            }
+        });
+    }));
+
     let args = env::args().skip(1).collect::<Vec<_>>();
 
     if Flags::try_parse_verbose_help(&args) {
diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs
index 66e14dae9b9ec..0f68988a683cf 100644
--- a/src/bootstrap/src/core/builder/mod.rs
+++ b/src/bootstrap/src/core/builder/mod.rs
@@ -21,16 +21,20 @@ use crate::core::build_steps::{
     check, clean, clippy, compile, dist, doc, gcc, install, llvm, run, setup, test, tool, vendor,
 };
 use crate::core::builder::cli_paths::CLIStepPath;
+use crate::core::builder::step_stack::StepRecord;
+pub use crate::core::builder::step_stack::StepStack;
 use crate::core::config::flags::Subcommand;
 use crate::core::config::{DryRun, TargetSelection};
 use crate::utils::build_stamp::BuildStamp;
 use crate::utils::cache::Cache;
 use crate::utils::exec::{BootstrapCommand, ExecutionContext, command};
 use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t};
+use crate::utils::tracing::format_location;
 use crate::{Build, Crate, trace};
 
 mod cargo;
 mod cli_paths;
+mod step_stack;
 #[cfg(test)]
 mod tests;
 
@@ -1112,6 +1116,7 @@ impl<'a> Builder<'a> {
             Subcommand::Perf { .. } => (Kind::Perf, &paths[..]),
         };
 
+        StepStack::with_current(|stack| stack.clear());
         Self::new_internal(build, kind, paths.to_owned())
     }
 
@@ -1574,6 +1579,12 @@ Alternatively, you can set `build.local-rebuild=true` and use a stage0 compiler
                 graph.register_step_execution(&step, parent, self.config.dry_run());
             }
 
+            // The location has to be gathered in this function, to be correctly propagated with
+            // #[track_caller].
+            let location = format_location(*std::panic::Location::caller());
+            StepStack::with_current(|stack| {
+                stack.push(StepRecord { info: pretty_print_step(&step), location });
+            });
             stack.push(Box::new(step.clone()));
         }
 
@@ -1599,7 +1610,7 @@ Alternatively, you can set `build.local-rebuild=true` and use a stage0 compiler
                     "step",
                     step_name = pretty_step_name::<S>(),
                     args = step_debug_args(&step),
-                    location = crate::utils::tracing::format_location(*std::panic::Location::caller())
+                    location = format_location(*std::panic::Location::caller())
                 );
                 span.entered()
             };
@@ -1626,6 +1637,10 @@ Alternatively, you can set `build.local-rebuild=true` and use a stage0 compiler
             let mut stack = self.stack.borrow_mut();
             let cur_step = stack.pop().expect("step stack empty");
             assert_eq!(cur_step.downcast_ref(), Some(&step));
+
+            StepStack::with_current(|stack| {
+                stack.pop();
+            });
         }
         self.cache.put(step, out.clone());
         out
diff --git a/src/bootstrap/src/core/builder/step_stack.rs b/src/bootstrap/src/core/builder/step_stack.rs
new file mode 100644
index 0000000000000..5a6bf31809a64
--- /dev/null
+++ b/src/bootstrap/src/core/builder/step_stack.rs
@@ -0,0 +1,48 @@
+use std::cell::RefCell;
+
+thread_local! {
+    static STEP_STACK: RefCell<StepStack> = const { RefCell::new(StepStack::new()) };
+}
+
+/// This type serves for recording the stack of executed steps in a thread-local variable.
+///
+/// It is used to print the currently running step stack in places where we do not have easy
+/// access to the currently used builder, e.g. in exit! macros or the panic handler.
+pub struct StepStack {
+    stack: Vec<StepRecord>,
+}
+
+pub struct StepRecord {
+    pub info: String,
+    pub location: String,
+}
+
+impl StepStack {
+    /// Return the currently active step stack for this thread.
+    pub fn with_current<F>(func: F)
+    where
+        F: FnOnce(&mut StepStack),
+    {
+        STEP_STACK.with(|stack| func(&mut stack.borrow_mut()));
+    }
+
+    const fn new() -> Self {
+        Self { stack: Vec::new() }
+    }
+
+    pub fn get_active_steps(&self) -> impl Iterator<Item = &StepRecord> {
+        self.stack.iter()
+    }
+
+    pub fn clear(&mut self) {
+        self.stack.clear();
+    }
+
+    pub fn push(&mut self, record: StepRecord) {
+        self.stack.push(record);
+    }
+
+    pub fn pop(&mut self) {
+        self.stack.pop();
+    }
+}
diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs
index 2c8e002123abf..64993224a36cc 100644
--- a/src/bootstrap/src/lib.rs
+++ b/src/bootstrap/src/lib.rs
@@ -42,9 +42,9 @@ use crate::utils::helpers::{self, dir_is_empty, exe, libdir, set_file_times, spl
 mod core;
 mod utils;
 
-pub use core::builder::PathSet;
 #[cfg(feature = "tracing")]
 pub use core::builder::STEP_SPAN_TARGET;
+pub use core::builder::{PathSet, StepStack};
 pub use core::config::flags::{Flags, Subcommand};
 pub use core::config::{ChangeId, Config};
 
diff --git a/src/bootstrap/src/utils/tracing.rs b/src/bootstrap/src/utils/tracing.rs
index 4e75347f4cbae..541aa088b9c30 100644
--- a/src/bootstrap/src/utils/tracing.rs
+++ b/src/bootstrap/src/utils/tracing.rs
@@ -66,7 +66,6 @@ macro_rules! trace_io {
     }
 }
 
-#[cfg(feature = "tracing")]
 pub fn format_location(location: std::panic::Location<'static>) -> String {
     format!("{}:{}", location.file(), location.line())
 }

From 07d3f8b49b533674db31bfb7e8f3d5cdd4786e65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Thu, 9 Jul 2026 23:22:49 +0200
Subject: [PATCH 14/16] Enable backtraces in bootstrap by default

---
 src/bootstrap/src/bin/main.rs | 8 ++++++++
 src/ci/run.sh                 | 3 ---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/bootstrap/src/bin/main.rs b/src/bootstrap/src/bin/main.rs
index a86dac84b1756..63e443bac2363 100644
--- a/src/bootstrap/src/bin/main.rs
+++ b/src/bootstrap/src/bin/main.rs
@@ -27,6 +27,14 @@ fn main() {
 
     let _start_time = Instant::now();
 
+    // Always print backtraces to provide richer errors, to help debug hard-to-reproduce panics
+    // when the user didn't specify RUST_BACKTRACE
+    if std::env::var("RUST_BACKTRACE").is_err() {
+        unsafe {
+            std::env::set_var("RUST_BACKTRACE", "1");
+        }
+    }
+
     let default_panic_hook = std::panic::take_hook();
     std::panic::set_hook(Box::new(move |info| {
         default_panic_hook(info);
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 2aba67d28aee8..339d9ffa55faf 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -206,9 +206,6 @@ if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-new-symbol-mangling"
 fi
 
-# If bootstrap fails, we want to see its backtrace
-export RUST_BACKTRACE=1
-
 # Print the date from the local machine and the date from an external source to
 # check for clock drifts. An HTTP URL is used instead of HTTPS since on Azure
 # Pipelines it happened that the certificates were marked as expired.

From 762fc14be68c97dee5d747e3ece6ef67d5e4116b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Thu, 9 Jul 2026 23:30:14 +0200
Subject: [PATCH 15/16] Move exit macro helpers from build_helper to bootstrap

Since the bootstrap command execution redesign, build_helper no longer has to implement these functions. By moving them to boostrap, we can enrich them with bootstrap-specific information.
---
 src/bootstrap/src/core/build_steps/clippy.rs  |  4 +-
 src/bootstrap/src/core/build_steps/compile.rs |  6 +-
 src/bootstrap/src/core/build_steps/llvm.rs    |  3 +-
 src/bootstrap/src/core/build_steps/run.rs     |  3 +-
 src/bootstrap/src/core/build_steps/test.rs    |  4 +-
 src/bootstrap/src/core/config/config.rs       | 11 ++--
 src/bootstrap/src/core/config/flags.rs        |  2 +-
 src/bootstrap/src/core/config/mod.rs          |  3 +-
 src/bootstrap/src/core/download.rs            |  2 +-
 src/bootstrap/src/lib.rs                      |  8 ++-
 src/bootstrap/src/utils/exec.rs               |  3 +-
 src/bootstrap/src/utils/helpers.rs            | 35 +++++++++++-
 src/build_helper/src/util.rs                  | 57 -------------------
 src/tools/rustdoc-gui-test/src/main.rs        | 24 +++++++-
 14 files changed, 80 insertions(+), 85 deletions(-)

diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs
index d3b533c954293..f03ae75c4bbf2 100644
--- a/src/bootstrap/src/core/build_steps/clippy.rs
+++ b/src/bootstrap/src/core/build_steps/clippy.rs
@@ -13,8 +13,6 @@
 //! to pass a prebuilt Clippy from the outside when running `cargo clippy`, but that would be
 //! (as usual) a massive undertaking/refactoring.
 
-use build_helper::exit;
-
 use super::compile::{ArtifactKeepMode, run_cargo, rustc_cargo, std_cargo};
 use super::tool::{SourceType, prepare_tool_cargo};
 use crate::builder::{Builder, ShouldRun};
@@ -23,7 +21,7 @@ use crate::core::build_steps::compile::std_crates_for_run_make;
 use crate::core::builder;
 use crate::core::builder::{Alias, Kind, RunConfig, Step, StepMetadata, crate_description};
 use crate::utils::build_stamp::{self, BuildStamp};
-use crate::{Compiler, Mode, Subcommand, TargetSelection};
+use crate::{Compiler, Mode, Subcommand, TargetSelection, exit};
 
 /// Disable the most spammy clippy lints
 const IGNORED_RULES_FOR_STD_AND_RUSTC: &[&str] = &[
diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
index 3216fd671c3f8..4944737fe047b 100644
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@@ -38,7 +38,7 @@ use crate::utils::helpers::{
 };
 use crate::{
     CLang, CodegenBackendKind, Compiler, DependencyType, FileType, GitRepo, LLVM_TOOLS, Mode,
-    debug, trace,
+    debug, exit, trace,
 };
 
 /// Build a standard library for the given `target` using the given `build_compiler`.
@@ -2068,7 +2068,7 @@ impl Step for Sysroot {
                         sysroot_lib_rustlib_src_rust.display(),
                     );
                 }
-                build_helper::exit!(1);
+                exit!(1);
             }
         }
 
@@ -2086,7 +2086,7 @@ impl Step for Sysroot {
                     builder.src.display(),
                     e,
                 );
-                build_helper::exit!(1);
+                exit!(1);
             }
         }
 
diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs
index 6f1b8532f031b..1a453ad6a7a55 100644
--- a/src/bootstrap/src/core/build_steps/llvm.rs
+++ b/src/bootstrap/src/core/build_steps/llvm.rs
@@ -14,7 +14,6 @@ use std::path::{Path, PathBuf};
 use std::sync::OnceLock;
 use std::{env, fs};
 
-use build_helper::exit;
 use build_helper::git::PathFreshness;
 
 use crate::core::build_steps::llvm;
@@ -25,7 +24,7 @@ use crate::utils::exec::command;
 use crate::utils::helpers::{
     self, exe, get_clang_cl_resource_dir, libdir, t, unhashed_basename, up_to_date,
 };
-use crate::{CLang, GitRepo, Kind, trace};
+use crate::{CLang, GitRepo, Kind, exit, trace};
 
 #[derive(Clone)]
 pub struct LlvmResult {
diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs
index 935844daa9d7c..1d704230902e4 100644
--- a/src/bootstrap/src/core/build_steps/run.rs
+++ b/src/bootstrap/src/core/build_steps/run.rs
@@ -5,7 +5,6 @@
 
 use std::path::PathBuf;
 
-use build_helper::exit;
 use build_helper::git::get_git_untracked_files;
 use clap_complete::{Generator, shells};
 
@@ -17,7 +16,7 @@ use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step, StepMetada
 use crate::core::config::TargetSelection;
 use crate::core::config::flags::{get_completion, top_level_help};
 use crate::utils::exec::command;
-use crate::{Mode, t};
+use crate::{Mode, exit, t};
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct BuildManifest;
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index 697f659816120..61529da631a06 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -13,8 +13,6 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::{env, fs, iter};
 
-use build_helper::exit;
-
 use crate::core::build_steps::compile::{ArtifactKeepMode, Std, run_cargo};
 use crate::core::build_steps::doc::{DocumentationFormat, prepare_doc_compiler};
 use crate::core::build_steps::gcc::{Gcc, GccTargetPair, add_cg_gcc_cargo_flags};
@@ -44,7 +42,7 @@ use crate::utils::helpers::{
     up_to_date,
 };
 use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests};
-use crate::{CLang, CodegenBackendKind, GitRepo, Mode, PathSet, TestTarget, envify};
+use crate::{CLang, CodegenBackendKind, GitRepo, Mode, PathSet, TestTarget, envify, exit};
 
 mod compiletest;
 pub mod failed_tests;
diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index 267e21c599728..59307b515a6de 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -22,7 +22,6 @@ use std::sync::{Arc, Mutex};
 use std::{cmp, env, fs};
 
 use build_helper::ci::CiEnv;
-use build_helper::exit;
 use build_helper::git::{GitConfig, PathFreshness, check_path_modifications};
 use serde::Deserialize;
 #[cfg(feature = "tracing")]
@@ -57,8 +56,10 @@ use crate::core::download::{
 };
 use crate::utils::channel;
 use crate::utils::exec::{ExecutionContext, command};
-use crate::utils::helpers::{exe, get_host_target};
-use crate::{CodegenBackendKind, GitInfo, OnceLock, TargetSelection, check_ci_llvm, helpers, t};
+use crate::utils::helpers::{exe, fail, get_host_target};
+use crate::{
+    CodegenBackendKind, GitInfo, OnceLock, TargetSelection, check_ci_llvm, exit, helpers, t,
+};
 
 /// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic.
 /// This means they can be modified and changes to these paths should never trigger a compiler build
@@ -2173,7 +2174,7 @@ fn postprocess_toml(
             }
         }
         eprintln!("failed to parse override `{option}`: `{err}");
-        exit!(2)
+        exit!(2);
     }
     toml.merge(None, &mut Default::default(), override_toml, ReplaceOpt::Override);
 }
@@ -2195,8 +2196,6 @@ pub fn check_stage0_version(
     src_dir: &Path,
     exec_ctx: &ExecutionContext,
 ) {
-    use build_helper::util::fail;
-
     if exec_ctx.dry_run() {
         return;
     }
diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs
index bf171f1de34e0..f5ab76b3078d3 100644
--- a/src/bootstrap/src/core/config/flags.rs
+++ b/src/bootstrap/src/core/config/flags.rs
@@ -757,7 +757,7 @@ pub fn get_completion(shell: &dyn Generator, path: &Path) -> Option<String> {
     } else {
         std::fs::read_to_string(path).unwrap_or_else(|_| {
             eprintln!("couldn't read {}", path.display());
-            crate::exit!(1)
+            crate::exit!(1);
         })
     };
     let mut buf = Vec::new();
diff --git a/src/bootstrap/src/core/config/mod.rs b/src/bootstrap/src/core/config/mod.rs
index c29849cf5a5d6..c07e7d28fbb26 100644
--- a/src/bootstrap/src/core/config/mod.rs
+++ b/src/bootstrap/src/core/config/mod.rs
@@ -30,7 +30,6 @@ pub mod toml;
 use std::collections::HashSet;
 use std::path::PathBuf;
 
-use build_helper::exit;
 pub use config::*;
 use serde::de::Unexpected;
 use serde::{Deserialize, Deserializer};
@@ -41,8 +40,8 @@ pub use toml::change_id::ChangeId;
 pub use toml::rust::BootstrapOverrideLld;
 pub use toml::target::Target;
 
-use crate::Display;
 use crate::str::FromStr;
+use crate::{Display, exit};
 
 // We are using a decl macro instead of a derive proc macro here to reduce the compile time of bootstrap.
 #[macro_export]
diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs
index 2ed7d55b00da6..078012be72c59 100644
--- a/src/bootstrap/src/core/download.rs
+++ b/src/bootstrap/src/core/download.rs
@@ -248,11 +248,11 @@ impl Config {
 
     #[cfg(not(test))]
     pub(crate) fn maybe_download_ci_llvm(&self) {
-        use build_helper::exit;
         use build_helper::git::PathFreshness;
 
         use crate::core::build_steps::llvm::detect_llvm_freshness;
         use crate::core::config::toml::llvm::check_incompatible_options_for_ci_llvm;
+        use crate::exit;
 
         if !self.llvm_from_ci {
             return;
diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs
index 64993224a36cc..f7ffc682e2bd7 100644
--- a/src/bootstrap/src/lib.rs
+++ b/src/bootstrap/src/lib.rs
@@ -26,7 +26,6 @@ use std::time::{Instant, SystemTime};
 use std::{env, fs, io, str};
 
 use build_helper::ci::gha;
-use build_helper::exit;
 use cc::Tool;
 use termcolor::{ColorChoice, StandardStream, WriteColor};
 use utils::build_stamp::BuildStamp;
@@ -2149,3 +2148,10 @@ pub fn prepare_behaviour_dump_dir(build: &Build) {
         t!(INITIALIZED.set(true));
     }
 }
+
+#[macro_export]
+macro_rules! exit {
+    ($code:expr) => {
+        $crate::utils::helpers::detail_exit($code, cfg!(test));
+    };
+}
diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs
index f9b0d2dbb2095..d3a7c4ec6e1a1 100644
--- a/src/bootstrap/src/utils/exec.rs
+++ b/src/bootstrap/src/utils/exec.rs
@@ -23,10 +23,9 @@ use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 
 use build_helper::drop_bomb::DropBomb;
-use build_helper::exit;
 
 use crate::core::config::DryRun;
-use crate::{PathBuf, t};
+use crate::{PathBuf, exit, t};
 
 /// What should be done when the command fails.
 #[derive(Debug, Copy, Clone)]
diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs
index db740d8dcf624..f0a39b0801ad2 100644
--- a/src/bootstrap/src/utils/helpers.rs
+++ b/src/bootstrap/src/utils/helpers.rs
@@ -10,13 +10,14 @@ use std::thread::panicking;
 use std::time::{Instant, SystemTime, UNIX_EPOCH};
 use std::{env, fs, io, panic, str};
 
+use build_helper::ci::CiEnv;
 use object::read::archive::ArchiveFile;
 
-use crate::BootstrapOverrideLld;
 use crate::core::builder::Builder;
 use crate::core::config::{Config, TargetSelection};
 use crate::utils::exec::{BootstrapCommand, command};
 pub use crate::utils::shared_helpers::{dylib_path, dylib_path_var};
+use crate::{BootstrapOverrideLld, StepStack};
 
 #[cfg(test)]
 mod tests;
@@ -562,3 +563,35 @@ pub fn set_file_times<P: AsRef<Path>>(path: P, times: fs::FileTimes) -> io::Resu
     };
     f.set_times(times)
 }
+
+/// If code is not 0 (successful exit status), exit status is 101 (rust's default error code.)
+/// If `is_test` true and code is an error code, it will cause a panic.
+pub fn detail_exit(code: i32, is_test: bool) -> ! {
+    // if in test and code is an error code, panic with status code provided
+    if is_test {
+        panic!("status code: {code}");
+    } else {
+        // If we're in CI, print the current bootstrap invocation command, to make it easier to
+        // figure out what exactly has failed.
+        if CiEnv::is_ci() {
+            // Skip the first argument, as it will be some absolute path to the bootstrap binary.
+            let bootstrap_args =
+                std::env::args().skip(1).map(|a| a.to_string()).collect::<Vec<_>>().join(" ");
+            eprintln!("Bootstrap failed while executing `{bootstrap_args}`");
+            eprintln!("Currently active steps:");
+            StepStack::with_current(|stack| {
+                for step in stack.get_active_steps() {
+                    eprintln!("{} at {}", step.info, step.location);
+                }
+            });
+        }
+
+        // otherwise, exit with provided status code
+        std::process::exit(code);
+    }
+}
+
+pub fn fail(s: &str) -> ! {
+    eprintln!("\n\n{s}\n\n");
+    detail_exit(1, cfg!(test));
+}
diff --git a/src/build_helper/src/util.rs b/src/build_helper/src/util.rs
index 1bdbb7515e252..6866d4f055846 100644
--- a/src/build_helper/src/util.rs
+++ b/src/build_helper/src/util.rs
@@ -1,63 +1,6 @@
 use std::fs::File;
 use std::io::{BufRead, BufReader};
 use std::path::Path;
-use std::process::Command;
-
-use crate::ci::CiEnv;
-
-/// Invokes `build_helper::util::detail_exit` with `cfg!(test)`
-///
-/// This is a macro instead of a function so that it uses `cfg(test)` in the *calling* crate, not in build helper.
-#[macro_export]
-macro_rules! exit {
-    ($code:expr) => {
-        $crate::util::detail_exit($code, cfg!(test));
-    };
-}
-
-/// If code is not 0 (successful exit status), exit status is 101 (rust's default error code.)
-/// If `is_test` true and code is an error code, it will cause a panic.
-pub fn detail_exit(code: i32, is_test: bool) -> ! {
-    // if in test and code is an error code, panic with status code provided
-    if is_test {
-        panic!("status code: {code}");
-    } else {
-        // If we're in CI, print the current bootstrap invocation command, to make it easier to
-        // figure out what exactly has failed.
-        if CiEnv::is_ci() {
-            // Skip the first argument, as it will be some absolute path to the bootstrap binary.
-            let bootstrap_args =
-                std::env::args().skip(1).map(|a| a.to_string()).collect::<Vec<_>>().join(" ");
-            eprintln!("Bootstrap failed while executing `{bootstrap_args}`");
-        }
-
-        // otherwise, exit with provided status code
-        std::process::exit(code);
-    }
-}
-
-pub fn fail(s: &str) -> ! {
-    eprintln!("\n\n{s}\n\n");
-    detail_exit(1, cfg!(test));
-}
-
-pub fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> Result<(), ()> {
-    let status = match cmd.status() {
-        Ok(status) => status,
-        Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")),
-    };
-    if !status.success() {
-        if print_cmd_on_fail {
-            println!(
-                "\n\ncommand did not execute successfully: {cmd:?}\n\
-                 expected success, got: {status}\n\n"
-            );
-        }
-        Err(())
-    } else {
-        Ok(())
-    }
-}
 
 /// Returns the submodule paths from the `.gitmodules` file in the given directory.
 pub fn parse_gitmodules(target_dir: &Path) -> Vec<String> {
diff --git a/src/tools/rustdoc-gui-test/src/main.rs b/src/tools/rustdoc-gui-test/src/main.rs
index 47cbd4fb20a70..043526b18c187 100644
--- a/src/tools/rustdoc-gui-test/src/main.rs
+++ b/src/tools/rustdoc-gui-test/src/main.rs
@@ -4,7 +4,6 @@ use std::process::Command;
 use std::sync::Arc;
 
 use build_helper::npm;
-use build_helper::util::try_run;
 use compiletest::rustdoc_gui_test::RustdocGuiTestProps;
 use config::Config;
 
@@ -93,3 +92,26 @@ fn main() -> Result<(), ()> {
 
     try_run(&mut command, config.verbose)
 }
+
+fn fail(s: &str) -> ! {
+    eprintln!("\n\n{s}\n\n");
+    std::process::exit(1);
+}
+
+fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> Result<(), ()> {
+    let status = match cmd.status() {
+        Ok(status) => status,
+        Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")),
+    };
+    if !status.success() {
+        if print_cmd_on_fail {
+            println!(
+                "\n\ncommand did not execute successfully: {cmd:?}\n\
+                 expected success, got: {status}\n\n"
+            );
+        }
+        Err(())
+    } else {
+        Ok(())
+    }
+}

From 5a870f7cc0897dbc827de5df7e4259ace54df9e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Fri, 10 Jul 2026 08:41:02 +0200
Subject: [PATCH 16/16] Print a loud error if the configured LLDB cannot be
 found

---
 src/bootstrap/src/core/debuggers/lldb.rs | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/bootstrap/src/core/debuggers/lldb.rs b/src/bootstrap/src/core/debuggers/lldb.rs
index 230239512da8d..9cf8b40d751e5 100644
--- a/src/bootstrap/src/core/debuggers/lldb.rs
+++ b/src/bootstrap/src/core/debuggers/lldb.rs
@@ -14,12 +14,16 @@ pub(crate) fn discover_lldb(builder: &Builder<'_>) -> Option<Lldb> {
     // explicit opt-in or configuration.
     let lldb_exe = builder.config.lldb.clone().unwrap_or_else(|| PathBuf::from("lldb"));
 
-    let lldb_version = command(&lldb_exe)
-        .allow_failure()
-        .arg("--version")
-        .run_capture(builder)
-        .stdout_if_ok()
-        .filter(|v| !v.trim().is_empty())?;
+    let mut cmd = command(&lldb_exe);
+    cmd.arg("--version");
+
+    // If a path to a LLDB binary was provided, it has to exist and return some version, to avoid
+    // silent failures.
+    let explicitly_set_lldb = builder.config.lldb.is_some();
+    if !explicitly_set_lldb {
+        cmd = cmd.allow_failure();
+    }
+    let lldb_version = cmd.run_capture(builder).stdout_if_ok().filter(|v| !v.trim().is_empty())?;
 
     Some(Lldb { lldb_exe, lldb_version })
 }