From 085678ad35d740494b98aa9774bae60ce824550b Mon Sep 17 00:00:00 2001 From: Flakebi Date: Tue, 1 Sep 2026 09:40:37 +0200 Subject: [PATCH 1/3] Add address_space and byref to abi PassMode::Indirect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both will be used by the amdgpu target to implement the `gpu-kernel` ABI. `address_space` specifies the address space of an indirect argument. `AmdgpuKernelArg` translates to LLVM’s byref, which is similar to on_stack/byval, however, there is no extra copy made, the pointer may not point to the stack but can point to some other address space, and the passed argument should not be modified. byval and byref are mutually exclusive, so change on_stack to an enum with the new states, Pointer (none), OnStack and AmdgpuKernelArg. --- compiler/rustc_abi/src/layout/ty.rs | 4 +- .../src/abi/pass_mode.rs | 24 ++-- .../src/abi/returning.rs | 17 +-- compiler/rustc_codegen_gcc/src/abi.rs | 33 ++++- compiler/rustc_codegen_llvm/src/abi.rs | 111 +++++++++++++--- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + compiler/rustc_codegen_llvm/src/llvm/mod.rs | 4 + compiler/rustc_codegen_ssa/src/mir/block.rs | 48 ++++--- compiler/rustc_codegen_ssa/src/mir/mod.rs | 16 ++- .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 5 + .../src/deduce_param_attrs.rs | 2 +- compiler/rustc_public/src/abi.rs | 18 ++- .../src/unstable/convert/stable/abi.rs | 35 +++-- compiler/rustc_target/src/callconv/mod.rs | 125 ++++++++++++++---- compiler/rustc_target/src/callconv/x86.rs | 5 +- compiler/rustc_target/src/callconv/xtensa.rs | 6 +- compiler/rustc_ty_utils/src/abi.rs | 14 +- tests/assembly-llvm/tail-call-indirect.rs | 6 +- tests/ui-fulldeps/rustc_public/check_abi.rs | 8 +- .../rustc_public/check_abi_cast.rs | 4 +- tests/ui/abi/c-zst.powerpc-linux.stderr | 3 +- tests/ui/abi/c-zst.s390x-linux.stderr | 3 +- tests/ui/abi/c-zst.sparc64-linux.stderr | 3 +- .../ui/abi/c-zst.x86_64-pc-windows-gnu.stderr | 3 +- tests/ui/abi/debug.generic.stderr | 6 +- tests/ui/abi/debug.loongarch64.stderr | 6 +- tests/ui/abi/debug.riscv64.stderr | 6 +- tests/ui/abi/pass-indirectly-attr.rs | 2 +- tests/ui/abi/pass-indirectly-attr.stderr | 3 +- .../pass-by-value-abi.aarch64.stderr | 3 +- tests/ui/c-variadic/pass-by-value-abi.rs | 8 +- .../pass-by-value-abi.x86_64.stderr | 9 +- tests/ui/explicit-tail-calls/indirect.rs | 10 +- 33 files changed, 402 insertions(+), 149 deletions(-) diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs index 3b3a58697b205..54dede083f595 100644 --- a/compiler/rustc_abi/src/layout/ty.rs +++ b/compiler/rustc_abi/src/layout/ty.rs @@ -253,8 +253,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> { } /// If this method returns `true`, then this type should always have a `PassMode` of - /// `Indirect { on_stack: false, .. }` when being used as the argument type of a function with a - /// non-Rustic ABI (this is true for structs annotated with the + /// `Indirect { mode: IndirectMode::Pointer, .. }` when being used as the argument type of a + /// function with a non-Rustic ABI (this is true for structs annotated with the /// `#[rustc_pass_indirectly_in_non_rustic_abis]` attribute). /// /// This is used to replicate some of the behaviour of C array-to-pointer decay; however unlike diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs index 1c552ca1a9c32..48ffc43c5cfa1 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs @@ -3,7 +3,7 @@ use cranelift_codegen::ir::ArgumentPurpose; use rustc_abi::{Reg, RegKind}; use rustc_target::callconv::{ - ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, + ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, IndirectMode, PassMode, }; use smallvec::{SmallVec, smallvec}; @@ -126,8 +126,12 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { assert_eq!(pad_i32_count, 0, "padding support not yet implemented"); cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect() } - PassMode::Indirect { attrs, meta_attrs: None, on_stack } => { - if on_stack { + PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => { + assert!( + mode != IndirectMode::AmdgpuKernelArg, + "unsupported amdgpu kernel argument" + ); + if mode == IndirectMode::OnStack { // Abi requires aligning struct size to pointer size let size = self.layout.size.align_to(tcx.data_layout.pointer_align().abi); let size = u32::try_from(size.bytes()).unwrap(); @@ -139,8 +143,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { smallvec![apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)] } } - PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { - assert!(!on_stack); + PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), address_space: _, mode } => { + assert!(mode == IndirectMode::Pointer); smallvec![ apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), meta_attrs), @@ -184,8 +188,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { None, cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(), ), - PassMode::Indirect { attrs, meta_attrs: None, on_stack } => { - assert!(!on_stack); + PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => { + assert!(mode == IndirectMode::Pointer); ( Some(apply_attrs_to_abi_param( AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn), @@ -194,7 +198,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { vec![], ) } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { unreachable!("unsized return value") } } @@ -324,7 +328,7 @@ pub(super) fn cvalue_for_param<'tcx>( PassMode::Cast { ref cast, .. } => { from_casted_value(fx, &block_params, arg_abi.layout, cast) } - PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode: _ } => { assert_eq!(block_params.len(), 1, "{:?}", block_params); if let Some(pointee_align) = attrs.pointee_align && pointee_align < arg_abi.layout.align.abi @@ -342,7 +346,7 @@ pub(super) fn cvalue_for_param<'tcx>( CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout) } } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { assert_eq!(block_params.len(), 2, "{:?}", block_params); CValue::by_ref_unsized(Pointer::new(block_params[0]), block_params[1], arg_abi.layout) } diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs index 36087f96dd776..7f4ee9435b506 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs @@ -17,12 +17,12 @@ pub(super) fn codegen_return_param<'tcx>( let is_ssa = ssa_analyzed[RETURN_PLACE].is_ssa(fx, fx.fn_abi.ret.layout.ty); (super::make_local_place(fx, RETURN_PLACE, fx.fn_abi.ret.layout, is_ssa), smallvec![]) } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => { let ret_param = block_params_iter.next().unwrap(); assert_eq!(fx.bcx.func.dfg.value_type(ret_param), fx.pointer_type); (CPlace::for_ptr(Pointer::new(ret_param), fx.fn_abi.ret.layout), smallvec![ret_param]) } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { unreachable!("unsized return value") } }; @@ -50,7 +50,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( ) { let (ret_temp_place, return_ptr) = match ret_arg_abi.mode { PassMode::Ignore => (None, None), - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => { if let Some(ret_ptr) = ret_place.try_to_ptr() { // This is an optimization to prevent unnecessary copies of the return value when // the return place is already a memory place as opposed to a register. @@ -61,7 +61,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( (Some(place), Some(place.to_ptr().get_addr(fx))) } } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { unreachable!("unsized return value") } PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None), @@ -86,14 +86,14 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast); ret_place.write_cvalue(fx, result); } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => { if let Some(ret_temp_place) = ret_temp_place { // If ret_temp_place is None, it is not necessary to copy the return value. let ret_temp_value = ret_temp_place.to_cvalue(fx); ret_place.write_cvalue(fx, ret_temp_value); } } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { unreachable!("unsized return value") } } @@ -102,10 +102,11 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( /// Codegen a return instruction with the right return value(s) if any. pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) { match fx.fn_abi.ret.mode { - PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + PassMode::Ignore + | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => { fx.bcx.ins().return_(&[]); } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { unreachable!("unsized return value") } PassMode::Direct(_) => { diff --git a/compiler/rustc_codegen_gcc/src/abi.rs b/compiler/rustc_codegen_gcc/src/abi.rs index 6a05f1cbbeef1..b5834ca57ebe1 100644 --- a/compiler/rustc_codegen_gcc/src/abi.rs +++ b/compiler/rustc_codegen_gcc/src/abi.rs @@ -11,7 +11,7 @@ use rustc_middle::ty::layout::LayoutOf; #[cfg(feature = "master")] use rustc_session::{Session, config}; use rustc_span::bug; -use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode}; #[cfg(feature = "master")] use rustc_target::spec::Arch; @@ -178,19 +178,42 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { let ty = cast.gcc_type(cx); apply_attrs(ty, &cast.attrs, argument_tys.len()) } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => { + PassMode::Indirect { + attrs: _, + meta_attrs: None, + address_space: _, + mode: IndirectMode::OnStack, + } => { // This is a "byval" argument, so we don't apply the `restrict` attribute on it. on_stack_param_indices.insert(argument_tys.len()); arg.layout.gcc_type(cx) } + PassMode::Indirect { + attrs: _, + meta_attrs: None, + address_space: _, + mode: IndirectMode::AmdgpuKernelArg, + } => { + unimplemented!("unsupported amdgpu kernel argument") + } PassMode::Direct(attrs) => { apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len()) } - PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { + PassMode::Indirect { + attrs, + meta_attrs: None, + address_space: _, + mode: IndirectMode::Pointer, + } => { apply_attrs(cx.type_ptr_to(arg.layout.gcc_type(cx)), &attrs, argument_tys.len()) } - PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { - assert!(!on_stack); + PassMode::Indirect { + attrs, + meta_attrs: Some(meta_attrs), + address_space: _, + mode, + } => { + assert!(mode == IndirectMode::Pointer); // Construct the type of a (wide) pointer to `ty`, and pass its two fields. // Any two ABI-compatible unsized types have the same metadata type and // moreover the same metadata value leads to the same dynamic size and diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index a45138849e4e0..703986fdab3fa 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -15,7 +15,7 @@ use rustc_middle::ty::layout::LayoutOf; use rustc_session::{Session, config}; use rustc_span::bug; use rustc_target::callconv::{ - ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode, + ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, IndirectMode, PassMode, }; use rustc_target::spec::{Arch, SanitizerSet}; use smallvec::SmallVec; @@ -242,12 +242,12 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { match &self.mode { PassMode::Ignore => {} // Sized indirect arguments - PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode: _ } => { let align = attrs.pointee_align.unwrap_or(self.layout.align.abi); OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst); } // Unsized indirect arguments cannot be stored - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Cast { cast, pad_i32_count: _ } => { @@ -303,11 +303,11 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { PassMode::Pair(..) => { OperandValue::Pair(next(), next()).store(bx, dst); } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Direct(_) - | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } + | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } | PassMode::Cast { .. } => { let next_arg = next(); self.store(bx, next_arg, dst); @@ -368,8 +368,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Ignore => cx.type_void(), PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), PassMode::Cast { cast, pad_i32_count: _ } => cast.llvm_type(cx), - PassMode::Indirect { .. } => { - llargument_tys.push(cx.type_ptr()); + PassMode::Indirect { address_space, .. } => { + let ty = if let Some(address_space) = address_space { + cx.type_ptr_ext(*address_space) + } else { + cx.type_ptr() + }; + llargument_tys.push(ty); cx.type_void() } }; @@ -394,7 +399,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { // Construct the type of a (wide) pointer to `ty`, and pass its two fields. // Any two ABI-compatible unsized types have the same metadata type and // moreover the same metadata value leads to the same dynamic size and @@ -405,7 +410,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(), + PassMode::Indirect { attrs: _, meta_attrs: None, address_space, mode: _ } => { + if let Some(address_space) = address_space { + cx.type_ptr_ext(*address_space) + } else { + cx.type_ptr() + } + } PassMode::Cast { cast, pad_i32_count } => { // Add padding. llargument_tys.extend(std::iter::repeat_n( @@ -495,8 +506,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { apply_range_attr(llvm::AttributePlace::ReturnValue, scalar); } } - PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { - assert!(!on_stack); + PassMode::Indirect { attrs, meta_attrs: _, address_space: _, mode } => { + assert!(*mode == IndirectMode::Pointer); let i = apply(attrs); let sret = llvm::CreateStructRetAttr( cx.llcx, @@ -522,7 +533,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { for arg in self.args.iter() { match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => { + PassMode::Indirect { + attrs, + meta_attrs: None, + address_space: _, + mode: IndirectMode::OnStack, + } => { let i = apply(attrs); let byval = llvm::CreateByValAttr( cx.llcx, @@ -530,13 +546,31 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } + PassMode::Indirect { + attrs, + meta_attrs: None, + address_space: _, + mode: IndirectMode::AmdgpuKernelArg, + } => { + let i = apply(attrs); + let byref = llvm::CreateByRefAttr( + cx.llcx, + cx.type_array(cx.type_i8(), arg.layout.size.bytes()), + ); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byref]); + } PassMode::Direct(attrs) => { let i = apply(attrs); if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { apply_range_attr(llvm::AttributePlace::Argument(i), scalar); } } - PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { + PassMode::Indirect { + attrs, + meta_attrs: None, + address_space: _, + mode: IndirectMode::Pointer, + } => { let i = apply(attrs); if cx.sess().opts.optimize != config::OptLevel::No { attributes::apply_to_llfn( @@ -546,8 +580,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { ); } } - PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { - assert!(!on_stack); + PassMode::Indirect { + attrs, + meta_attrs: Some(meta_attrs), + address_space: _, + mode, + } => { + assert!(*mode == IndirectMode::Pointer); apply(attrs); apply(meta_attrs); } @@ -625,8 +664,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Direct(attrs) => { attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); } - PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { - assert!(!on_stack); + PassMode::Indirect { attrs, meta_attrs: _, address_space: _, mode } => { + assert!(*mode == IndirectMode::Pointer); let i = apply(bx.cx, attrs); let sret = llvm::CreateStructRetAttr( bx.cx.llcx, @@ -646,7 +685,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { for arg in self.args.iter() { match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => { + PassMode::Indirect { + attrs, + meta_attrs: None, + address_space: _, + mode: IndirectMode::OnStack, + } => { let i = apply(bx.cx, attrs); let byval = llvm::CreateByValAttr( bx.cx.llcx, @@ -658,11 +702,38 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { &[byval], ); } + PassMode::Indirect { + attrs, + meta_attrs: None, + address_space: _, + mode: IndirectMode::AmdgpuKernelArg, + } => { + let i = apply(bx.cx, attrs); + let byref = llvm::CreateByRefAttr( + bx.cx.llcx, + bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()), + ); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(i), + &[byref], + ); + } PassMode::Direct(attrs) - | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { + | PassMode::Indirect { + attrs, + meta_attrs: None, + address_space: _, + mode: IndirectMode::Pointer, + } => { apply(bx.cx, attrs); } - PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack: _ } => { + PassMode::Indirect { + attrs, + meta_attrs: Some(meta_attrs), + address_space: _, + mode: _, + } => { apply(bx.cx, attrs); apply(bx.cx, meta_attrs); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index d1cdf7bada0b1..63fcdf8dcbd9c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2015,6 +2015,7 @@ unsafe extern "C" { pub(crate) fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; pub(crate) fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; pub(crate) fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub(crate) fn LLVMRustCreateByRefAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub(crate) fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub(crate) fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub(crate) fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 5452f4abc5c33..89e4d60656d34 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -122,6 +122,10 @@ pub(crate) fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll At unsafe { LLVMRustCreateByValAttr(llcx, ty) } } +pub(crate) fn CreateByRefAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { + unsafe { LLVMRustCreateByRefAttr(llcx, ty) } +} + pub(crate) fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { unsafe { LLVMRustCreateStructRetAttr(llcx, ty) } } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 6b0def4ffa182..f99009a0f4243 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -18,7 +18,7 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt}; use rustc_session::config::OptLevel; use rustc_span::{Span, Spanned, bug, span_bug}; -use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode}; use tracing::{debug, info}; use super::operand::OperandRef; @@ -1257,7 +1257,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { (args, None) }; - // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments. + // Special logic for tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments. // // Normally an indirect argument that is allocated in the caller's stack frame // would be passed as a pointer into the callee's stack frame. @@ -1282,10 +1282,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let mut tail_call_temporaries = vec![]; if kind == CallKind::Tail { tail_call_temporaries = vec![None; first_args.len()]; - // Copy the arguments that use `PassMode::Indirect { on_stack: false , ..}` + // Copy the arguments that use `PassMode::Indirect { mode: IndirectMode::Pointer , ..}` // to temporary stack allocations. See the comment above. for (i, arg) in first_args.iter().enumerate() { - if !matches!(fn_abi.args[i].mode, PassMode::Indirect { on_stack: false, .. }) { + if !matches!( + fn_abi.args[i].mode, + PassMode::Indirect { mode: IndirectMode::Pointer, .. } + ) { continue; } @@ -1353,10 +1356,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - let by_move = if let PassMode::Indirect { on_stack: false, .. } = fn_abi.args[i].mode + let by_move = if let PassMode::Indirect { mode: IndirectMode::Pointer, .. } = + fn_abi.args[i].mode && kind == CallKind::Tail { - // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments. + // Special logic for tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments. // // Normally an indirect argument that is allocated in the caller's stack frame // would be passed as a pointer into the callee's stack frame. @@ -1977,14 +1981,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } _ => bug!("codegen_argument: {:?} invalid for pair argument", op), }, - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => match op.val { - Ref(PlaceValue { llval: a, llextra: Some(b), .. }) => { - llargs.push(a); - llargs.push(b); - return; + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { + match op.val { + Ref(PlaceValue { llval: a, llextra: Some(b), .. }) => { + llargs.push(a); + llargs.push(b); + return; + } + _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op), } - _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op), - }, + } _ => {} } @@ -2014,7 +2020,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { PassMode::Ignore | PassMode::Pair(..) => unreachable!("handled above"), }, Ref(op_place_val) => match arg.mode { - PassMode::Indirect { attrs, on_stack, .. } => { + PassMode::Indirect { attrs, mode, .. } => { + if mode == IndirectMode::AmdgpuKernelArg { + bug!("{op:?} passed as amdgpu kernel argument with abi {arg:?}"); + } // For `foo(packed.large_field)`, and types with <4 byte alignment on x86, // alignment requirements may be higher than the type's alignment, so copy // to a higher-aligned alloca. @@ -2023,7 +2032,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { None => arg.layout.align.abi, }; // Copy to an alloca when the argument is neither by-val nor by-move. - if op_place_val.align < required_align || (!on_stack && !by_move) { + if op_place_val.align < required_align + || (mode == IndirectMode::Pointer && !by_move) + { let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align); bx.lifetime_start(scratch.llval, arg.layout.size); op.store_with_annotation(bx, scratch.with_type(arg.layout)); @@ -2036,8 +2047,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => (op_place_val.llval, op_place_val.align, true), }, ZeroSized => match arg.mode { - PassMode::Indirect { on_stack, .. } => { - if on_stack { + PassMode::Indirect { mode, .. } => { + if mode == IndirectMode::AmdgpuKernelArg { + bug!("{op:?} passed as amdgpu kernel argument with abi {arg:?}"); + } + if mode == IndirectMode::OnStack { // It doesn't seem like any target can have `byval` ZSTs, so this assert // is here to replace a would-be untested codepath. bug!("ZST {op:?} passed on stack with abi {arg:?}"); diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index b5cecf4b5c434..aefa8356536dc 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -8,7 +8,7 @@ use rustc_middle::mir::{Body, Local, UnwindTerminateReason, traversal}; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; use rustc_span::{ErrorGuaranteed, bug, span_bug}; -use rustc_target::callconv::{FnAbi, PassMode}; +use rustc_target::callconv::{FnAbi, IndirectMode, PassMode}; use tracing::{debug, instrument}; use crate::base; @@ -561,15 +561,21 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( match arg.mode { // Sized indirect arguments - PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => { // Don't copy an indirect argument to an alloca, the caller already put it // in a temporary alloca and gave it up. + // AmdgpuKernelArg/byref arguments must not be modified, so always create a + // local alloca for them. + // If the argument is underaligned, then we need to copy it to a higher-aligned + // alloca. // FIXME: lifetimes + let mut needs_alloca = mode == IndirectMode::AmdgpuKernelArg; if let Some(pointee_align) = attrs.pointee_align && pointee_align < arg.layout.align.abi { - // ...unless the argument is underaligned, then we need to copy it to - // a higher-aligned alloca. + needs_alloca = true; + } + if needs_alloca { let tmp = PlaceRef::alloca(bx, arg.layout); bx.store_fn_arg(arg, &mut llarg_idx, tmp); LocalRef::Place(tmp) @@ -580,7 +586,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } // Unsized indirect arguments - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => { // As the storage for the indirect argument lives during // the whole function call, we just copy the wide pointer. let llarg = bx.get_param(llarg_idx); diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 161b5bdb952d3..bc8fa60b66a52 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -480,6 +480,11 @@ extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C, return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty))); } +extern "C" LLVMAttributeRef LLVMRustCreateByRefAttr(LLVMContextRef C, + LLVMTypeRef Ty) { + return wrap(Attribute::getWithByRefType(*unwrap(C), unwrap(Ty))); +} + extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, LLVMTypeRef Ty) { return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty))); diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs index 5bba125aefc58..8814670ca4300 100644 --- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs +++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs @@ -135,7 +135,7 @@ impl<'tcx> Visitor<'tcx> for DeduceParamAttrs { } // Like a call, but more conservative because the backend may introduce writes to an - // argument if the argument is passed as `PassMode::Indirect { on_stack: false, ... }`. + // argument if the argument is passed as `PassMode::Indirect { mode: IndirectMode::Pointer, ... }`. TerminatorKind::TailCall { .. } => { for usage in self.usage.iter_mut() { *usage |= UsageSummary::MUTATE; diff --git a/compiler/rustc_public/src/abi.rs b/compiler/rustc_public/src/abi.rs index b760ed98c7111..c6d1d2c77a13f 100644 --- a/compiler/rustc_public/src/abi.rs +++ b/compiler/rustc_public/src/abi.rs @@ -41,6 +41,19 @@ pub struct ArgAbi { pub mode: PassMode, } +/// Different modes in which indirect arguments can be passed. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize)] +pub enum IndirectMode { + /// Passed as a normal pointer, nothing special. + Pointer, + /// The value is placed at a fixed stack offset rather than passed as a regular pointer + /// argument. + OnStack, + /// Similar to `OnStack` except that the pointer does not necessarily point to the stack, no + /// extra copy is made, and the passed argument should not be modified. + AmdgpuKernelArg, +} + /// How a function argument should be passed in to the target function. /// /// The pass mode is determined by the platform's calling convention and the @@ -74,14 +87,13 @@ pub enum PassMode { /// Pass the argument indirectly via a pointer. /// /// The caller places the value in memory and passes a pointer to it. - /// When `on_stack` is true, the value is placed at a fixed stack offset - /// rather than passed as a regular pointer argument. Indirect { attrs: ArgAttributes, /// Attributes for the metadata pointer (vtable or length) of unsized arguments. /// Only present for unsized types (e.g., `dyn Trait`, `[T]`). meta_attrs: Option, - on_stack: bool, + address_space: Option, + mode: IndirectMode, }, } diff --git a/compiler/rustc_public/src/unstable/convert/stable/abi.rs b/compiler/rustc_public/src/unstable/convert/stable/abi.rs index 766c522958db7..65b8e9bd72761 100644 --- a/compiler/rustc_public/src/unstable/convert/stable/abi.rs +++ b/compiler/rustc_public/src/unstable/convert/stable/abi.rs @@ -11,9 +11,9 @@ use rustc_target::callconv; use crate::IndexedVal; use crate::abi::{ AddressSpace, ArgAbi, ArgAttributes, ArgExtension, CallConvention, CastTarget, FieldsShape, - FloatLength, FnAbi, IntegerLength, IntegerType, Layout, LayoutShape, NumScalableVectors, - PassMode, Primitive, Reg, RegKind, ReprFlags, ReprOptions, Scalar, TagEncoding, TyAndLayout, - Uniform, ValueRepr, VariantFields, VariantsShape, WrappingRange, + FloatLength, FnAbi, IndirectMode, IntegerLength, IntegerType, Layout, LayoutShape, + NumScalableVectors, PassMode, Primitive, Reg, RegKind, ReprFlags, ReprOptions, Scalar, + TagEncoding, TyAndLayout, Uniform, ValueRepr, VariantFields, VariantsShape, WrappingRange, }; use crate::compiler_interface::BridgeTys; use crate::target::MachineSize as Size; @@ -155,6 +155,22 @@ impl<'tcx> Stable<'tcx> for CanonAbi { } } +impl<'tcx> Stable<'tcx> for callconv::IndirectMode { + type T = IndirectMode; + + fn stable<'cx>( + &self, + _tables: &mut Tables<'cx, BridgeTys>, + _cx: &CompilerCtxt<'cx, BridgeTys>, + ) -> Self::T { + match self { + callconv::IndirectMode::Pointer => IndirectMode::Pointer, + callconv::IndirectMode::OnStack => IndirectMode::OnStack, + callconv::IndirectMode::AmdgpuKernelArg => IndirectMode::AmdgpuKernelArg, + } + } +} + impl<'tcx> Stable<'tcx> for callconv::PassMode { type T = PassMode; @@ -172,11 +188,14 @@ impl<'tcx> Stable<'tcx> for callconv::PassMode { callconv::PassMode::Cast { pad_i32_count, cast } => { PassMode::Cast { pad_i32_count: *pad_i32_count, cast: cast.stable(tables, cx) } } - callconv::PassMode::Indirect { attrs, meta_attrs, on_stack } => PassMode::Indirect { - attrs: attrs.stable(tables, cx), - meta_attrs: meta_attrs.map(|a| a.stable(tables, cx)), - on_stack: *on_stack, - }, + callconv::PassMode::Indirect { attrs, meta_attrs, address_space, mode } => { + PassMode::Indirect { + attrs: attrs.stable(tables, cx), + meta_attrs: meta_attrs.map(|a| a.stable(tables, cx)), + address_space: address_space.stable(tables, cx), + mode: mode.stable(tables, cx), + } + } } } } diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 9fe22a3a174b6..474f45b54e9b2 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -36,6 +36,25 @@ mod x86_win32; mod x86_win64; mod xtensa; +/// Different modes in which indirect arguments can be passed. +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, StableHash)] +pub enum IndirectMode { + /// Passed as a normal pointer, nothing special. + Pointer, + /// The value should be passed at a fixed stack offset in accordance to + /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument + /// attribute. The `byval` argument will use a byte array with the same size as the Rust type + /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout), + /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's + /// alignment (if `None`). This means that the alignment will not always + /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info. + OnStack, + /// `AmdgpuKernelArg` behaves similar to `OnStack` except that the pointer does not necessarily + /// point to the stack, no extra copy is made, and the passed argument should not be modified. + /// This corresponds to the `byref` LLVM argument attribute. + AmdgpuKernelArg, +} + #[derive(Clone, PartialEq, Eq, Hash, Debug, StableHash)] pub enum PassMode { /// Ignore the argument. @@ -63,16 +82,17 @@ pub enum PassMode { /// The `meta_attrs` value, if any, is for the metadata (vtable or length) of an unsized /// argument. (This is the only mode that supports unsized arguments.) /// - /// `on_stack` defines that the value should be passed at a fixed stack offset in accordance to - /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument - /// attribute. The `byval` argument will use a byte array with the same size as the Rust type - /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout), - /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's - /// alignment (if `None`). This means that the alignment will not always - /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info. + /// `address_space` specifies if the pointer is in a special address space or the default one. /// - /// `on_stack` cannot be true for unsized arguments, i.e., when `meta_attrs` is `Some`. - Indirect { attrs: ArgAttributes, meta_attrs: Option, on_stack: bool }, + /// `mode` can be a special way to pass an argument indirectly. + /// `OnStack` and `AmdgpuKernelArg` cannot be used for unsized arguments, i.e., when + /// `meta_attrs` is `Some`. + Indirect { + attrs: ArgAttributes, + meta_attrs: Option, + address_space: Option, + mode: IndirectMode, + }, } impl PassMode { @@ -89,13 +109,23 @@ impl PassMode { PassMode::Cast { cast: c2, pad_i32_count: pad2 }, ) => c1.eq_abi(c2) && pad1 == pad2, ( - PassMode::Indirect { attrs: a1, meta_attrs: None, on_stack: s1 }, - PassMode::Indirect { attrs: a2, meta_attrs: None, on_stack: s2 }, - ) => a1.eq_abi(a2) && s1 == s2, + PassMode::Indirect { attrs: a1, meta_attrs: None, address_space: as1, mode: m1 }, + PassMode::Indirect { attrs: a2, meta_attrs: None, address_space: as2, mode: m2 }, + ) => a1.eq_abi(a2) && as1 == as2 && m1 == m2, ( - PassMode::Indirect { attrs: a1, meta_attrs: Some(e1), on_stack: s1 }, - PassMode::Indirect { attrs: a2, meta_attrs: Some(e2), on_stack: s2 }, - ) => a1.eq_abi(a2) && e1.eq_abi(e2) && s1 == s2, + PassMode::Indirect { + attrs: a1, + meta_attrs: Some(e1), + address_space: as1, + mode: m1, + }, + PassMode::Indirect { + attrs: a2, + meta_attrs: Some(e2), + address_space: as2, + mode: m2, + }, + ) => a1.eq_abi(a2) && as1 == as2 && e1.eq_abi(e2) && m1 == m2, _ => false, } } @@ -424,7 +454,7 @@ impl<'a, Ty> ArgAbi<'a, Ty> { let meta_attrs = layout.is_unsized().then_some(ArgAttributes::new()); - PassMode::Indirect { attrs, meta_attrs, on_stack: false } + PassMode::Indirect { attrs, meta_attrs, address_space: None, mode: IndirectMode::Pointer } } /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead. @@ -435,13 +465,31 @@ impl<'a, Ty> ArgAbi<'a, Ty> { PassMode::Direct(_) | PassMode::Pair(_, _) => { self.mode = Self::indirect_pass_mode(&self.layout); } - PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => { + PassMode::Indirect { + attrs: _, + meta_attrs: _, + address_space: _, + mode: IndirectMode::Pointer, + } => { // already indirect } _ => panic!("Tried to make {:?} indirect", self.mode), } } + /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead. + /// This is valid for both sized and unsized arguments. + #[track_caller] + pub fn make_indirect_addrspace(&mut self, addrspace: AddressSpace) { + self.make_indirect(); + match self.mode { + PassMode::Indirect { ref mut address_space, .. } => { + *address_space = Some(addrspace); + } + _ => unreachable!(), + } + } + /// Same as `make_indirect`, but for arguments that are ignored. Only needed for ABIs that pass /// ZSTs indirectly. #[track_caller] @@ -450,7 +498,12 @@ impl<'a, Ty> ArgAbi<'a, Ty> { PassMode::Ignore => { self.mode = Self::indirect_pass_mode(&self.layout); } - PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => { + PassMode::Indirect { + attrs: _, + meta_attrs: _, + address_space: _, + mode: IndirectMode::Pointer, + } => { // already indirect } _ => panic!("Tried to make {:?} indirect (expected `PassMode::Ignore`)", self.mode), @@ -477,8 +530,8 @@ impl<'a, Ty> ArgAbi<'a, Ty> { assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout"); self.make_indirect(); match self.mode { - PassMode::Indirect { ref mut attrs, meta_attrs: _, ref mut on_stack } => { - *on_stack = true; + PassMode::Indirect { ref mut attrs, meta_attrs: _, address_space: _, ref mut mode } => { + *mode = IndirectMode::OnStack; // Some platforms, like 32-bit x86, change the alignment of the type when passing // `byval`. Account for that. @@ -492,6 +545,22 @@ impl<'a, Ty> ArgAbi<'a, Ty> { } } + /// Pass this argument indirectly. + /// This corresponds to the `byref` LLVM argument attribute. + /// + /// `address_space` specifies the address space of the passed pointer. + pub fn pass_amdgpu_kernel_arg(&mut self, addrspace: Option) { + assert!(!self.layout.is_unsized(), "used amdgpu kernel arg ABI for unsized layout"); + self.make_indirect(); + match self.mode { + PassMode::Indirect { attrs: _, meta_attrs: _, ref mut address_space, ref mut mode } => { + *mode = IndirectMode::AmdgpuKernelArg; + *address_space = addrspace; + } + _ => unreachable!(), + } + } + pub fn extend_integer_width_to(&mut self, bits: u64) { // Only integers have signedness if let BackendRepr::Scalar(scalar) = self.layout.backend_repr @@ -545,11 +614,17 @@ impl<'a, Ty> ArgAbi<'a, Ty> { } pub fn is_sized_indirect(&self) -> bool { - matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ }) + matches!( + self.mode, + PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } + ) } pub fn is_unsized_indirect(&self) -> bool { - matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ }) + matches!( + self.mode, + PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } + ) } pub fn is_ignore(&self) -> bool { @@ -834,7 +909,7 @@ impl<'a, Ty> FnAbi<'a, Ty> { // Compute `Aggregate` ABI. let is_indirect_not_on_stack = - matches!(arg.mode, PassMode::Indirect { on_stack: false, .. }); + matches!(arg.mode, PassMode::Indirect { mode: IndirectMode::Pointer, .. }); assert!(is_indirect_not_on_stack); let size = arg.layout.size; @@ -949,7 +1024,7 @@ mod size_asserts { use super::*; // tidy-alphabetical-start - static_assert_size!(ArgAbi<'_, usize>, 56); - static_assert_size!(FnAbi<'_, usize>, 80); + static_assert_size!(ArgAbi<'_, usize>, 64); + static_assert_size!(FnAbi<'_, usize>, 88); // tidy-alphabetical-end } diff --git a/compiler/rustc_target/src/callconv/x86.rs b/compiler/rustc_target/src/callconv/x86.rs index fd608fcf62919..f51e29b34e1d3 100644 --- a/compiler/rustc_target/src/callconv/x86.rs +++ b/compiler/rustc_target/src/callconv/x86.rs @@ -167,12 +167,13 @@ pub(crate) fn fill_inregs<'a, Ty, C>( for arg in fn_abi.args.iter_mut() { let attrs = match arg.mode { - PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + PassMode::Ignore + | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => { continue; } PassMode::Direct(ref mut attrs) => attrs, PassMode::Pair(..) - | PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } + | PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } | PassMode::Cast { .. } => { unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode) } diff --git a/compiler/rustc_target/src/callconv/xtensa.rs b/compiler/rustc_target/src/callconv/xtensa.rs index 4dc9fad650636..49005adeb33c0 100644 --- a/compiler/rustc_target/src/callconv/xtensa.rs +++ b/compiler/rustc_target/src/callconv/xtensa.rs @@ -7,7 +7,7 @@ use rustc_abi::{BackendRepr, HasDataLayout, Size, TyAbiInterface}; -use crate::callconv::{ArgAbi, FnAbi, Reg, Uniform}; +use crate::callconv::{ArgAbi, FnAbi, IndirectMode, Reg, Uniform}; use crate::spec::HasTargetSpec; const NUM_ARG_GPRS: u64 = 6; @@ -29,8 +29,8 @@ where classify_arg_ty(cx, arg, &mut arg_gprs_left, true); // Ret args cannot be passed via stack, we lower to indirect and let the backend handle the invisible reference match arg.mode { - super::PassMode::Indirect { attrs: _, meta_attrs: _, ref mut on_stack } => { - *on_stack = false; + super::PassMode::Indirect { attrs: _, meta_attrs: _, address_space: _, ref mut mode } => { + *mode = IndirectMode::Pointer; } _ => {} } diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 55140d2c5458d..e8f9ded9562d5 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -12,7 +12,9 @@ use rustc_middle::ty::layout::{ use rustc_middle::ty::{self, InstanceKind, ShimKind, Ty, TyCtxt, Unnormalized}; use rustc_span::def_id::DefId; use rustc_span::{DUMMY_SP, bug}; -use rustc_target::callconv::{AbiMap, ArgAbi, ArgAttribute, ArgAttributes, FnAbi, PassMode}; +use rustc_target::callconv::{ + AbiMap, ArgAbi, ArgAttribute, ArgAttributes, FnAbi, IndirectMode, PassMode, +}; use tracing::debug; pub(crate) fn provide(providers: &mut Providers) { @@ -444,15 +446,15 @@ fn fn_abi_sanity_check<'tcx>( // omitted entirely in the calling convention. assert!(arg.is_ignore()); } - if let PassMode::Indirect { on_stack, .. } = arg.mode + if let PassMode::Indirect { mode, .. } = arg.mode && spec_abi != ExternAbi::RustTail { - assert!(!on_stack, "rustic abi {spec_abi:?} shouldn't use on_stack"); + assert!(mode == IndirectMode::Pointer, "rust abi must use plain pointer mode"); } } else if arg.layout.pass_indirectly_in_non_rustic_abis(cx) { assert_matches!( arg.mode, - PassMode::Indirect { on_stack: false, .. }, + PassMode::Indirect { mode: IndirectMode::Pointer, .. }, "the {spec_abi} ABI does not implement `#[rustc_pass_indirectly_in_non_rustic_abis]`" ); } @@ -506,9 +508,9 @@ fn fn_abi_sanity_check<'tcx>( // Indirect returns are arguments from an ABI perspective. fn_arg_attrs_sanity_check(attrs, false); } - PassMode::Indirect { meta_attrs: Some(meta_attrs), attrs, on_stack } => { + PassMode::Indirect { meta_attrs: Some(meta_attrs), attrs, address_space: _, mode } => { // With metadata. Must be unsized and not on the stack. - assert!(arg.layout.is_unsized() && !on_stack); + assert!(arg.layout.is_unsized() && *mode == IndirectMode::Pointer); // Also, must not be `extern` type. let tail = tcx.struct_tail_for_codegen(arg.layout.ty, cx.typing_env); if matches!(tail.kind(), ty::Foreign(..)) { diff --git a/tests/assembly-llvm/tail-call-indirect.rs b/tests/assembly-llvm/tail-call-indirect.rs index 2bc1743a9bafd..918283966b405 100644 --- a/tests/assembly-llvm/tail-call-indirect.rs +++ b/tests/assembly-llvm/tail-call-indirect.rs @@ -10,10 +10,10 @@ #![no_core] #![crate_type = "lib"] -// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments. +// Test tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments. // -// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the -// caller's stack frame. For tail calls, that would be unsound, because the caller's stack +// Normally an indirect argument with `mode: IndirectMode::Pointer` would be passed as a pointer to +// the caller's stack frame. For tail calls, that would be unsound, because the caller's stack // frame is overwritten by the callee's stack frame. // // The solution is to write the argument into the caller's argument place (stored somewhere further diff --git a/tests/ui-fulldeps/rustc_public/check_abi.rs b/tests/ui-fulldeps/rustc_public/check_abi.rs index f6c95fb745409..92312cd4c8712 100644 --- a/tests/ui-fulldeps/rustc_public/check_abi.rs +++ b/tests/ui-fulldeps/rustc_public/check_abi.rs @@ -15,8 +15,8 @@ extern crate rustc_middle; extern crate rustc_public; use rustc_public::abi::{ - ArgAbi, ArgExtension, CallConvention, FieldsShape, IntegerLength, PassMode, Primitive, Scalar, - ValueRepr, VariantsShape, + ArgAbi, ArgExtension, CallConvention, FieldsShape, IndirectMode, IntegerLength, PassMode, + Primitive, Scalar, ValueRepr, VariantsShape, }; use rustc_public::mir::MirVisitor; use rustc_public::mir::mono::Instance; @@ -122,14 +122,14 @@ fn check_primitive(abi: &ArgAbi) { /// Check the return value: `Result`. fn check_result(abi: &ArgAbi) { assert!(abi.ty.kind().is_enum()); - let PassMode::Indirect { ref attrs, ref meta_attrs, on_stack } = abi.mode else { + let PassMode::Indirect { ref attrs, ref meta_attrs, address_space: _, mode } = abi.mode else { panic!("Expected PassMode::Indirect for Result, got: {:?}", abi.mode); }; // Indirect arguments have a pointee alignment (the pointer must be aligned). assert!(attrs.pointee_align().is_some()); // Result is a sized type, so no metadata pointer. assert!(meta_attrs.is_none()); - assert!(!on_stack); + assert!(mode == IndirectMode::Pointer); let layout = abi.layout.shape(); assert!(layout.is_sized()); assert_matches!(layout.fields, FieldsShape::Arbitrary { .. }); diff --git a/tests/ui-fulldeps/rustc_public/check_abi_cast.rs b/tests/ui-fulldeps/rustc_public/check_abi_cast.rs index 0bd4ac684066e..a54abdd5deeaf 100644 --- a/tests/ui-fulldeps/rustc_public/check_abi_cast.rs +++ b/tests/ui-fulldeps/rustc_public/check_abi_cast.rs @@ -23,7 +23,7 @@ use std::convert::TryFrom; use std::io::Write; use std::ops::ControlFlow; -use rustc_public::abi::{CallConvention, PassMode, RegKind}; +use rustc_public::abi::{CallConvention, IndirectMode, PassMode, RegKind}; use rustc_public::mir::mono::Instance; use rustc_public::{CrateDef, ItemKind}; @@ -147,7 +147,7 @@ fn test_abi_cast() -> ControlFlow<()> { } // Fourth TwoWords has no registers left → Indirect (on stack) assert!( - matches!(&abi.args[3].mode, PassMode::Indirect { on_stack: true, .. }), + matches!(&abi.args[3].mode, PassMode::Indirect { mode: IndirectMode::OnStack, .. }), "Expected arg 3 to be Indirect on stack, got: {:?}", abi.args[3].mode ); diff --git a/tests/ui/abi/c-zst.powerpc-linux.stderr b/tests/ui/abi/c-zst.powerpc-linux.stderr index edea2d5772280..cefc2ba085182 100644 --- a/tests/ui/abi/c-zst.powerpc-linux.stderr +++ b/tests/ui/abi/c-zst.powerpc-linux.stderr @@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/abi/c-zst.s390x-linux.stderr b/tests/ui/abi/c-zst.s390x-linux.stderr index edea2d5772280..cefc2ba085182 100644 --- a/tests/ui/abi/c-zst.s390x-linux.stderr +++ b/tests/ui/abi/c-zst.s390x-linux.stderr @@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/abi/c-zst.sparc64-linux.stderr b/tests/ui/abi/c-zst.sparc64-linux.stderr index edea2d5772280..cefc2ba085182 100644 --- a/tests/ui/abi/c-zst.sparc64-linux.stderr +++ b/tests/ui/abi/c-zst.sparc64-linux.stderr @@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr index edea2d5772280..cefc2ba085182 100644 --- a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr +++ b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr @@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr index 6242d93b09534..1793674fa462a 100644 --- a/tests/ui/abi/debug.generic.stderr +++ b/tests/ui/abi/debug.generic.stderr @@ -446,7 +446,8 @@ error: ABIs are not compatible ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], @@ -519,7 +520,8 @@ error: ABIs are not compatible ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/abi/debug.loongarch64.stderr b/tests/ui/abi/debug.loongarch64.stderr index 176c68ecd4c7b..29ec7846101f1 100644 --- a/tests/ui/abi/debug.loongarch64.stderr +++ b/tests/ui/abi/debug.loongarch64.stderr @@ -446,7 +446,8 @@ error: ABIs are not compatible ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], @@ -519,7 +520,8 @@ error: ABIs are not compatible ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr index 176c68ecd4c7b..29ec7846101f1 100644 --- a/tests/ui/abi/debug.riscv64.stderr +++ b/tests/ui/abi/debug.riscv64.stderr @@ -446,7 +446,8 @@ error: ABIs are not compatible ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], @@ -519,7 +520,8 @@ error: ABIs are not compatible ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/abi/pass-indirectly-attr.rs b/tests/ui/abi/pass-indirectly-attr.rs index 54aafc716587c..bb90b8354ea91 100644 --- a/tests/ui/abi/pass-indirectly-attr.rs +++ b/tests/ui/abi/pass-indirectly-attr.rs @@ -20,7 +20,7 @@ pub struct Type(u8); pub extern "C" fn extern_c(_: Type) {} //~^ ERROR fn_abi_of(extern_c) = FnAbi { //~| ERROR mode: Indirect -//~| ERROR on_stack: false, +//~| ERROR mode: Pointer, //~| ERROR conv: C, #[rustc_abi(debug)] diff --git a/tests/ui/abi/pass-indirectly-attr.stderr b/tests/ui/abi/pass-indirectly-attr.stderr index efeec0d86982b..5821e6279bb85 100644 --- a/tests/ui/abi/pass-indirectly-attr.stderr +++ b/tests/ui/abi/pass-indirectly-attr.stderr @@ -48,7 +48,8 @@ error: fn_abi_of(extern_c) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr index 45edd7bc0e0ee..c9e77ac941901 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr +++ b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr @@ -35,7 +35,8 @@ error: fn_abi_of(take_va_list) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/c-variadic/pass-by-value-abi.rs b/tests/ui/c-variadic/pass-by-value-abi.rs index bcca09e90438a..317840601c050 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.rs +++ b/tests/ui/c-variadic/pass-by-value-abi.rs @@ -27,9 +27,9 @@ use std::ffi::VaList; pub extern "C" fn take_va_list(_: VaList<'_>) {} //~^ ERROR fn_abi_of(take_va_list) = FnAbi { //[x86_64]~^^ ERROR mode: Indirect { -//[x86_64]~^^^ ERROR on_stack: false, +//[x86_64]~^^^ ERROR mode: Pointer, //[aarch64]~^^^^ ERROR mode: Indirect { -//[aarch64]~^^^^^ ERROR on_stack: false, +//[aarch64]~^^^^^ ERROR mode: Pointer, //[win]~^^^^^^ ERROR mode: Direct( #[cfg(all(target_arch = "x86_64", not(windows)))] @@ -37,11 +37,11 @@ pub extern "C" fn take_va_list(_: VaList<'_>) {} pub extern "sysv64" fn take_va_list_sysv64(_: VaList<'_>) {} //[x86_64]~^ ERROR fn_abi_of(take_va_list_sysv64) = FnAbi { //[x86_64]~^^ ERROR mode: Indirect { -//[x86_64]~^^^ ERROR on_stack: false, +//[x86_64]~^^^ ERROR mode: Pointer, #[cfg(all(target_arch = "x86_64", not(windows)))] #[rustc_abi(debug)] pub extern "win64" fn take_va_list_win64(_: VaList<'_>) {} //[x86_64]~^ ERROR: fn_abi_of(take_va_list_win64) = FnAbi { //[x86_64]~^^ ERROR mode: Indirect { -//[x86_64]~^^^ ERROR on_stack: false, +//[x86_64]~^^^ ERROR mode: Pointer, diff --git a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr index 1e203b93e66b3..04320a5312361 100644 --- a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr +++ b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr @@ -35,7 +35,8 @@ error: fn_abi_of(take_va_list) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], @@ -113,7 +114,8 @@ error: fn_abi_of(take_va_list_sysv64) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], @@ -193,7 +195,8 @@ error: fn_abi_of(take_va_list_win64) = FnAbi { ), }, meta_attrs: None, - on_stack: false, + address_space: None, + mode: Pointer, }, }, ], diff --git a/tests/ui/explicit-tail-calls/indirect.rs b/tests/ui/explicit-tail-calls/indirect.rs index b3e2613efad25..71107ef420c35 100644 --- a/tests/ui/explicit-tail-calls/indirect.rs +++ b/tests/ui/explicit-tail-calls/indirect.rs @@ -25,17 +25,17 @@ #![feature(explicit_tail_calls)] #![expect(incomplete_features)] -// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments. +// Test tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments. // -// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the -// caller's stack frame. For tail calls, that would be unsound, because the caller's stack +// Normally an indirect argument with `mode: IndirectMode::Pointer` would be passed as a pointer to +// the caller's stack frame. For tail calls, that would be unsound, because the caller's stack // frame is overwritten by the callee's stack frame. // // The solution is to write the argument into the caller's argument place (stored somewhere further // up the stack), and forward that place. // A struct big enough that it is not passed via registers, so that the rust calling convention uses -// `Indirect { on_stack: false, .. }`. +// `Indirect { mode: IndirectMode::Pointer, .. }`. #[repr(C)] #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] pub struct Big([u64; 4]); @@ -79,7 +79,7 @@ fn main() { assert_eq!(update_in_caller(Big::default()), 0 + 2 + 3 + 4); assert_eq!(swapper(u8::MIN, u8::MAX), (u8::MAX, u8::MIN)); - // i128 uses `PassMode::Indirect { on_stack: false, .. }` on x86_64 MSVC. + // i128 uses `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` on x86_64 MSVC. assert_eq!(swapper(i128::MIN, i128::MAX), (i128::MAX, i128::MIN)); assert_eq!(swapper(Big([1; 4]), Big([2; 4])), (Big([2; 4]), Big([1; 4]))); From bf91751cda156af266c0e731089783970c8de368 Mon Sep 17 00:00:00 2001 From: Flakebi Date: Thu, 3 Sep 2026 09:21:39 +0200 Subject: [PATCH 2/3] Pre-commit amdgpu gpu-kernel ABI test --- tests/codegen-llvm/amdgpu-abi/struct-abi.rs | 133 ++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 tests/codegen-llvm/amdgpu-abi/struct-abi.rs diff --git a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs new file mode 100644 index 0000000000000..bf51cbfa7f7a0 --- /dev/null +++ b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs @@ -0,0 +1,133 @@ +//@ add-minicore +//@ compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 -Copt-level=3 +//@ needs-llvm-components: amdgpu +#![feature(no_core, abi_gpu_kernel)] +#![no_core] +#![allow(improper_gpu_kernel_arg)] + +extern crate minicore; +use minicore::num::Complex; + +// Tests from llvm-project/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl + +#[repr(C)] +pub struct SingleElementStructArg { + i: i32, +} + +#[repr(C)] +pub struct NestedSingleElementStructArg { + i: SingleElementStructArg, +} + +#[repr(C)] +pub struct StructArg { + i1: i32, + f: f32, + i2: i32, +} + +#[repr(C)] +pub struct StructPaddingArg { + i1: i8, + f: i64, +} + +#[repr(C)] +pub struct StructOfArraysArg { + i1: [i32; 2], + f1: f32, + i2: [i32; 4], + f2: [f32; 3], + i3: i32, +} + +#[repr(C)] +pub struct StructOfStructsArg { + i1: i32, + f1: f32, + s1: StructArg, + i2: i32, +} + +#[repr(C)] +pub union U { + b1: i32, + b2: f32, +} + +#[repr(C)] +pub struct SingleArrayElementStructArg { + i: [i32; 4], +} + +#[repr(C)] +pub struct SingleStructElementStructArgInner { + i: i32, + b: i64, +} + +#[repr(C)] +pub struct SingleStructElementStructArg { + s: SingleStructElementStructArgInner, +} + +#[repr(C)] +pub struct DifferentSizeTypePair { + l: i64, + i: i32, +} + +// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_nested_single_element_struct_arg( + _: NestedSingleElementStructArg, +) { +} + +// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(12) {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_struct_arg(_: StructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(i8 noundef {{%.+}}, i64 noundef {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_struct_padding_arg(_: StructPaddingArg) {} + +// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(44) {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_struct_of_arrays_arg(_: StructOfArraysArg) {} + +// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(24) {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_struct_of_structs_arg(_: StructOfStructsArg) {} + +// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn test_kernel_union_arg(_: U) {} + +// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(16) {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_single_array_element_struct_arg(_: SingleArrayElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(i32 noundef {{%.+}}, i64 noundef {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_single_struct_element_struct_arg( + _: SingleStructElementStructArg, +) { +} + +// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(i64 noundef {{%.+}}, i32 noundef {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_different_size_type_pair_arg(_: DifferentSizeTypePair) {} + +// CHECK: define amdgpu_kernel void @kernel_complex(float noundef {{%.+}}, float noundef {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_complex(_: Complex) {} + +// CHECK: define amdgpu_kernel void @kernel_slice(ptr noalias nofree noundef nonnull readonly align 4 captures(none) {{%.+}}, i64 noundef range(i64 0, 2305843009213693952) {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_slice(_: &[u32]) {} From c4982542b6f702e25d3cebb55db4c2e29ea51662 Mon Sep 17 00:00:00 2001 From: Flakebi Date: Tue, 15 Sep 2026 10:33:14 +0200 Subject: [PATCH 3/3] Properly implement the gpu-kernel ABI for amdgpu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support to pass structs, arrays and vectors to amdgpu kernels. Scalars and vectors are taken by value, aggregates are passed by byref pointers. Structs containing a single scalar/vector are handled like a scalar. Judging from clang tests, nvptx seems to do somewhat the same, just using byval instead of byref: https://github.com/llvm/llvm-project/blob/3a8affeef4da19d39191aac316e189eca3214a8c/clang/test/CodeGenCUDA/kernel-args.cu I tested a couple of the lit test signatures on real hardware and it seems to work fine. Given the relatively simple implementation, I hope this amount of testing is enough (the C calling convention seems like a worse fit for Rust’s current ABI code, it’s still giving me headaches). --- compiler/rustc_abi/src/lib.rs | 4 + compiler/rustc_target/src/callconv/amdgpu.rs | 82 ++++++++++++++---- tests/codegen-llvm/amdgpu-abi/struct-abi.rs | 88 ++++++++++++++++---- 3 files changed, 139 insertions(+), 35 deletions(-) diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index b056fdc73d40b..79c0309c60aa3 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1772,6 +1772,10 @@ pub struct AddressSpace(pub u32); impl AddressSpace { /// LLVM's `0` address space. pub const ZERO: Self = AddressSpace(0); + /// The address space for constant memory on nvptx and amdgpu. + /// This address space is used e.g. for kernel arguments that are constant throughout the + /// execution. + pub const GPU_CONSTANT: Self = AddressSpace(4); /// The address space for workgroup memory on nvptx and amdgpu. /// See e.g. the `gpu_launch_sized_workgroup_mem` intrinsic for details. pub const GPU_WORKGROUP: Self = AddressSpace(3); diff --git a/compiler/rustc_target/src/callconv/amdgpu.rs b/compiler/rustc_target/src/callconv/amdgpu.rs index 98ab3ce8eb746..7a9eeaba19c96 100644 --- a/compiler/rustc_target/src/callconv/amdgpu.rs +++ b/compiler/rustc_target/src/callconv/amdgpu.rs @@ -1,25 +1,60 @@ -use rustc_abi::{HasDataLayout, TyAbiInterface}; +use rustc_abi::{ + AddressSpace, BackendRepr, CanonAbi, HasDataLayout, Reg, RegKind, TyAbiInterface, TyAndLayout, +}; -use crate::callconv::{ArgAbi, FnAbi}; +use crate::callconv::{FnAbi, Uniform}; -fn classify_ret<'a, Ty, C>(_cx: &C, ret: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - ret.extend_integer_width_to(32); -} +// For reference, see llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp -fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) +/// If the given type is a (potentially nested) struct containing a single scalar, return +/// a `Uniform` for the contained, single element. +fn single_element_struct_to_reg<'a, Ty, C>(cx: &C, ty: TyAndLayout<'a, Ty>) -> Option where Ty: TyAbiInterface<'a, C> + Copy, C: HasDataLayout, { - if arg.layout.pass_indirectly_in_non_rustic_abis(cx) { - arg.make_indirect(); - return; + assert!(ty.is_aggregate(), "Only handles aggregate types"); + if ty.layout.fields.count() != 1 { + return None; + } + let field = ty.field(cx, 0); + match field.backend_repr { + BackendRepr::SimdScalableVector { .. } => panic!("scalable vectors are unsupported"), + BackendRepr::Scalar(_) => { + // Check that the size is the same as the size for ty, so no extra padding + let size = field.layout.size.bytes(); + if ty.layout.size.bytes() != size { + return None; + } + + // clang passes the inner type directly, we emulate it with fitting integer types + match size { + 1 => Some(Uniform::new(Reg::i8(), field.layout.size)), + 2 => Some(Uniform::new(Reg::i16(), field.layout.size)), + 4 => Some(Uniform::new(Reg::i32(), field.layout.size)), + 8 => Some(Uniform::new(Reg::i64(), field.layout.size)), + 16 => Some(Uniform::new(Reg::i128(), field.layout.size)), + s => panic!("Unhandled scalar of size {s} in amdgpu gpu-kernel ABI"), + } + } + BackendRepr::SimdVector { element, .. } => { + // Check that the size is the same as the size for ty, so no extra padding + let size = field.layout.size.bytes(); + if ty.layout.size.bytes() != size { + return None; + } + + // clang passes the inner type directly, we emulate it with a vector of the same type. + // The size is rounded up to the size of the complete type (including alignment). + let reg = Reg { + kind: RegKind::Vector { hint_vector_elem: element.primitive() }, + size: field.layout.size, + }; + Some(Uniform::new(reg, field.layout.size)) + } + BackendRepr::Memory { .. } => single_element_struct_to_reg(cx, field), + BackendRepr::ScalarPair { .. } => None, } - arg.extend_integer_width_to(32); } pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) @@ -27,14 +62,25 @@ where Ty: TyAbiInterface<'a, C> + Copy, C: HasDataLayout, { - if !fn_abi.ret.is_ignore() { - classify_ret(cx, &mut fn_abi.ret); - } + // Kernels cannot return values, so do not handle return types + // Try to fill first registers with values and pass by_ref pointers for later indirect arguments for arg in fn_abi.args.iter_mut() { if arg.is_ignore() { continue; } - classify_arg(cx, arg); + if fn_abi.conv == CanonAbi::GpuKernel { + if arg.layout.is_aggregate() { + if let Some(uniform) = single_element_struct_to_reg(cx, arg.layout) { + // Single element structs are passed directly as the inner type + arg.cast_to(uniform); + } else { + // All other aggregates are passed as by_ref pointer in the constant address space + arg.pass_amdgpu_kernel_arg(Some(AddressSpace::GPU_CONSTANT)); + } + } + } else { + // FIXME: C ABI is not yet implemented + } } } diff --git a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs index bf51cbfa7f7a0..bc83f6510a6fe 100644 --- a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs +++ b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs @@ -1,7 +1,7 @@ //@ add-minicore //@ compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 -Copt-level=3 //@ needs-llvm-components: amdgpu -#![feature(no_core, abi_gpu_kernel)] +#![feature(no_core, abi_gpu_kernel, repr_simd)] #![no_core] #![allow(improper_gpu_kernel_arg)] @@ -10,14 +10,32 @@ use minicore::num::Complex; // Tests from llvm-project/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl +#[repr(simd)] +pub struct I8X2([i8; 2]); + +#[repr(simd)] +pub struct I16X2([i16; 2]); + +#[repr(simd)] +pub struct I16X3([i16; 3]); + +#[repr(simd)] +pub struct I16X4([i16; 4]); + +#[repr(simd)] +pub struct I32X3([i32; 3]); + +#[repr(simd)] +pub struct I32X4([i32; 4]); + #[repr(C)] -pub struct SingleElementStructArg { - i: i32, +pub struct SingleElementStructArg { + i: T, } #[repr(C)] pub struct NestedSingleElementStructArg { - i: SingleElementStructArg, + i: SingleElementStructArg, } #[repr(C)] @@ -78,56 +96,92 @@ pub struct DifferentSizeTypePair { i: i32, } -// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(i32 %0) #[no_mangle] -pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg) {} +pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg) {} -// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(i32 %0) #[no_mangle] pub extern "gpu-kernel" fn kernel_nested_single_element_struct_arg( _: NestedSingleElementStructArg, ) { } -// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(12) {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([12 x i8]) align 4 captures(none) dereferenceable(12) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_struct_arg(_: StructArg) {} -// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(i8 noundef {{%.+}}, i64 noundef {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_struct_padding_arg(_: StructPaddingArg) {} -// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(44) {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr addrspace(4) noalias nofree noundef readnone byref([44 x i8]) align 4 captures(none) dereferenceable(44) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_struct_of_arrays_arg(_: StructOfArraysArg) {} -// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(24) {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr addrspace(4) noalias nofree noundef readnone byref([24 x i8]) align 4 captures(none) dereferenceable(24) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_struct_of_structs_arg(_: StructOfStructsArg) {} -// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}}) +// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr addrspace(4) noalias nofree noundef readnone byref([4 x i8]) align 4 captures(none) dereferenceable(4) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn test_kernel_union_arg(_: U) {} -// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(16) {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 4 captures(none) dereferenceable(16) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_single_array_element_struct_arg(_: SingleArrayElementStructArg) {} -// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(i32 noundef {{%.+}}, i64 noundef {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_single_struct_element_struct_arg( _: SingleStructElementStructArg, ) { } -// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(i64 noundef {{%.+}}, i32 noundef {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_different_size_type_pair_arg(_: DifferentSizeTypePair) {} -// CHECK: define amdgpu_kernel void @kernel_complex(float noundef {{%.+}}, float noundef {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_complex(ptr addrspace(4) noalias nofree noundef readnone byref([8 x i8]) align 4 captures(none) dereferenceable(8) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_complex(_: Complex) {} -// CHECK: define amdgpu_kernel void @kernel_slice(ptr noalias nofree noundef nonnull readonly align 4 captures(none) {{%.+}}, i64 noundef range(i64 0, 2305843009213693952) {{%.+}}) +// CHECK: define amdgpu_kernel void @kernel_slice(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}}) #[no_mangle] pub extern "gpu-kernel" fn kernel_slice(_: &[u32]) {} + +// CHECK: define amdgpu_kernel void @kernel_i64(i64 noundef {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i64(_: i64) {} + +// CHECK: define amdgpu_kernel void @kernel_i64_struct(i64 {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i64_struct(_: SingleElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_i128_struct(i128 {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i128_struct(_: SingleElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_i8x2_struct(<2 x i8> {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i8x2_struct(_: SingleElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_i16x2_struct(<2 x i16> {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i16x2_struct(_: SingleElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_i16x3_struct(<4 x i16> {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i16x3_struct(_: SingleElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_i16x4_struct(<4 x i16> {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i16x4_struct(_: SingleElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_i32x3_struct(<4 x i32> {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i32x3_struct(_: SingleElementStructArg) {} + +// CHECK: define amdgpu_kernel void @kernel_i32x4_struct(<4 x i32> {{%.+}}) +#[no_mangle] +pub extern "gpu-kernel" fn kernel_i32x4_struct(_: SingleElementStructArg) {}