From c321d90e6fcaf329a2ca52e524d61ff4b9d9edb7 Mon Sep 17 00:00:00 2001 From: sgasho Date: Fri, 4 Sep 2026 16:49:05 +0000 Subject: [PATCH 1/4] offload: automate manual clang-linker-wrapper step --- compiler/rustc_codegen_llvm/src/back/write.rs | 23 ++++-- .../src/builder/gpu_offload.rs | 74 ------------------- compiler/rustc_codegen_llvm/src/context.rs | 5 -- .../rustc_codegen_llvm/src/diagnostics.rs | 4 + compiler/rustc_codegen_llvm/src/intrinsic.rs | 5 +- .../src/llvm/offload_ffi.rs | 13 ++++ compiler/rustc_codegen_ssa/src/back/link.rs | 6 ++ .../llvm-wrapper/offload/OffloadWrapper.cpp | 56 ++++++++++++++ 8 files changed, 98 insertions(+), 88 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index b8952ffc6bf81..90b2cab5b63e2 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -612,6 +612,8 @@ pub(crate) unsafe fn llvm_optimize( let pgo_use_path = get_pgo_use_path(config); let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; + let is_final_stage = + !matches!(opt_stage, llvm::OptStage::PreLinkFatLTO | llvm::OptStage::PreLinkThinLTO); let instr_profile_output_path = get_instr_profile_output_path(config); let sanitize_dataflow_abilist: Vec<_> = config .sanitizer_dataflow_abilist @@ -840,7 +842,7 @@ pub(crate) unsafe fn llvm_optimize( // don't need any other artifacts from the previous run. We will embed this artifact into our // LLVM-IR host module, to create a `host.o` ObjectFile, which we will write to disk. // The last, not yet automated steps uses the `clang-linker-wrapper` to process `host.o`. - if !cgcx.target_is_like_gpu { + if !cgcx.target_is_like_gpu && is_final_stage { if let Some(device_path) = config .offload .iter() @@ -866,10 +868,11 @@ pub(crate) unsafe fn llvm_optimize( // 2) Finalize host: lib.bc + device.bin -> host.o (host TM) // We create a full clone of our LLVM host module, since we will embed the device IR // into it, and this might break caching or incremental compilation otherwise. - let llmod2 = llvm::LLVMCloneModule(module.module_llvm.llmod()); let ok = unsafe { - llvm::RustOffloadWrapper::get_instance() - .llvm_rust_offload_embed_buffer_in_module(llmod2, device_bin_c.as_c_str()) + llvm::RustOffloadWrapper::get_instance().llvm_rust_offload_embed_buffer_in_module( + module.module_llvm.llmod(), + device_bin_c.as_c_str(), + ) }; if !ok { dcx.emit_err(crate::diagnostics::OffloadEmbedFailed); @@ -878,7 +881,7 @@ pub(crate) unsafe fn llvm_optimize( dcx, module.module_llvm.tm.raw(), config.no_builtins, - llmod2, + module.module_llvm.llmod(), &out_obj, None, llvm::FileType::ObjectFile, @@ -888,6 +891,16 @@ pub(crate) unsafe fn llvm_optimize( // We ignore cgcx.save_temps here and unconditionally always keep our `device.bin` artifact. // Otherwise, recompiling the host code would fail since we deleted that device artifact // in the previous host compilation, which would be confusing at best. + + let ok = unsafe { + llvm::RustOffloadWrapper::get_instance().llvm_rust_offload_wrap_images( + module.module_llvm.llmod(), + device_bin_c.as_c_str(), + ) + }; + if !ok { + dcx.emit_err(crate::diagnostics::OffloadWrapImagesFailed); + } } } result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index d20a73e8e6825..e2ec20226e3ce 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -57,80 +57,6 @@ impl<'ll> OffloadGlobals<'ll> { } } -// We need to register offload before using it. We also should unregister it once we are done, for -// good measures. Previously we have done so before and after each individual offload intrinsic -// call, but that comes at a performance cost. The repeated (un)register calls might also confuse -// the LLVM ompOpt pass, which tries to move operations to a better location. The easiest solution, -// which we copy from clang, is to just have those two calls once, in the global ctor/dtor section -// of the final binary. -pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) { - // First we check quickly whether we already have done our setup, in which case we return early. - // Shouldn't be needed for correctness. - let register_lib_name = "__tgt_register_lib"; - if cx.get_function(register_lib_name).is_some() { - return; - } - - let reg_lib_decl = cx.type_func(&[cx.type_ptr()], cx.type_void()); - let register_lib = declare_offload_fn(&cx, register_lib_name, reg_lib_decl); - let unregister_lib = declare_offload_fn(&cx, "__tgt_unregister_lib", reg_lib_decl); - - let ptr_null = cx.const_null(cx.type_ptr()); - let const_struct = cx.const_struct(&[cx.get_const_i32(0), ptr_null, ptr_null, ptr_null], false); - let omp_descriptor = - add_global(cx, ".omp_offloading.descriptor", const_struct, InternalLinkage); - // @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 1, ptr @.omp_offloading.device_images, ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries } - // @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 0, ptr null, ptr null, ptr null } - - let atexit = cx.type_func(&[cx.type_ptr()], cx.type_i32()); - let atexit_fn = declare_offload_fn(cx, "atexit", atexit); - - // FIXME(offload): Drop this, once we fully automated our offload compilation pipeline, since - // LLVM will initialize them for us if it sees gpu kernels being registered. - let init_ty = cx.type_func(&[], cx.type_void()); - let init_rtls = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty); - - let desc_ty = cx.type_func(&[], cx.type_void()); - let reg_name = ".omp_offloading.descriptor_reg"; - let unreg_name = ".omp_offloading.descriptor_unreg"; - let desc_reg_fn = declare_offload_fn(cx, reg_name, desc_ty); - let desc_unreg_fn = declare_offload_fn(cx, unreg_name, desc_ty); - llvm::set_linkage(desc_reg_fn, InternalLinkage); - llvm::set_linkage(desc_unreg_fn, InternalLinkage); - llvm::set_section(desc_reg_fn, c".text.startup"); - llvm::set_section(desc_unreg_fn, c".text.startup"); - - // define internal void @.omp_offloading.descriptor_reg() section ".text.startup" { - // entry: - // call void @__tgt_register_lib(ptr @.omp_offloading.descriptor) - // call void @__tgt_init_all_rtls() - // %0 = call i32 @atexit(ptr @.omp_offloading.descriptor_unreg) - // ret void - // } - let bb = Builder::append_block(cx, desc_reg_fn, "entry"); - let mut a = Builder::build(cx, bb); - a.call(reg_lib_decl, None, None, register_lib, &[omp_descriptor], None, None); - a.call(init_ty, None, None, init_rtls, &[], None, None); - a.call(atexit, None, None, atexit_fn, &[desc_unreg_fn], None, None); - a.ret_void(); - - // define internal void @.omp_offloading.descriptor_unreg() section ".text.startup" { - // entry: - // call void @__tgt_unregister_lib(ptr @.omp_offloading.descriptor) - // ret void - // } - let bb = Builder::append_block(cx, desc_unreg_fn, "entry"); - let mut a = Builder::build(cx, bb); - a.call(reg_lib_decl, None, None, unregister_lib, &[omp_descriptor], None, None); - a.ret_void(); - - // @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }] - let args = vec![cx.get_const_i32(101), desc_reg_fn, ptr_null]; - let const_struct = cx.const_struct(&args, false); - let arr = cx.const_array(cx.val_ty(const_struct), &[const_struct]); - add_global(cx, "llvm.global_ctors", arr, AppendingLinkage); -} - pub(crate) struct OffloadKernelDims<'ll> { num_workgroups: &'ll Value, threads_per_block: &'ll Value, diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 853c4bfc9ca3f..ecad6e5629172 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -883,11 +883,6 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { self.get_const_int(self.type_i8(), n) } - pub(crate) fn get_function(&self, name: &str) -> Option<&'ll Value> { - let name = SmallCStr::new(name); - unsafe { llvm::LLVMGetNamedFunction((**self).borrow().llmod, name.as_ptr()) } - } - pub(crate) fn get_md_kind_id(&self, name: &str) -> llvm::MetadataKindId { unsafe { llvm::LLVMGetMDKindIDInContext( diff --git a/compiler/rustc_codegen_llvm/src/diagnostics.rs b/compiler/rustc_codegen_llvm/src/diagnostics.rs index fb43b36fe39b9..3f6ce440ef3f7 100644 --- a/compiler/rustc_codegen_llvm/src/diagnostics.rs +++ b/compiler/rustc_codegen_llvm/src/diagnostics.rs @@ -107,6 +107,10 @@ pub(crate) struct OffloadBundleImagesFailed; #[diag("call to EmbedBufferInModule failed, `host.o` was not created")] pub(crate) struct OffloadEmbedFailed; +#[derive(Diagnostic)] +#[diag("call to WrapImages failed, `wrapper.o` was not created")] +pub(crate) struct OffloadWrapImagesFailed; + #[derive(Diagnostic)] #[diag("failed to get bitcode from object file for LTO ({$err})")] pub(crate) struct LtoBitcodeFromRlib { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 58957b46964c7..1f36a9543df72 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -36,9 +36,7 @@ use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; -use crate::builder::gpu_offload::{ - self, OffloadKernelDims, declare_omp_get_num_devices, register_offload, -}; +use crate::builder::gpu_offload::{self, OffloadKernelDims, declare_omp_get_num_devices}; use crate::context::CodegenCx; use crate::declare::declare_raw_fn; use crate::diagnostics::{ @@ -1880,7 +1878,6 @@ fn codegen_offload<'ll, 'tcx>( return; } }; - register_offload(cx); let offload_data = gpu_offload::gen_define_handling(&cx, &metadata, target_symbol, offload_globals); gpu_offload::gen_call_handling( diff --git a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs index 46d9320248a9b..6e7f164d2bd4a 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs @@ -6,6 +6,7 @@ use super::ffi::{Module, TargetMachine, Value}; type LLVMRustBundleImagesFn = unsafe extern "C" fn(&Module, &TargetMachine, *const c_char) -> bool; type LLVMRustOffloadEmbedBufferInModuleFn = unsafe extern "C" fn(&Module, *const c_char) -> bool; type LLVMRustOffloadMapperFn = unsafe extern "C" fn(&Value, &Value, *const &Value); +type LLVMRustOffloadWrapImagesFn = unsafe extern "C" fn(&Module, *const c_char) -> bool; use rustc_session::config::host_tuple; use rustc_session::filesearch; @@ -16,6 +17,7 @@ pub(crate) struct RustOffloadWrapper { LLVMRustBundleImages: LLVMRustBundleImagesFn, LLVMRustOffloadEmbedBufferInModule: LLVMRustOffloadEmbedBufferInModuleFn, LLVMRustOffloadMapper: LLVMRustOffloadMapperFn, + LLVMRustOffloadWrapImages: LLVMRustOffloadWrapImagesFn, // Keep the dynamic library loaded while the function pointers are used. _lib: libloading::Library, } @@ -71,6 +73,14 @@ impl RustOffloadWrapper { unsafe { (self.LLVMRustOffloadMapper)(v1, v2, vs.as_ptr()) } } + pub(crate) unsafe fn llvm_rust_offload_wrap_images( + &self, + host_m: &Module, + device_bin_path: &CStr, + ) -> bool { + unsafe { (self.LLVMRustOffloadWrapImages)(host_m, device_bin_path.as_ptr()) } + } + fn call_dynamic( sysroot: &rustc_session::config::Sysroot, ) -> Result { @@ -86,11 +96,14 @@ impl RustOffloadWrapper { }; let llvm_rust_offload_wrapper = *unsafe { lib.get::(b"LLVMRustOffloadMapper\0")? }; + let llvm_rust_offload_wrap_images = + *unsafe { lib.get::(b"LLVMRustOffloadWrapImages\0")? }; Ok(Self { LLVMRustBundleImages: llvm_rust_bundle_images, LLVMRustOffloadEmbedBufferInModule: llvm_rust_offload_embed_buffer_in_module, LLVMRustOffloadMapper: llvm_rust_offload_wrapper, + LLVMRustOffloadWrapImages: llvm_rust_offload_wrap_images, _lib: lib, }) } diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 25003e071beb7..e7eac04649334 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -3029,6 +3029,12 @@ fn linker_with_args( link_output_kind, ); + if sess.opts.unstable_opts.offload.iter().any(|o| matches!(o, config::Offload::Host(_))) { + cmd.link_dylib_by_name("omptarget", false, true); + cmd.link_dylib_by_name("omp", false, true); + cmd.link_args(["-z", "nostart-stop-gc"]); + } + // Upstream rust crates and their non-dynamic native libraries. add_upstream_rust_crates( cmd, diff --git a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp index 8c18f2453e9d8..8bd555807f2a8 100644 --- a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp @@ -3,6 +3,8 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Frontend/Offloading/OffloadWrapper.h" +#include "llvm/Frontend/Offloading/Utility.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Object/OffloadBinary.h" #include "llvm/Support/CBindingWrapping.h" @@ -115,3 +117,57 @@ extern "C" void LLVMRustOffloadMapper(LLVMValueRef OldFn, LLVMValueRef NewFn, IRBuilder<> B(&entry); B.CreateBr(&clonedEntry); } + +static Error extractImages(StringRef DeviceBinPath, + SmallVectorImpl &Binaries) { + ErrorOr> BufOrErr = + MemoryBuffer::getFile(DeviceBinPath); + if (std::error_code EC = BufOrErr.getError()) + return createFileError(DeviceBinPath, EC); + std::unique_ptr Buf = std::move(*BufOrErr); + + if (!isAddrAligned(Align(OffloadBinary::getAlignment()), + Buf->getBufferStart())) + Buf = MemoryBuffer::getMemBufferCopy(Buf->getBuffer(), + Buf->getBufferIdentifier()); + + return extractOffloadBinaries(*Buf, Binaries); +} + +static bool hasOffloadEntries(Module &M) { + for (GlobalVariable &GV : M.globals()) + if (GV.hasSection() && GV.getSection() == "llvm_offload_entries") + return true; + return false; +} + +static bool reportAndFailWrappingImages(Error E, const char *What) { + handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) { + errs() << "LLVMRustOffloadWrapImages: " << What << ": " << EI.message() + << "\n"; + }); + return false; +} + +extern "C" bool LLVMRustOffloadWrapImages(LLVMModuleRef HostMRef, + const char *DeviceBinPath) { + Module &M = *unwrap(HostMRef); + if (!hasOffloadEntries(M)) + return true; + + SmallVector Binaries; + if (Error E = extractImages(DeviceBinPath, Binaries)) + return reportAndFailWrappingImages(std::move(E), "extract"); + + SmallVector> Images; + for (OffloadFile &F : Binaries) { + StringRef Img = F.getBinary()->getImage(); + Images.emplace_back(Img.data(), Img.size()); + } + + if (Error E = offloading::wrapOpenMPBinaries( + M, Images, offloading::getOffloadEntryArray(M), /*Suffix=*/"", + /*Relocatable=*/false)) + return reportAndFailWrappingImages(std::move(E), "wrap"); + return true; +} From 18ada5c34dc7495b1d80ec5b94fe63f3512472a3 Mon Sep 17 00:00:00 2001 From: sgasho Date: Sun, 6 Sep 2026 13:31:35 +0000 Subject: [PATCH 2/4] compile bitcode in device.bin beforehand in order to avoid LLVM JIT, which causes Unable to find target error --- .../src/llvm/offload_ffi.rs | 79 +++++++------ .../llvm-wrapper/offload/OffloadWrapper.cpp | 105 ++++++++++++++++-- src/bootstrap/src/core/build_steps/compile.rs | 13 ++- src/bootstrap/src/core/build_steps/dist.rs | 17 ++- src/bootstrap/src/core/build_steps/llvm.rs | 57 ++++++++++ 5 files changed, 226 insertions(+), 45 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs index 6e7f164d2bd4a..c4b6ed6611add 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs @@ -1,4 +1,5 @@ use std::ffi::{CStr, c_char}; +use std::path::PathBuf; use std::sync::OnceLock; use super::ffi::{Module, TargetMachine, Value}; @@ -6,8 +7,10 @@ use super::ffi::{Module, TargetMachine, Value}; type LLVMRustBundleImagesFn = unsafe extern "C" fn(&Module, &TargetMachine, *const c_char) -> bool; type LLVMRustOffloadEmbedBufferInModuleFn = unsafe extern "C" fn(&Module, *const c_char) -> bool; type LLVMRustOffloadMapperFn = unsafe extern "C" fn(&Value, &Value, *const &Value); -type LLVMRustOffloadWrapImagesFn = unsafe extern "C" fn(&Module, *const c_char) -> bool; +type LLVMRustOffloadWrapImagesFn = + unsafe extern "C" fn(&Module, *const c_char, *const c_char) -> bool; +use rustc_fs_util::path_to_c_string; use rustc_session::config::host_tuple; use rustc_session::filesearch; @@ -18,6 +21,7 @@ pub(crate) struct RustOffloadWrapper { LLVMRustOffloadEmbedBufferInModule: LLVMRustOffloadEmbedBufferInModuleFn, LLVMRustOffloadMapper: LLVMRustOffloadMapperFn, LLVMRustOffloadWrapImages: LLVMRustOffloadWrapImagesFn, + clang_path: PathBuf, // Keep the dynamic library loaded while the function pointers are used. _lib: libloading::Library, } @@ -78,13 +82,19 @@ impl RustOffloadWrapper { host_m: &Module, device_bin_path: &CStr, ) -> bool { - unsafe { (self.LLVMRustOffloadWrapImages)(host_m, device_bin_path.as_ptr()) } + unsafe { + (self.LLVMRustOffloadWrapImages)( + host_m, + path_to_c_string(&self.clang_path).as_ptr(), + device_bin_path.as_ptr(), + ) + } } fn call_dynamic( sysroot: &rustc_session::config::Sysroot, ) -> Result { - let rust_offload_path = Self::get_rust_offload_path(sysroot)?; + let (rust_offload_path, clang_path) = Self::get_offload_and_clang_paths(sysroot)?; let lib = unsafe { libloading::Library::new(rust_offload_path)? }; let llvm_rust_bundle_images = @@ -104,43 +114,42 @@ impl RustOffloadWrapper { LLVMRustOffloadEmbedBufferInModule: llvm_rust_offload_embed_buffer_in_module, LLVMRustOffloadMapper: llvm_rust_offload_wrapper, LLVMRustOffloadWrapImages: llvm_rust_offload_wrap_images, + clang_path, _lib: lib, }) } - fn get_rust_offload_path( + fn get_offload_and_clang_paths( sysroot: &rustc_session::config::Sysroot, - ) -> Result { + ) -> Result<(PathBuf, PathBuf), RustOffloadLibraryError> { let llvm_version_major = unsafe { LLVMRustVersionMajor() }; - - let path_buf = sysroot - .all_paths() - .find_map(|p| { - let candidate = filesearch::make_target_lib_path(p, host_tuple()) - .join(format!("libRustOffload-{}", llvm_version_major)) - .with_extension(std::env::consts::DLL_EXTENSION); - - candidate.exists().then_some(candidate) - }) - .ok_or_else(|| { - let candidates = sysroot - .all_paths() - .map(|p| p.join("lib").display().to_string()) - .collect::>() - .join("\n* "); - RustOffloadLibraryError::NotFound { - err: format!( - "failed to find a `libRustOffload-{llvm_version_major}` \ - in the sysroot candidates:\n* {candidates}" - ), - } - })?; - - Ok(path_buf - .to_str() - .ok_or_else(|| RustOffloadLibraryError::LoadFailed { - err: format!("invalid UTF-8 in path: {}", path_buf.display()), - })? - .to_string()) + let clang_name = format!("clang{}", std::env::consts::EXE_SUFFIX); + let mut searched = Vec::new(); + + for root in sysroot.all_paths() { + let rust_offload_path = filesearch::make_target_lib_path(root, host_tuple()) + .join(format!("libRustOffload-{llvm_version_major}")) + .with_extension(std::env::consts::DLL_EXTENSION); + + let clang_path = filesearch::make_target_bin_path(root, host_tuple()).join(&clang_name); + + if rust_offload_path.is_file() && clang_path.is_file() { + return Ok((rust_offload_path, clang_path)); + } + + searched.extend([rust_offload_path, clang_path]); + } + + Err(RustOffloadLibraryError::NotFound { + err: format!( + "could not find both libRustOffload-{llvm_version_major} and Clang \ + in the same sysroot. Searched:\n{}", + searched + .iter() + .map(|path| path.parent().unwrap().display().to_string()) + .collect::>() + .join("\n"), + ), + }) } } diff --git a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp index 8bd555807f2a8..9360850a91f32 100644 --- a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp @@ -1,20 +1,32 @@ #include "../SuppressLLVMWarnings.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/Frontend/Offloading/OffloadWrapper.h" #include "llvm/Frontend/Offloading/Utility.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Object/OffloadBinary.h" -#include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/Error.h" #include "llvm/Support/FileOutputBuffer.h" +#include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/ModuleUtils.h" #include "llvm/Transforms/Utils/ValueMapper.h" +#include +#include +#include + using namespace llvm; using namespace llvm::object; @@ -149,7 +161,73 @@ static bool reportAndFailWrappingImages(Error E, const char *What) { return false; } +static Expected> +compileAndLinkDeviceImages(const OffloadBinary &Input, const char *ClangPath) { + const Triple DeviceTriple(Input.getTriple()); + const StringRef Arch = Input.getArch(); + + SmallString<128> TempDir; + if (std::error_code E = + sys::fs::createUniqueDirectory("rust-offload", TempDir)) + return errorCodeToError(E); + + auto Cleanup = scope_exit([&] { + if (std::error_code E = sys::fs::remove_directories(TempDir)) + (void)reportAndFailWrappingImages( + errorCodeToError(E), "compileAndLinkDeviceImages: tempdir cleanup"); + }); + + SmallString<128> OutputPath(TempDir); + sys::path::append(OutputPath, "device.img"); + + SmallVector ArgStorage{ + ClangPath, + "--no-default-config", + "--target=" + DeviceTriple.str(), + "-o", + OutputPath.str().str(), + "-dumpdir", + OutputPath.str().str() + ".", + }; + + if (!Arch.empty() && Arch != "generic") + ArgStorage.push_back( + ((DeviceTriple.isAMDGPU() ? "-mcpu=" : "-march=") + Arch).str()); + + if (DeviceTriple.isAMDGPU()) + ArgStorage.push_back("-Wl,--no-undefined"); + + SmallString<128> InputPath(TempDir); + sys::path::append(InputPath, "input.o"); + + if (Error E = writeFile(InputPath, Input.getImage())) + return std::move(E); + + ArgStorage.push_back(InputPath.str().str()); + + SmallVector CmdArgs; + for (const StringRef Arg : ArgStorage) + CmdArgs.push_back(Arg); + + std::string ExecError; + int Status = sys::ExecuteAndWait(ClangPath, CmdArgs, std::nullopt, {}, 0, 0, + &ExecError); + + if (Status != 0) + return createStringError("compileAndLinkDeviceImages: device compiler " + "failed for %s/%s (status %d): %s", + DeviceTriple.str().c_str(), Arch.str().c_str(), + Status, ExecError.c_str()); + + auto ImageOrErr = MemoryBuffer::getFileAsStream(OutputPath); + if (!ImageOrErr) + return createFileError(OutputPath, ImageOrErr.getError()); + + return std::move(*ImageOrErr); +} + extern "C" bool LLVMRustOffloadWrapImages(LLVMModuleRef HostMRef, + const char *ClangPath, const char *DeviceBinPath) { Module &M = *unwrap(HostMRef); if (!hasOffloadEntries(M)) @@ -159,15 +237,26 @@ extern "C" bool LLVMRustOffloadWrapImages(LLVMModuleRef HostMRef, if (Error E = extractImages(DeviceBinPath, Binaries)) return reportAndFailWrappingImages(std::move(E), "extract"); - SmallVector> Images; - for (OffloadFile &F : Binaries) { - StringRef Img = F.getBinary()->getImage(); - Images.emplace_back(Img.data(), Img.size()); - } + // LLVMRustBundleImages writes exactly one device image + if (Binaries.size() != 1) + return reportAndFailWrappingImages( + createStringError("expected exactly one device image, found %zu", + Binaries.size()), + "extract"); + + const OffloadBinary &Input = *Binaries.front().getBinary(); + + auto ImageOrErr = compileAndLinkDeviceImages(Input, ClangPath); + if (!ImageOrErr) + return reportAndFailWrappingImages(ImageOrErr.takeError(), "device link"); + + StringRef ImageBuf = (*ImageOrErr)->getBuffer(); + ArrayRef Image(ImageBuf.data(), ImageBuf.size()); if (Error E = offloading::wrapOpenMPBinaries( - M, Images, offloading::getOffloadEntryArray(M), /*Suffix=*/"", - /*Relocatable=*/false)) + M, {Image}, offloading::getOffloadEntryArray(M), /*Suffix=*/"", + /*Relocatable=*/ + false)) return reportAndFailWrappingImages(std::move(E), "wrap"); return true; } diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index fb83e4ae5754c..b224f4b4dafed 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -21,7 +21,9 @@ use tracing::span; use crate::core::backend::CodegenBackendKind; use crate::core::build_steps::gcc::{Gcc, GccOutput, GccTargetPair}; -use crate::core::build_steps::llvm::{LlvmFromCi, LlvmKind, prebuilt_llvm_output}; +use crate::core::build_steps::llvm::{ + LlvmFromCi, LlvmKind, offload_clang_lib_paths, offload_tool_paths, prebuilt_llvm_output, +}; use crate::core::build_steps::tool::{RustcPrivateCompilers, SourceType, copy_lld_artifacts}; use crate::core::build_steps::{dist, llvm}; use crate::core::builder::{ @@ -2321,6 +2323,15 @@ impl CommandLineStep for Assemble { let dst_lib = target_libdir.join(libname); builder.resolve_symlink_and_copy(&p, &dst_lib); } + + for (source, filename) in offload_tool_paths(builder, target_compiler.host) { + builder.resolve_symlink_and_copy(&source, &libdir_bin.join(filename)); + } + + for source in offload_clang_lib_paths(builder, target_compiler.host) { + let filename = source.file_name().unwrap(); + builder.resolve_symlink_and_copy(&source, &target_libdir.join(filename)); + } } } diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index a335172631307..3825b2e90d97d 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -27,6 +27,7 @@ use crate::core::build_steps::doc::DocumentationFormat; use crate::core::build_steps::gcc::GccTargetPair; use crate::core::build_steps::llvm::{ LLVM_CI_LINK_TYPE_PATH, LlvmBuildStatus, LlvmKind, get_llvm_build_status, + offload_clang_lib_paths, offload_tool_paths, }; use crate::core::build_steps::tool::{ self, RustcPrivateCompilers, ToolTargetBuildMode, get_tool_target_compiler, @@ -2900,7 +2901,21 @@ impl CommandLineStep for Offload { tarball.add_file(path, destdir, FileType::NativeLibrary); } - tarball.add_file(rust_offload.rust_offload_path(), target_libdir, FileType::NativeLibrary); + tarball.add_file(rust_offload.rust_offload_path(), &target_libdir, FileType::NativeLibrary); + + let target_bindir = PathBuf::from(format!("lib/rustlib/{}/bin", target.triple)); + for (source, filename) in offload_tool_paths(builder, target) { + let source = t!(fs::canonicalize(source)); + tarball.add_renamed_file(source, &target_bindir, &filename, FileType::Executable); + } + + for source in offload_clang_lib_paths(builder, target) { + let filename = source.file_name().unwrap().to_str().unwrap(); + let resolved = t!(fs::canonicalize(&source)); + tarball.add_renamed_file(resolved, &target_libdir, filename, FileType::NativeLibrary); + } + + maybe_install_llvm_target(builder, target, tarball.image_dir()); Some(tarball.generate()) } diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index f1864c736ed03..d39be4b19e7ff 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -1211,6 +1211,63 @@ impl CommandLineStep for RustOffload { } } +/// Returns the binary and library directories of the selected clang. +fn offload_clang_dirs(builder: &Builder<'_>, target: TargetSelection) -> (PathBuf, PathBuf) { + if builder.config.llvm_clang { + let llvm = builder.ensure(llvm::Llvm { target }); + return (llvm.root_dir().join("bin"), llvm.root_dir().join("lib")); + } + + let lib_dir = builder + .config + .offload_clang_dir + .as_deref() + .and_then(|dir| dir.ancestors().nth(2)) + .expect("llvm.offload-clang-dir must point to //cmake/clang"); + + let root = lib_dir.parent().expect("Clang library directory must have a parent"); + + (root.join("bin"), lib_dir.into()) +} + +/// Returns (source path, destination filename) pairs for offloading tools such as clang. +pub(crate) fn offload_tool_paths( + builder: &Builder<'_>, + target: TargetSelection, +) -> Vec<(PathBuf, String)> { + let lld = builder.ensure(llvm::Lld { target }); + let (clang_bin_dir, _) = offload_clang_dirs(builder, target); + + let mut tools = Vec::new(); + + for name in ["clang", "clang-nvlink-wrapper"] { + let filename = exe(name, target); + let source = clang_bin_dir.join(&filename); + tools.push((source, filename)); + } + + tools.push((lld.join("bin").join(exe("lld", target)), exe("ld.lld", target))); + + tools +} + +pub(crate) fn offload_clang_lib_paths( + builder: &Builder<'_>, + target: TargetSelection, +) -> Vec { + let (_, lib_dir) = offload_clang_dirs(builder, target); + + let mut paths = Vec::new(); + for entry in builder.read_dir(&lib_dir) { + let filename = entry.file_name(); + let filename = filename.to_string_lossy(); + if filename == "libclang-cpp.so" || filename.starts_with("libclang-cpp.so.") { + paths.push(entry.path()); + } + } + paths +} + #[derive(Clone)] pub struct BuiltOmpOffload { /// Path to the omp and offload dylibs. From f9017b16304e7191d3057302d91874772884fbe7 Mon Sep 17 00:00:00 2001 From: sgasho Date: Sun, 6 Sep 2026 14:35:53 +0000 Subject: [PATCH 3/4] add rpath --- compiler/rustc_codegen_ssa/src/back/link.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index e7eac04649334..abe41f8ae8e7c 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -3033,6 +3033,8 @@ fn linker_with_args( cmd.link_dylib_by_name("omptarget", false, true); cmd.link_dylib_by_name("omp", false, true); cmd.link_args(["-z", "nostart-stop-gc"]); + cmd.link_arg("-rpath"); + cmd.link_arg(std::path::absolute(&*sess.target_tlib_path.dir).unwrap()); } // Upstream rust crates and their non-dynamic native libraries. From ff680e2717baf80131eebf7c3691a72100263fd8 Mon Sep 17 00:00:00 2001 From: sgasho Date: Mon, 7 Sep 2026 17:47:04 +0000 Subject: [PATCH 4/4] compile device image without adding extra binaries --- .../rustc_codegen_llvm/src/diagnostics.rs | 2 +- .../src/llvm/offload_ffi.rs | 40 ++-- .../llvm-wrapper/offload/OffloadWrapper.cpp | 181 +++++++++++++----- src/bootstrap/src/core/build_steps/compile.rs | 13 +- src/bootstrap/src/core/build_steps/dist.rs | 17 +- src/bootstrap/src/core/build_steps/llvm.rs | 57 ------ 6 files changed, 155 insertions(+), 155 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/diagnostics.rs b/compiler/rustc_codegen_llvm/src/diagnostics.rs index 3f6ce440ef3f7..70a14288aec0c 100644 --- a/compiler/rustc_codegen_llvm/src/diagnostics.rs +++ b/compiler/rustc_codegen_llvm/src/diagnostics.rs @@ -108,7 +108,7 @@ pub(crate) struct OffloadBundleImagesFailed; pub(crate) struct OffloadEmbedFailed; #[derive(Diagnostic)] -#[diag("call to WrapImages failed, `wrapper.o` was not created")] +#[diag("call to WrapImages failed, device image was not wrapped into the host module")] pub(crate) struct OffloadWrapImagesFailed; #[derive(Diagnostic)] diff --git a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs index c4b6ed6611add..7ecf450ab1dba 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/offload_ffi.rs @@ -21,7 +21,7 @@ pub(crate) struct RustOffloadWrapper { LLVMRustOffloadEmbedBufferInModule: LLVMRustOffloadEmbedBufferInModuleFn, LLVMRustOffloadMapper: LLVMRustOffloadMapperFn, LLVMRustOffloadWrapImages: LLVMRustOffloadWrapImagesFn, - clang_path: PathBuf, + lld_path: Option, // Keep the dynamic library loaded while the function pointers are used. _lib: libloading::Library, } @@ -82,19 +82,16 @@ impl RustOffloadWrapper { host_m: &Module, device_bin_path: &CStr, ) -> bool { + let lld_c = self.lld_path.as_deref().map(path_to_c_string).unwrap_or_default(); unsafe { - (self.LLVMRustOffloadWrapImages)( - host_m, - path_to_c_string(&self.clang_path).as_ptr(), - device_bin_path.as_ptr(), - ) + (self.LLVMRustOffloadWrapImages)(host_m, lld_c.as_ptr(), device_bin_path.as_ptr()) } } fn call_dynamic( sysroot: &rustc_session::config::Sysroot, ) -> Result { - let (rust_offload_path, clang_path) = Self::get_offload_and_clang_paths(sysroot)?; + let (rust_offload_path, lld_path) = Self::get_offload_and_lld_paths(sysroot)?; let lib = unsafe { libloading::Library::new(rust_offload_path)? }; let llvm_rust_bundle_images = @@ -114,16 +111,15 @@ impl RustOffloadWrapper { LLVMRustOffloadEmbedBufferInModule: llvm_rust_offload_embed_buffer_in_module, LLVMRustOffloadMapper: llvm_rust_offload_wrapper, LLVMRustOffloadWrapImages: llvm_rust_offload_wrap_images, - clang_path, + lld_path, _lib: lib, }) } - fn get_offload_and_clang_paths( + fn get_offload_and_lld_paths( sysroot: &rustc_session::config::Sysroot, - ) -> Result<(PathBuf, PathBuf), RustOffloadLibraryError> { + ) -> Result<(PathBuf, Option), RustOffloadLibraryError> { let llvm_version_major = unsafe { LLVMRustVersionMajor() }; - let clang_name = format!("clang{}", std::env::consts::EXE_SUFFIX); let mut searched = Vec::new(); for root in sysroot.all_paths() { @@ -131,24 +127,22 @@ impl RustOffloadWrapper { .join(format!("libRustOffload-{llvm_version_major}")) .with_extension(std::env::consts::DLL_EXTENSION); - let clang_path = filesearch::make_target_bin_path(root, host_tuple()).join(&clang_name); - - if rust_offload_path.is_file() && clang_path.is_file() { - return Ok((rust_offload_path, clang_path)); + if !rust_offload_path.is_file() { + searched.push(rust_offload_path); + continue; } - searched.extend([rust_offload_path, clang_path]); + let lld_path = filesearch::make_target_bin_path(root, host_tuple()) + .join(format!("rust-lld{}", std::env::consts::EXE_SUFFIX)); + let lld_path = lld_path.is_file().then_some(lld_path); + + return Ok((rust_offload_path, lld_path)); } Err(RustOffloadLibraryError::NotFound { err: format!( - "could not find both libRustOffload-{llvm_version_major} and Clang \ - in the same sysroot. Searched:\n{}", - searched - .iter() - .map(|path| path.parent().unwrap().display().to_string()) - .collect::>() - .join("\n"), + "could not find libRustOffload-{llvm_version_major} in the sysroot candidates:\n* {}", + searched.iter().map(|p| p.display().to_string()).collect::>().join("\n* ") ), }) } diff --git a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp index 9360850a91f32..bed54da0a7045 100644 --- a/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/offload/OffloadWrapper.cpp @@ -6,18 +6,26 @@ #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" #include "llvm/Frontend/Offloading/OffloadWrapper.h" #include "llvm/Frontend/Offloading/Utility.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/MC/TargetRegistry.h" #include "llvm/Object/OffloadBinary.h" +#include "llvm/Support/CodeGen.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileOutputBuffer.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" +#include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/Program.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/ModuleUtils.h" @@ -26,6 +34,7 @@ #include #include #include +#include using namespace llvm; using namespace llvm::object; @@ -162,72 +171,151 @@ static bool reportAndFailWrappingImages(Error E, const char *What) { } static Expected> -compileAndLinkDeviceImages(const OffloadBinary &Input, const char *ClangPath) { - const Triple DeviceTriple(Input.getTriple()); - const StringRef Arch = Input.getArch(); +assembleWithPtxas(StringRef Ptx, StringRef Arch) { + const ErrorOr Ptxas = sys::findProgramByName("ptxas"); + if (!Ptxas) + return createStringError(Ptxas.getError(), "ptxas not found in PATH"); + + SmallString<128> PtxFilePath; + if (std::error_code E = + sys::fs::createTemporaryFile("rust-offload", "ptx", PtxFilePath)) + return errorCodeToError(E); - SmallString<128> TempDir; + SmallString<128> CubinFilePath; if (std::error_code E = - sys::fs::createUniqueDirectory("rust-offload", TempDir)) + sys::fs::createTemporaryFile("rust-offload", "cubin", CubinFilePath)) return errorCodeToError(E); auto Cleanup = scope_exit([&] { - if (std::error_code E = sys::fs::remove_directories(TempDir)) + if (std::error_code E = sys::fs::remove(PtxFilePath)) + (void)reportAndFailWrappingImages( + errorCodeToError(E), "assembleWithPtxas: PtxFilePath cleanup"); + if (std::error_code E = sys::fs::remove(CubinFilePath)) (void)reportAndFailWrappingImages( - errorCodeToError(E), "compileAndLinkDeviceImages: tempdir cleanup"); + errorCodeToError(E), "assembleWithPtxas: CubinFilePath cleanup"); }); - SmallString<128> OutputPath(TempDir); - sys::path::append(OutputPath, "device.img"); - - SmallVector ArgStorage{ - ClangPath, - "--no-default-config", - "--target=" + DeviceTriple.str(), - "-o", - OutputPath.str().str(), - "-dumpdir", - OutputPath.str().str() + ".", + if (Error E = writeFile(PtxFilePath, Ptx)) + return std::move(E); + + const StringRef Args[] = { + *Ptxas, "-m64", "-O3", "--gpu-name", + Arch, "--output-file", CubinFilePath, PtxFilePath, }; - if (!Arch.empty() && Arch != "generic") - ArgStorage.push_back( - ((DeviceTriple.isAMDGPU() ? "-mcpu=" : "-march=") + Arch).str()); + std::string ErrorMsg; + const int Status = + sys::ExecuteAndWait(*Ptxas, Args, std::nullopt, {}, 0, 0, &ErrorMsg); - if (DeviceTriple.isAMDGPU()) - ArgStorage.push_back("-Wl,--no-undefined"); + if (Status != 0) + return createStringError("assembleWithPtxas: status %d: %s", Status, + ErrorMsg.c_str()); - SmallString<128> InputPath(TempDir); - sys::path::append(InputPath, "input.o"); + ErrorOr> CubinOrError = + MemoryBuffer::getFileAsStream(CubinFilePath); + if (!CubinOrError) + return errorCodeToError(CubinOrError.getError()); - if (Error E = writeFile(InputPath, Input.getImage())) - return std::move(E); + return std::move(*CubinOrError); +} + +static Expected> +linkWithRustLld(StringRef Obj, StringRef LldPath) { + SmallString<128> ObjFilePath; + if (std::error_code E = + sys::fs::createTemporaryFile("rust-offload", "o", ObjFilePath)) + return errorCodeToError(E); + + SmallString<128> SoFilePath; + if (std::error_code E = + sys::fs::createTemporaryFile("rust-offload", "so", SoFilePath)) + return errorCodeToError(E); + + auto Cleanup = scope_exit([&] { + if (std::error_code E = sys::fs::remove(ObjFilePath)) + (void)reportAndFailWrappingImages(errorCodeToError(E), + "linkWithRustLld: ObjFilePath cleanup"); + if (std::error_code E = sys::fs::remove(SoFilePath)) + (void)reportAndFailWrappingImages(errorCodeToError(E), + "linkWithRustLld: SoFilePath cleanup"); + }); - ArgStorage.push_back(InputPath.str().str()); + if (Error E = writeFile(ObjFilePath, Obj)) + return std::move(E); - SmallVector CmdArgs; - for (const StringRef Arg : ArgStorage) - CmdArgs.push_back(Arg); + const StringRef Args[] = { + LldPath, "-flavor", "gnu", "-shared", + "--no-undefined", "-o", SoFilePath, ObjFilePath, + }; - std::string ExecError; - int Status = sys::ExecuteAndWait(ClangPath, CmdArgs, std::nullopt, {}, 0, 0, - &ExecError); + std::string ErrorMsg; + const int Status = + sys::ExecuteAndWait(LldPath, Args, std::nullopt, {}, 0, 0, &ErrorMsg); if (Status != 0) - return createStringError("compileAndLinkDeviceImages: device compiler " - "failed for %s/%s (status %d): %s", - DeviceTriple.str().c_str(), Arch.str().c_str(), - Status, ExecError.c_str()); + return createStringError("linkWithRustLld: status %d: %s", Status, + ErrorMsg.c_str()); - auto ImageOrErr = MemoryBuffer::getFileAsStream(OutputPath); - if (!ImageOrErr) - return createFileError(OutputPath, ImageOrErr.getError()); + ErrorOr> ElfOrError = + MemoryBuffer::getFileAsStream(SoFilePath); + if (!ElfOrError) + return errorCodeToError(ElfOrError.getError()); + + return std::move(*ElfOrError); +} + +static Expected> +compileDeviceImage(const OffloadBinary &Input, const char *LldPath) { + const Triple DeviceTriple(Input.getTriple()); + const StringRef Arch = Input.getArch(); + + LLVMContext Ctx; + Expected> ImageObjOrError = + parseBitcodeFile(MemoryBufferRef(Input.getImage(), "device.bc"), Ctx); + if (!ImageObjOrError) + return ImageObjOrError.takeError(); + + std::string ErrorMsg; + const Target *DeviceTarget = + TargetRegistry::lookupTarget(DeviceTriple, ErrorMsg); + if (!DeviceTarget) + return createStringError(ErrorMsg); + + std::unique_ptr TM(DeviceTarget->createTargetMachine( + DeviceTriple, Arch, /*Features=*/"", TargetOptions(), Reloc::PIC_)); + if (!TM) + return createStringError("createTargetMachine failed for %s", + DeviceTriple.str().c_str()); + + const bool IsNvptx = DeviceTriple.isNVPTX(); + + legacy::PassManager PM; + SmallString<0> Emitted; + raw_svector_ostream OS(Emitted); + const CodeGenFileType FileType = + IsNvptx ? CodeGenFileType::AssemblyFile : CodeGenFileType::ObjectFile; + if (TM->addPassesToEmitFile(PM, OS, nullptr, FileType)) + return createStringError("target %s cannot emit %s", + DeviceTriple.str().c_str(), + IsNvptx ? "assembly" : "object"); + + PM.run(**ImageObjOrError); + + if (IsNvptx) + return assembleWithPtxas(Emitted, Arch); + if (DeviceTriple.isAMDGPU()) { + if (!LldPath || !*LldPath) + return createStringError("rust-lld path was not provided for %s", + DeviceTriple.str().c_str()); + return linkWithRustLld(Emitted, LldPath); + } - return std::move(*ImageOrErr); + return createStringError("unsupported offload target %s", + DeviceTriple.str().c_str()); } extern "C" bool LLVMRustOffloadWrapImages(LLVMModuleRef HostMRef, - const char *ClangPath, + const char *LldPath, const char *DeviceBinPath) { Module &M = *unwrap(HostMRef); if (!hasOffloadEntries(M)) @@ -246,9 +334,10 @@ extern "C" bool LLVMRustOffloadWrapImages(LLVMModuleRef HostMRef, const OffloadBinary &Input = *Binaries.front().getBinary(); - auto ImageOrErr = compileAndLinkDeviceImages(Input, ClangPath); + auto ImageOrErr = compileDeviceImage(Input, LldPath); if (!ImageOrErr) - return reportAndFailWrappingImages(ImageOrErr.takeError(), "device link"); + return reportAndFailWrappingImages(ImageOrErr.takeError(), + "device compile"); StringRef ImageBuf = (*ImageOrErr)->getBuffer(); ArrayRef Image(ImageBuf.data(), ImageBuf.size()); diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index b224f4b4dafed..fb83e4ae5754c 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -21,9 +21,7 @@ use tracing::span; use crate::core::backend::CodegenBackendKind; use crate::core::build_steps::gcc::{Gcc, GccOutput, GccTargetPair}; -use crate::core::build_steps::llvm::{ - LlvmFromCi, LlvmKind, offload_clang_lib_paths, offload_tool_paths, prebuilt_llvm_output, -}; +use crate::core::build_steps::llvm::{LlvmFromCi, LlvmKind, prebuilt_llvm_output}; use crate::core::build_steps::tool::{RustcPrivateCompilers, SourceType, copy_lld_artifacts}; use crate::core::build_steps::{dist, llvm}; use crate::core::builder::{ @@ -2323,15 +2321,6 @@ impl CommandLineStep for Assemble { let dst_lib = target_libdir.join(libname); builder.resolve_symlink_and_copy(&p, &dst_lib); } - - for (source, filename) in offload_tool_paths(builder, target_compiler.host) { - builder.resolve_symlink_and_copy(&source, &libdir_bin.join(filename)); - } - - for source in offload_clang_lib_paths(builder, target_compiler.host) { - let filename = source.file_name().unwrap(); - builder.resolve_symlink_and_copy(&source, &target_libdir.join(filename)); - } } } diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 3825b2e90d97d..a335172631307 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -27,7 +27,6 @@ use crate::core::build_steps::doc::DocumentationFormat; use crate::core::build_steps::gcc::GccTargetPair; use crate::core::build_steps::llvm::{ LLVM_CI_LINK_TYPE_PATH, LlvmBuildStatus, LlvmKind, get_llvm_build_status, - offload_clang_lib_paths, offload_tool_paths, }; use crate::core::build_steps::tool::{ self, RustcPrivateCompilers, ToolTargetBuildMode, get_tool_target_compiler, @@ -2901,21 +2900,7 @@ impl CommandLineStep for Offload { tarball.add_file(path, destdir, FileType::NativeLibrary); } - tarball.add_file(rust_offload.rust_offload_path(), &target_libdir, FileType::NativeLibrary); - - let target_bindir = PathBuf::from(format!("lib/rustlib/{}/bin", target.triple)); - for (source, filename) in offload_tool_paths(builder, target) { - let source = t!(fs::canonicalize(source)); - tarball.add_renamed_file(source, &target_bindir, &filename, FileType::Executable); - } - - for source in offload_clang_lib_paths(builder, target) { - let filename = source.file_name().unwrap().to_str().unwrap(); - let resolved = t!(fs::canonicalize(&source)); - tarball.add_renamed_file(resolved, &target_libdir, filename, FileType::NativeLibrary); - } - - maybe_install_llvm_target(builder, target, tarball.image_dir()); + tarball.add_file(rust_offload.rust_offload_path(), target_libdir, FileType::NativeLibrary); Some(tarball.generate()) } diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index d39be4b19e7ff..f1864c736ed03 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -1211,63 +1211,6 @@ impl CommandLineStep for RustOffload { } } -/// Returns the binary and library directories of the selected clang. -fn offload_clang_dirs(builder: &Builder<'_>, target: TargetSelection) -> (PathBuf, PathBuf) { - if builder.config.llvm_clang { - let llvm = builder.ensure(llvm::Llvm { target }); - return (llvm.root_dir().join("bin"), llvm.root_dir().join("lib")); - } - - let lib_dir = builder - .config - .offload_clang_dir - .as_deref() - .and_then(|dir| dir.ancestors().nth(2)) - .expect("llvm.offload-clang-dir must point to //cmake/clang"); - - let root = lib_dir.parent().expect("Clang library directory must have a parent"); - - (root.join("bin"), lib_dir.into()) -} - -/// Returns (source path, destination filename) pairs for offloading tools such as clang. -pub(crate) fn offload_tool_paths( - builder: &Builder<'_>, - target: TargetSelection, -) -> Vec<(PathBuf, String)> { - let lld = builder.ensure(llvm::Lld { target }); - let (clang_bin_dir, _) = offload_clang_dirs(builder, target); - - let mut tools = Vec::new(); - - for name in ["clang", "clang-nvlink-wrapper"] { - let filename = exe(name, target); - let source = clang_bin_dir.join(&filename); - tools.push((source, filename)); - } - - tools.push((lld.join("bin").join(exe("lld", target)), exe("ld.lld", target))); - - tools -} - -pub(crate) fn offload_clang_lib_paths( - builder: &Builder<'_>, - target: TargetSelection, -) -> Vec { - let (_, lib_dir) = offload_clang_dirs(builder, target); - - let mut paths = Vec::new(); - for entry in builder.read_dir(&lib_dir) { - let filename = entry.file_name(); - let filename = filename.to_string_lossy(); - if filename == "libclang-cpp.so" || filename.starts_with("libclang-cpp.so.") { - paths.push(entry.path()); - } - } - paths -} - #[derive(Clone)] pub struct BuiltOmpOffload { /// Path to the omp and offload dylibs.