diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs
index f89ccb4c2a6a8..096184ee9e5b4 100644
--- a/src/bootstrap/src/core/build_steps/llvm.rs
+++ b/src/bootstrap/src/core/build_steps/llvm.rs
@@ -1409,6 +1409,21 @@ impl CommandLineStep for OmpOffload {
             ldflags.push_all(format!("-L{}", dir.display()));
         }
 
+        if builder.config.rpath_enabled(target)
+            && helpers::use_host_linker(target)
+            && llvm_output.link_shared()
+            && target.contains("linux")
+        {
+            // Same logic as in Lld::run
+            // We inform libomptarget.so where it can find LLVM's libraries
+            // by adding an rpath entry to the expected parent `lib` directory.
+            //
+            // Be careful when changing this path, we need to ensure it's quoted or escaped:
+            // `$ORIGIN` would otherwise be expanded when the `LdFlags` are passed verbatim to
+            // cmake.
+            ldflags.push_all("-Wl,-rpath,'$ORIGIN/../../../'");
+        }
+
         configure_cmake(builder, target, &mut cfg, true, ldflags, cflags, &[]);
 
         cfg.define("CMAKE_C_COMPILER", &clang)