From 2fa7f74c9e401f80f3a3e74615009244ba353a66 Mon Sep 17 00:00:00 2001 From: Stephan Seitz Date: Tue, 21 Jul 2026 13:44:48 +0200 Subject: [PATCH 1/2] fix: remove lifetime from RuntimeCache RuntimeCache can outlive an engine and be shared between engines. --- trtx/src/runtime_cache.rs | 16 ++++++---------- trtx/src/runtime_config.rs | 8 ++++---- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/trtx/src/runtime_cache.rs b/trtx/src/runtime_cache.rs index f2025ab..84567cc 100644 --- a/trtx/src/runtime_cache.rs +++ b/trtx/src/runtime_cache.rs @@ -2,8 +2,6 @@ //! //! [`RuntimeCache`] wraps [`trtx_sys::nvinfer1::IRuntimeCache`] (C++ [`nvinfer1::IRuntimeCache`](https://docs.nvidia.com/deeplearning/tensorrt-rtx/latest/_static/cpp-api/classnvinfer1_1_1_i_runtime_cache.html). -use std::marker::PhantomData; - use crate::error::{PropertySetAttempt, Result}; use crate::host_memory::HostMemory; use crate::Error; @@ -11,19 +9,18 @@ use cxx::UniquePtr; use trtx_sys::nvinfer1::{self, IRuntimeCache}; /// [`trtx_sys::nvinfer1::IRuntimeCache`] — C++ [`nvinfer1::IRuntimeCache`](https://docs.nvidia.com/deeplearning/tensorrt-rtx/latest/_static/cpp-api/classnvinfer1_1_1_i_runtime_cache.html). -pub struct RuntimeCache<'engine> { +pub struct RuntimeCache { pub(crate) inner: UniquePtr, - _engine: PhantomData<&'engine nvinfer1::ICudaEngine>, } /// # Safety /// /// IRuntimeCache is internally protected by a shared mutex and /// UniquePtr holds after initialization a valid IRuntimeCache (or nullptr in mock mode) -unsafe impl Send for RuntimeCache<'_> {} -unsafe impl Sync for RuntimeCache<'_> {} +unsafe impl Send for RuntimeCache {} +unsafe impl Sync for RuntimeCache {} -impl std::fmt::Debug for RuntimeCache<'_> { +impl std::fmt::Debug for RuntimeCache { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RuntimeCache") .field("inner", &format!("{:x}", self.inner.as_ptr() as usize)) @@ -31,7 +28,7 @@ impl std::fmt::Debug for RuntimeCache<'_> { } } -impl<'engine> RuntimeCache<'engine> { +impl RuntimeCache { pub(crate) fn new(cache: *mut nvinfer1::IRuntimeCache) -> Result { #[cfg(not(feature = "mock"))] if cache.is_null() { @@ -39,12 +36,11 @@ impl<'engine> RuntimeCache<'engine> { } Ok(Self { inner: unsafe { UniquePtr::from_raw(cache) }, - _engine: Default::default(), }) } /// See [IRuntimeCache::serialize]. - pub fn serialize(&self) -> Result> { + pub fn serialize(&self) -> Result> { #[cfg(not(feature = "mock"))] { let host_mem = unsafe { self.inner.serialize().as_mut() } diff --git a/trtx/src/runtime_config.rs b/trtx/src/runtime_config.rs index b143d7a..e2a8a2d 100644 --- a/trtx/src/runtime_config.rs +++ b/trtx/src/runtime_config.rs @@ -27,8 +27,8 @@ pub struct RuntimeConfig<'engine> { // this also makes it safe when we modify through our mutex, while cpp calls are made through // IExecution calls #[cfg(not(feature = "enterprise"))] - _cache: Option>>>, // Mutex, could now be removed with a - // breaking change to set_runtime_cache + _cache: Option>>, // Mutex, could now be removed with a + // breaking change to set_runtime_cache } impl std::fmt::Debug for RuntimeConfig<'_> { @@ -83,7 +83,7 @@ impl<'engine> RuntimeConfig<'engine> { #[cfg(not(feature = "enterprise"))] /// See [IRuntimeConfig::createRuntimeCache]. - pub fn create_runtime_cache(&self) -> Result> { + pub fn create_runtime_cache(&self) -> Result { #[cfg(not(feature = "mock"))] let cache_ptr = self.inner.createRuntimeCache(); #[cfg(feature = "mock")] @@ -93,7 +93,7 @@ impl<'engine> RuntimeConfig<'engine> { #[cfg(not(feature = "enterprise"))] /// See [IRuntimeConfig::setRuntimeCache]. - pub fn set_runtime_cache(&mut self, cache: Arc>>) -> Result<()> { + pub fn set_runtime_cache(&mut self, cache: Arc>) -> Result<()> { if cfg!(not(feature = "mock")) { if self.inner.pin_mut().setRuntimeCache( cache From c511217c02c653d630c90a5966922ae065815c43 Mon Sep 17 00:00:00 2001 From: Stephan Seitz Date: Thu, 30 Jul 2026 14:16:50 +0200 Subject: [PATCH 2/2] chore: make all methods of RuntimeCache work with a shared reference RuntimeCache is internally synchronized with a shared mutex. So all methods can be used safely with a shared reference. --- trtexec-rs/Cargo.toml | 4 ++-- trtx-sys/build.rs | 7 +++++++ trtx/src/runtime_cache.rs | 7 +++---- trtx/src/runtime_config.rs | 9 +++------ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/trtexec-rs/Cargo.toml b/trtexec-rs/Cargo.toml index c8ae798..fd0d608 100644 --- a/trtexec-rs/Cargo.toml +++ b/trtexec-rs/Cargo.toml @@ -7,9 +7,9 @@ authors.workspace = true repository.workspace = true [dependencies] -rustnn = { git = "https://github.com/rustnn/rustnn/", features = [ +rustnn = { git = "https://github.com/rustnn/rustnn/", branch = "update-trtx-1.6", features = [ "trtx-runtime", -], default-features = false, branch = "main", optional = true } +], default-features = false, optional = true } nvidia-nvtx = { version = "0.2", git = "https://github.com/NVIDIA/NVTX", branch = "release-v3" } log = "0.4" diff --git a/trtx-sys/build.rs b/trtx-sys/build.rs index 5dcb25c..5b32ee6 100644 --- a/trtx-sys/build.rs +++ b/trtx-sys/build.rs @@ -94,6 +94,13 @@ fn prepare_transformed_headers(header_dir: &Path, out_dir: &Path) -> PathBuf { "void log(Severity severity, AsciiChar const* msg)", "void log(int32_t severity, char const* msg)", ) + // RuntimeCache is internally synchronized via shared_mutex + .replace( + "bool deserialize(void const* blob, size_t size) noexcept", + "bool deserialize(void const* blob, size_t size) const noexcept", + ) + // RuntimeCache is internally synchronized via shared_mutex + .replace("bool reset() noexcept", "bool reset() const noexcept") .replace("//!", "///") .replace(r"\returns", " - Returns "); diff --git a/trtx/src/runtime_cache.rs b/trtx/src/runtime_cache.rs index 84567cc..c811dea 100644 --- a/trtx/src/runtime_cache.rs +++ b/trtx/src/runtime_cache.rs @@ -52,11 +52,10 @@ impl RuntimeCache { } /// See [IRuntimeCache::deserialize]. - pub fn deserialize(&mut self, blob: &[u8]) -> Result<()> { + pub fn deserialize(&self, blob: &[u8]) -> Result<()> { if cfg!(not(feature = "mock")) { if unsafe { self.inner - .pin_mut() .deserialize(blob.as_ptr() as *const autocxx::c_void, blob.len()) } { Ok(()) @@ -71,9 +70,9 @@ impl RuntimeCache { } /// See [IRuntimeCache::reset]. - pub fn reset(&mut self) -> Result<()> { + pub fn reset(&self) -> Result<()> { if cfg!(not(feature = "mock")) { - if self.inner.pin_mut().reset() { + if self.inner.reset() { Ok(()) } else { Err(Error::FailedToResetRuntimeCache) diff --git a/trtx/src/runtime_config.rs b/trtx/src/runtime_config.rs index e2a8a2d..b5191e9 100644 --- a/trtx/src/runtime_config.rs +++ b/trtx/src/runtime_config.rs @@ -4,7 +4,7 @@ use std::marker::PhantomData; #[cfg(not(feature = "enterprise"))] -use std::sync::{Arc, Mutex}; +use std::sync::Arc; #[cfg(not(feature = "enterprise"))] use crate::error::PropertySetAttempt; @@ -27,8 +27,7 @@ pub struct RuntimeConfig<'engine> { // this also makes it safe when we modify through our mutex, while cpp calls are made through // IExecution calls #[cfg(not(feature = "enterprise"))] - _cache: Option>>, // Mutex, could now be removed with a - // breaking change to set_runtime_cache + _cache: Option>, } impl std::fmt::Debug for RuntimeConfig<'_> { @@ -93,12 +92,10 @@ impl<'engine> RuntimeConfig<'engine> { #[cfg(not(feature = "enterprise"))] /// See [IRuntimeConfig::setRuntimeCache]. - pub fn set_runtime_cache(&mut self, cache: Arc>) -> Result<()> { + pub fn set_runtime_cache(&mut self, cache: Arc) -> Result<()> { if cfg!(not(feature = "mock")) { if self.inner.pin_mut().setRuntimeCache( cache - .lock() - .unwrap() .inner .as_ref() .expect("RuntimeCache inner must be non-null"),