From 7856c4b4fbb63b3a114e8d1f813ce261a69ac6d4 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Fri, 6 Feb 2026 04:00:50 +0100 Subject: [PATCH 01/16] perf: lock-free reads --- src/lib.rs | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6ca55e0..875e46d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -307,7 +307,34 @@ where bucket: &Bucket<(K, V)>, tag: usize, ) -> Option { - if bucket.try_lock(Some(tag)) { + if !Self::NEEDS_DROP { + // Lock-free speculative read for types without destructors. + // + // 1. Load tag (Acquire — orders subsequent reads after this). + // 2. Speculatively read data via ptr::read (no lock held). + // 3. Re-load tag — if unchanged, no writer was active during our window. + // 4. Validate key and return. + // + // SAFETY: `!NEEDS_DROP` guarantees no destructors, so a speculative read cannot + // cause use-after-free. If a concurrent write produces a torn read, the tag re-check + // will detect it (the writer changes the tag under its lock/unlock cycle). + let seq1 = bucket.tag.load(Ordering::Acquire); + if seq1 == tag { + let copy = unsafe { std::ptr::read(bucket.data.get()) }; + if seq1 == bucket.tag.load(Ordering::Acquire) { + let (ck, v) = unsafe { copy.assume_init() }; + if key.equivalent(&ck) { + #[cfg(feature = "stats")] + if C::STATS + && let Some(stats) = &self.stats + { + stats.record_hit(&ck, &v); + } + return Some(v); + } + } + } + } else if bucket.try_lock(Some(tag)) { // SAFETY: We hold the lock and bucket is alive, so we have exclusive access. let (ck, v) = unsafe { (*bucket.data.get()).assume_init_ref() }; if key.equivalent(ck) { @@ -433,7 +460,8 @@ where where F: FnOnce(&K) -> V, { - self.get_or_try_insert_with(key, |key| Ok::<_, Infallible>(f(key))).unwrap() + let Ok(v) = self.get_or_try_insert_with(key, |key| Ok::<_, Infallible>(f(key))); + v } /// Gets a value from the cache, or inserts one computed by `f` if not present. @@ -452,7 +480,8 @@ where F: FnOnce(&'a Q) -> V, Cvt: FnOnce(&'a Q) -> K, { - self.get_or_try_insert_with_ref(key, |key| Ok::<_, Infallible>(f(key)), cvt).unwrap() + let Ok(v) = self.get_or_try_insert_with_ref(key, |key| Ok::<_, Infallible>(f(key)), cvt); + v } /// Gets a value from the cache, or attempts to insert one computed by `f` if not present. From 8edc8723a142b082a5899681490395ff2139e205 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Fri, 6 Feb 2026 05:20:44 +0100 Subject: [PATCH 02/16] cleanup --- src/lib.rs | 64 ++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 875e46d..7311dc5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -320,18 +320,15 @@ where // will detect it (the writer changes the tag under its lock/unlock cycle). let seq1 = bucket.tag.load(Ordering::Acquire); if seq1 == tag { - let copy = unsafe { std::ptr::read(bucket.data.get()) }; - if seq1 == bucket.tag.load(Ordering::Acquire) { - let (ck, v) = unsafe { copy.assume_init() }; - if key.equivalent(&ck) { - #[cfg(feature = "stats")] - if C::STATS - && let Some(stats) = &self.stats - { - stats.record_hit(&ck, &v); - } - return Some(v); + let (ck, v) = unsafe { std::ptr::read(bucket.data.get().cast::<(K, V)>()) }; + if seq1 == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { + #[cfg(feature = "stats")] + if C::STATS + && let Some(stats) = &self.stats + { + stats.record_hit(&ck, &v); } + return Some(v); } } } else if bucket.try_lock(Some(tag)) { @@ -362,7 +359,7 @@ where /// Insert an entry into the cache. pub fn insert(&self, key: K, value: V) { let (bucket, tag) = self.calc(&key); - self.insert_inner(|| key, || value, bucket, tag); + self.insert_inner(bucket, tag, || (key, value)); } /// Remove an entry from the cache. @@ -397,11 +394,25 @@ where #[inline] fn insert_inner( &self, - make_key: impl FnOnce() -> K, - make_value: impl FnOnce() -> V, bucket: &Bucket<(K, V)>, tag: usize, + make_entry: impl FnOnce() -> (K, V), ) { + #[inline(always)] + unsafe fn do_write(ptr: *mut T, f: impl FnOnce() -> T) { + // This function is translated as: + // - allocate space for a T on the stack + // - call f() with the return value being put onto this stack space + // - memcpy from the stack to the heap + // + // Ideally we want LLVM to always realize that doing a stack + // allocation is unnecessary and optimize the code so it writes + // directly into the heap instead. It seems we get it to realize + // this most consistently if we put this critical line into it's + // own function instead of inlining it into the surrounding code. + unsafe { core::ptr::write(ptr, f()) }; + } + if let Some(prev_tag) = bucket.try_lock_ret(None) { // SAFETY: We hold the lock, so we have exclusive access. unsafe { @@ -409,12 +420,11 @@ where let is_alive = (prev_tag & !LOCKED_BIT) != 0; let data = (&mut *bucket.data.get()).as_mut_ptr(); - #[cfg(feature = "stats")] - if C::STATS { + if C::STATS && cfg!(feature = "stats") { + #[cfg(feature = "stats")] if is_alive { // Replace key/value and get the old ones for stats and dropping - let old_key = std::ptr::replace(&raw mut (*data).0, make_key()); - let old_value = std::ptr::replace(&raw mut (*data).1, make_value()); + let (old_key, old_value) = std::ptr::replace(data, make_entry()); if let Some(stats) = &self.stats { stats.record_insert( &(*data).0, @@ -423,8 +433,7 @@ where ); } } else { - (&raw mut (*data).0).write(make_key()); - (&raw mut (*data).1).write(make_value()); + do_write(data, make_entry); if let Some(stats) = &self.stats { stats.record_insert(&(*data).0, &(*data).1, None); } @@ -433,18 +442,7 @@ where if Self::NEEDS_DROP && is_alive { std::ptr::drop_in_place(data); } - (&raw mut (*data).0).write(make_key()); - (&raw mut (*data).1).write(make_value()); - } - - #[cfg(not(feature = "stats"))] - { - // Drop old value if bucket was alive. - if Self::NEEDS_DROP && is_alive { - std::ptr::drop_in_place(data); - } - (&raw mut (*data).0).write(make_key()); - (&raw mut (*data).1).write(make_value()); + do_write(data, make_entry); } } bucket.unlock(tag); @@ -531,7 +529,7 @@ where return Ok(v); } let value = f(key)?; - self.insert_inner(|| cvt(key), || value.clone(), bucket, tag); + self.insert_inner(bucket, tag, || (cvt(key), value.clone())); Ok(value) } From 1fa99cc7371227f085e6c8055012c8a90594acae Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Fri, 6 Feb 2026 07:40:01 +0100 Subject: [PATCH 03/16] test --- src/lib.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 7311dc5..a329f2c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -951,6 +951,32 @@ mod tests { }); } + #[test] + fn seqlock_aba() { + if cfg!(miri) { + return; + } + + const VALUE_N: usize = 16; + + let cache: Cache = new_cache(1024); + let n = iters(500_000); + + run_concurrent(4, |t| { + if t == 0 { + for i in 0..n as u64 { + cache.insert(1, [i; VALUE_N]); + } + } else { + for _ in 0..n { + if let Some(v) = cache.get(&1) { + assert!(v.windows(2).all(|w| w[0] == w[1]), "torn read: {v:?}"); + } + } + } + }); + } + #[test] fn concurrent_get_or_insert() { let cache: Cache = new_cache(1024); From 17582067e63728be544fc2b13e4186deef0e15a4 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 02:59:26 +0100 Subject: [PATCH 04/16] fix --- src/lib.rs | 214 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 165 insertions(+), 49 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a329f2c..dbcf74f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,11 @@ const EPOCH_SHIFT: usize = NEEDED_BITS; const EPOCH_MASK: usize = ((1 << EPOCH_BITS) - 1) << EPOCH_SHIFT; const EPOCH_NEEDED_BITS: usize = NEEDED_BITS + EPOCH_BITS; +const VERSION_BITS: u32 = 8; +const VERSION_SHIFT: u32 = usize::BITS - VERSION_BITS; +const VERSION_MASK: usize = ((1usize << VERSION_BITS) - 1) << VERSION_SHIFT; +const VERSION_INCREMENT: usize = 1usize << VERSION_SHIFT; + #[cfg(feature = "rapidhash")] type DefaultBuildHasher = std::hash::BuildHasherDefault>; #[cfg(not(feature = "rapidhash"))] @@ -308,20 +313,13 @@ where tag: usize, ) -> Option { if !Self::NEEDS_DROP { - // Lock-free speculative read for types without destructors. - // - // 1. Load tag (Acquire — orders subsequent reads after this). - // 2. Speculatively read data via ptr::read (no lock held). - // 3. Re-load tag — if unchanged, no writer was active during our window. - // 4. Validate key and return. - // - // SAFETY: `!NEEDS_DROP` guarantees no destructors, so a speculative read cannot - // cause use-after-free. If a concurrent write produces a torn read, the tag re-check - // will detect it (the writer changes the tag under its lock/unlock cycle). - let seq1 = bucket.tag.load(Ordering::Acquire); - if seq1 == tag { - let (ck, v) = unsafe { std::ptr::read(bucket.data.get().cast::<(K, V)>()) }; - if seq1 == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { + let tag2 = bucket.tag.load(Ordering::Acquire); + if (tag2 & LOCKED_BIT) == 0 && (tag2 & !VERSION_MASK) == tag { + let (ck, v) = unsafe { bucket.data.get().cast::<(K, V)>().read() }; + std::sync::atomic::compiler_fence(Ordering::AcqRel); + #[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))] + std::sync::atomic::fence(Ordering::Acquire); + if tag2 == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { #[cfg(feature = "stats")] if C::STATS && let Some(stats) = &self.stats @@ -367,7 +365,7 @@ where /// Returns the value if the key was present in the cache. pub fn remove>(&self, key: &Q) -> Option { let (bucket, tag) = self.calc(key); - if bucket.try_lock(Some(tag)) { + if let Some(prev) = bucket.try_lock_ret(Some(tag)) { // SAFETY: We hold the lock and bucket is alive, so we have exclusive access. let data = unsafe { &mut *bucket.data.get() }; let (ck, v) = unsafe { data.assume_init_ref() }; @@ -383,10 +381,12 @@ where // SAFETY: We hold the lock, so we have exclusive access. unsafe { data.assume_init_drop() }; } - bucket.unlock(0); + let new_version = prev.wrapping_add(VERSION_INCREMENT) & VERSION_MASK; + bucket.unlock(new_version); return Some(v); } - bucket.unlock(tag); + let new_version = prev & VERSION_MASK; + bucket.unlock(tag | new_version); } None } @@ -413,40 +413,46 @@ where unsafe { core::ptr::write(ptr, f()) }; } - if let Some(prev_tag) = bucket.try_lock_ret(None) { - // SAFETY: We hold the lock, so we have exclusive access. - unsafe { - // Check if bucket had data (any bits besides LOCKED_BIT means it was written to) - let is_alive = (prev_tag & !LOCKED_BIT) != 0; - let data = (&mut *bucket.data.get()).as_mut_ptr(); + let existing = bucket.tag.load(Ordering::Relaxed); + if existing & LOCKED_BIT != 0 { + return; + } + let new_version = existing.wrapping_add(VERSION_INCREMENT) & VERSION_MASK; + let to_store = tag | new_version; + if bucket + .tag + .compare_exchange(existing, to_store | LOCKED_BIT, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + return; + } - if C::STATS && cfg!(feature = "stats") { - #[cfg(feature = "stats")] - if is_alive { - // Replace key/value and get the old ones for stats and dropping - let (old_key, old_value) = std::ptr::replace(data, make_entry()); - if let Some(stats) = &self.stats { - stats.record_insert( - &(*data).0, - &(*data).1, - Some((&old_key, &old_value)), - ); - } - } else { - do_write(data, make_entry); - if let Some(stats) = &self.stats { - stats.record_insert(&(*data).0, &(*data).1, None); - } + // SAFETY: We hold the lock, so we have exclusive access. + unsafe { + let is_alive = existing & ALIVE_BIT != 0; + let data = bucket.data.get().cast::<(K, V)>(); + + if C::STATS && cfg!(feature = "stats") { + #[cfg(feature = "stats")] + if is_alive { + let (old_key, old_value) = std::ptr::replace(data, make_entry()); + if let Some(stats) = &self.stats { + stats.record_insert(&(*data).0, &(*data).1, Some((&old_key, &old_value))); } } else { - if Self::NEEDS_DROP && is_alive { - std::ptr::drop_in_place(data); - } do_write(data, make_entry); + if let Some(stats) = &self.stats { + stats.record_insert(&(*data).0, &(*data).1, None); + } } + } else { + if Self::NEEDS_DROP && is_alive { + std::ptr::drop_in_place(data); + } + do_write(data, make_entry); } - bucket.unlock(tag); } + bucket.unlock(to_store); } /// Gets a value from the cache, or inserts one computed by `f` if not present. @@ -538,10 +544,7 @@ where let hash = self.hash_key(key); // SAFETY: index is masked to be within bounds. let bucket = unsafe { (&*self.entries).get_unchecked(hash & self.index_mask()) }; - let mut tag = hash & self.tag_mask(); - if Self::NEEDS_DROP { - tag |= ALIVE_BIT; - } + let mut tag = hash & self.tag_mask() & !VERSION_MASK | ALIVE_BIT; if C::EPOCHS { tag = (tag & !EPOCH_MASK) | ((self.epoch() << EPOCH_SHIFT) & EPOCH_MASK); } @@ -602,7 +605,7 @@ impl Bucket { fn try_lock_ret(&self, expected: Option) -> Option { let state = self.tag.load(Ordering::Relaxed); if let Some(expected) = expected { - if state != expected { + if state & !VERSION_MASK != expected { return None; } } else if state & LOCKED_BIT != 0 { @@ -1242,4 +1245,117 @@ mod tests { assert_eq!(cache.get(&42), None, "failed at clear #{i}"); } } + + #[test] + fn remove_seqlock_type() { + let cache = new_cache::(64); + + cache.insert(1, 100); + assert_eq!(cache.get(&1), Some(100)); + + let removed = cache.remove(&1); + assert_eq!(removed, Some(100)); + assert_eq!(cache.get(&1), None); + + cache.insert(1, 200); + assert_eq!(cache.get(&1), Some(200)); + } + + #[test] + fn remove_then_reinsert_seqlock() { + let cache = new_cache::(64); + + for i in 0..100u64 { + cache.insert(1, i); + assert_eq!(cache.get(&1), Some(i)); + assert_eq!(cache.remove(&1), Some(i)); + assert_eq!(cache.get(&1), None); + } + } + + #[test] + fn epoch_with_needs_drop() { + let cache: EpochCache = EpochCache::new(4096, Default::default()); + + cache.insert("key".to_string(), "value".to_string()); + assert_eq!(cache.get("key"), Some("value".to_string())); + + cache.clear(); + assert_eq!(cache.get("key"), None); + + cache.insert("key".to_string(), "value2".to_string()); + assert_eq!(cache.get("key"), Some("value2".to_string())); + } + + #[test] + fn epoch_remove() { + let cache: EpochCache = EpochCache::new(4096, Default::default()); + + cache.insert(1, 100); + assert_eq!(cache.remove(&1), Some(100)); + assert_eq!(cache.get(&1), None); + + cache.insert(1, 200); + assert_eq!(cache.get(&1), Some(200)); + + cache.clear(); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.remove(&1), None); + } + + #[test] + fn no_stats_needs_drop() { + let cache: NoStatsCache = NoStatsCache::new(64, Default::default()); + + cache.insert("a".to_string(), "b".to_string()); + assert_eq!(cache.get("a"), Some("b".to_string())); + + cache.insert("a".to_string(), "c".to_string()); + assert_eq!(cache.get("a"), Some("c".to_string())); + + cache.remove(&"a".to_string()); + assert_eq!(cache.get("a"), None); + } + + #[test] + fn no_stats_get_or_insert() { + let cache: NoStatsCache = NoStatsCache::new(64, Default::default()); + + let v = cache.get_or_insert_with_ref("hello", |s| s.len(), |s| s.to_string()); + assert_eq!(v, 5); + + let v2 = cache.get_or_insert_with_ref("hello", |_| 999, |s| s.to_string()); + assert_eq!(v2, 5); + } + + #[test] + fn epoch_concurrent_seqlock() { + if cfg!(miri) { + return; + } + + let cache: EpochCache = EpochCache::new(4096, Default::default()); + let n = iters(10_000); + + run_concurrent(4, |t| { + for i in 0..n as u64 { + match t { + 0 => { + cache.insert(i % 50, i); + } + 1 => { + let _ = cache.get(&(i % 50)); + } + 2 => { + if i % 100 == 0 { + cache.clear(); + } + } + _ => { + let _ = cache.remove(&(i % 50)); + } + } + } + }); + } } From 666fc43b88f98017e7fae9efb22b39f361811e81 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 05:13:17 +0100 Subject: [PATCH 05/16] fixty --- src/lib.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index dbcf74f..0432d35 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -329,7 +329,7 @@ where return Some(v); } } - } else if bucket.try_lock(Some(tag)) { + } else if let Some(prev) = bucket.try_lock_ret(Some(tag)) { // SAFETY: We hold the lock and bucket is alive, so we have exclusive access. let (ck, v) = unsafe { (*bucket.data.get()).assume_init_ref() }; if key.equivalent(ck) { @@ -340,10 +340,10 @@ where stats.record_hit(ck, v); } let v = v.clone(); - bucket.unlock(tag); + bucket.unlock(prev & !LOCKED_BIT); return Some(v); } - bucket.unlock(tag); + bucket.unlock(prev & !LOCKED_BIT); } #[cfg(feature = "stats")] if C::STATS @@ -596,11 +596,6 @@ impl Bucket { Self { tag: AtomicUsize::new(0), data: UnsafeCell::new(MaybeUninit::zeroed()) } } - #[inline] - fn try_lock(&self, expected: Option) -> bool { - self.try_lock_ret(expected).is_some() - } - #[inline] fn try_lock_ret(&self, expected: Option) -> Option { let state = self.tag.load(Ordering::Relaxed); From 3df299f3f7a41f917391a754b611edacc0540cf6 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 06:55:33 +0100 Subject: [PATCH 06/16] nit --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 0432d35..ac5d479 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -316,7 +316,7 @@ where let tag2 = bucket.tag.load(Ordering::Acquire); if (tag2 & LOCKED_BIT) == 0 && (tag2 & !VERSION_MASK) == tag { let (ck, v) = unsafe { bucket.data.get().cast::<(K, V)>().read() }; - std::sync::atomic::compiler_fence(Ordering::AcqRel); + std::sync::atomic::compiler_fence(Ordering::Acquire); #[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))] std::sync::atomic::fence(Ordering::Acquire); if tag2 == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { From 2f89c6e604518bd3e580edf95ee8a72b98ed5ea8 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 06:59:20 +0100 Subject: [PATCH 07/16] impls --- Cargo.toml | 1 + src/lib.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index aedf051..1c6da3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ categories = ["caching", "concurrency", "data-structures"] [dependencies] equivalent = "1" +impls = "1" rapidhash = { version = "4", default-features = false, optional = true } typeid = { version = "1", default-features = false, optional = true } diff --git a/src/lib.rs b/src/lib.rs index ac5d479..4f24aea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -138,6 +138,7 @@ where C: CacheConfig, { const NEEDS_DROP: bool = Bucket::<(K, V)>::NEEDS_DROP; + const ENTRY_IMPLS_COPY: bool = impls::impls!((K, V): Copy); /// Create a new cache with the specified number of entries and hasher. /// @@ -312,7 +313,7 @@ where bucket: &Bucket<(K, V)>, tag: usize, ) -> Option { - if !Self::NEEDS_DROP { + if Self::ENTRY_IMPLS_COPY { let tag2 = bucket.tag.load(Ordering::Acquire); if (tag2 & LOCKED_BIT) == 0 && (tag2 & !VERSION_MASK) == tag { let (ck, v) = unsafe { bucket.data.get().cast::<(K, V)>().read() }; From 79c907032da655def36dbc612a495ad3a2e2c5d3 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 06:59:55 +0100 Subject: [PATCH 08/16] nit --- src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4f24aea..b0f4261 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -317,9 +317,11 @@ where let tag2 = bucket.tag.load(Ordering::Acquire); if (tag2 & LOCKED_BIT) == 0 && (tag2 & !VERSION_MASK) == tag { let (ck, v) = unsafe { bucket.data.get().cast::<(K, V)>().read() }; - std::sync::atomic::compiler_fence(Ordering::Acquire); - #[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))] - std::sync::atomic::fence(Ordering::Acquire); + if cfg!(any(target_arch = "x86_64", target_arch = "x86")) { + std::sync::atomic::compiler_fence(Ordering::Acquire); + } else { + std::sync::atomic::fence(Ordering::Acquire); + } if tag2 == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { #[cfg(feature = "stats")] if C::STATS From dd4233a8efc0656f2c90a2b0f4c3016c734101e9 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 16:59:32 +0100 Subject: [PATCH 09/16] clean+comments --- src/lib.rs | 66 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b0f4261..6f9973f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,29 @@ mod stats; #[cfg_attr(docsrs, doc(cfg(feature = "stats")))] pub use stats::{AnyRef, CountingStatsHandler, Stats, StatsHandler}; +// Tag bit layout (64-bit): +// +// 63 56 55 12 11 2 1 0 +// +----------+----------------------------------+----------+--+--+ +// | version | hash signature | epoch | A| L| +// | (8-bit) | (variable, capacity-dependent) | (10-bit) | | | +// +----------+----------------------------------+----------+--+--+ +// +// L (locked): Set during writes. Readers (seqlock path) skip the bucket; locked-path readers +// fail the CAS and fall through to a miss. Writers fail the CAS and abandon the insert. +// A (alive): Indicates the bucket contains initialized data. Cleared on remove, set on insert. +// Part of the tag identity: readers require it to match, so empty buckets are never "hit". +// epoch: Tracks which epoch the entry belongs to, enabling O(1) cache invalidation via +// `Cache::clear`. Only present when `CacheConfig::EPOCHS` is true; otherwise these bits +// are part of the hash signature. +// hash signature: Upper bits of the key's hash, used to reject non-matching keys without +// reading the bucket data. The exact width depends on the cache capacity (more buckets = +// fewer signature bits). +// version: Monotonic counter incremented on every mutation (insert/remove). Used by the +// seqlock read path to detect concurrent writes: the reader snapshots the tag, speculatively +// reads the data, then verifies the tag hasn't changed. The 8-bit counter makes ABA +// wraparound (256 writes between two reader loads) effectively impossible in practice. + const LOCKED_BIT: usize = 1 << 0; const ALIVE_BIT: usize = 1 << 1; const NEEDED_BITS: usize = 2; @@ -314,6 +337,11 @@ where tag: usize, ) -> Option { if Self::ENTRY_IMPLS_COPY { + // Seqlock fast path: speculatively read the tag, then the data, then re-read + // the tag. If the tag hasn't changed between the two reads (including the version + // counter), no writer intervened and the data is consistent. This avoids acquiring + // the lock entirely on cache hits, at the cost of occasionally discarding a read + // if a concurrent write raced with us. let tag2 = bucket.tag.load(Ordering::Acquire); if (tag2 & LOCKED_BIT) == 0 && (tag2 & !VERSION_MASK) == tag { let (ck, v) = unsafe { bucket.data.get().cast::<(K, V)>().read() }; @@ -332,7 +360,7 @@ where return Some(v); } } - } else if let Some(prev) = bucket.try_lock_ret(Some(tag)) { + } else if let Some(prev) = bucket.try_lock_ret(Some(tag), false) { // SAFETY: We hold the lock and bucket is alive, so we have exclusive access. let (ck, v) = unsafe { (*bucket.data.get()).assume_init_ref() }; if key.equivalent(ck) { @@ -343,10 +371,10 @@ where stats.record_hit(ck, v); } let v = v.clone(); - bucket.unlock(prev & !LOCKED_BIT); + bucket.unlock(prev); return Some(v); } - bucket.unlock(prev & !LOCKED_BIT); + bucket.unlock(prev); } #[cfg(feature = "stats")] if C::STATS @@ -368,7 +396,7 @@ where /// Returns the value if the key was present in the cache. pub fn remove>(&self, key: &Q) -> Option { let (bucket, tag) = self.calc(key); - if let Some(prev) = bucket.try_lock_ret(Some(tag)) { + if let Some(prev) = bucket.try_lock_ret(Some(tag), false) { // SAFETY: We hold the lock and bucket is alive, so we have exclusive access. let data = unsafe { &mut *bucket.data.get() }; let (ck, v) = unsafe { data.assume_init_ref() }; @@ -388,8 +416,7 @@ where bucket.unlock(new_version); return Some(v); } - let new_version = prev & VERSION_MASK; - bucket.unlock(tag | new_version); + bucket.unlock(prev); } None } @@ -416,23 +443,14 @@ where unsafe { core::ptr::write(ptr, f()) }; } - let existing = bucket.tag.load(Ordering::Relaxed); - if existing & LOCKED_BIT != 0 { - return; - } - let new_version = existing.wrapping_add(VERSION_INCREMENT) & VERSION_MASK; - let to_store = tag | new_version; - if bucket - .tag - .compare_exchange(existing, to_store | LOCKED_BIT, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { + let Some(locked) = bucket.try_lock_ret(None, true) else { return; - } + }; + let to_store = tag | (locked & VERSION_MASK); // SAFETY: We hold the lock, so we have exclusive access. unsafe { - let is_alive = existing & ALIVE_BIT != 0; + let is_alive = locked & ALIVE_BIT != 0; let data = bucket.data.get().cast::<(K, V)>(); if C::STATS && cfg!(feature = "stats") { @@ -600,7 +618,7 @@ impl Bucket { } #[inline] - fn try_lock_ret(&self, expected: Option) -> Option { + fn try_lock_ret(&self, expected: Option, bump_version: bool) -> Option { let state = self.tag.load(Ordering::Relaxed); if let Some(expected) = expected { if state & !VERSION_MASK != expected { @@ -609,9 +627,15 @@ impl Bucket { } else if state & LOCKED_BIT != 0 { return None; } + let mut locked = state | LOCKED_BIT; + if bump_version { + let new_version = state.wrapping_add(VERSION_INCREMENT) & VERSION_MASK; + locked = (locked & !VERSION_MASK) | new_version; + } self.tag - .compare_exchange(state, state | LOCKED_BIT, Ordering::Acquire, Ordering::Relaxed) + .compare_exchange(state, locked, Ordering::Acquire, Ordering::Relaxed) .ok() + .map(|prev| if bump_version { locked } else { prev }) } #[inline] From 7a291474deb2c5242bb13bfe7d91e210f4d856ea Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:10:46 +0100 Subject: [PATCH 10/16] name --- src/lib.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6f9973f..afbef3a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -161,7 +161,9 @@ where C: CacheConfig, { const NEEDS_DROP: bool = Bucket::<(K, V)>::NEEDS_DROP; - const ENTRY_IMPLS_COPY: bool = impls::impls!((K, V): Copy); + + // TODO: Not entirely correct. + const ENTRY_IMPLS_COPY: bool = !Self::NEEDS_DROP; /// Create a new cache with the specified number of entries and hasher. /// @@ -342,15 +344,15 @@ where // counter), no writer intervened and the data is consistent. This avoids acquiring // the lock entirely on cache hits, at the cost of occasionally discarding a read // if a concurrent write raced with us. - let tag2 = bucket.tag.load(Ordering::Acquire); - if (tag2 & LOCKED_BIT) == 0 && (tag2 & !VERSION_MASK) == tag { + let seqlock = bucket.tag.load(Ordering::Acquire); + if (seqlock & LOCKED_BIT) == 0 && (seqlock & !VERSION_MASK) == tag { let (ck, v) = unsafe { bucket.data.get().cast::<(K, V)>().read() }; if cfg!(any(target_arch = "x86_64", target_arch = "x86")) { std::sync::atomic::compiler_fence(Ordering::Acquire); } else { std::sync::atomic::fence(Ordering::Acquire); } - if tag2 == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { + if seqlock == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { #[cfg(feature = "stats")] if C::STATS && let Some(stats) = &self.stats From 1037ae7fc9ade3e86dab006feac9da2d4ecbaab8 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:37:59 +0100 Subject: [PATCH 11/16] cleanups --- Cargo.toml | 1 - src/lib.rs | 15 +++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1c6da3e..aedf051 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,6 @@ categories = ["caching", "concurrency", "data-structures"] [dependencies] equivalent = "1" -impls = "1" rapidhash = { version = "4", default-features = false, optional = true } typeid = { version = "1", default-features = false, optional = true } diff --git a/src/lib.rs b/src/lib.rs index afbef3a..f44b05a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -344,15 +344,18 @@ where // counter), no writer intervened and the data is consistent. This avoids acquiring // the lock entirely on cache hits, at the cost of occasionally discarding a read // if a concurrent write raced with us. - let seqlock = bucket.tag.load(Ordering::Acquire); - if (seqlock & LOCKED_BIT) == 0 && (seqlock & !VERSION_MASK) == tag { + + let seq1 = bucket.tag.load(Ordering::Acquire); + if (seq1 & LOCKED_BIT) == 0 && (seq1 & !VERSION_MASK) == tag { + // SAFETY: Speculative read. `(K, V): !Drop` (and ideally also `Copy`) let (ck, v) = unsafe { bucket.data.get().cast::<(K, V)>().read() }; - if cfg!(any(target_arch = "x86_64", target_arch = "x86")) { - std::sync::atomic::compiler_fence(Ordering::Acquire); - } else { + + // Skip fence on x86. Thanks to TSO, these loads are never reordered. + if !cfg!(any(target_arch = "x86_64", target_arch = "x86")) { std::sync::atomic::fence(Ordering::Acquire); } - if seqlock == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { + + if seq1 == bucket.tag.load(Ordering::Acquire) && key.equivalent(&ck) { #[cfg(feature = "stats")] if C::STATS && let Some(stats) = &self.stats From 53b26a632cdeca30dfd82a0a86062ed075e2b685 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:39:33 +0100 Subject: [PATCH 12/16] docs --- README.md | 3 ++- src/lib.rs | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 834f786..0d120d2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ with predictable memory usage and minimal overhead. ## Features - **Fixed size**: Memory is allocated once at creation time -- **Lock-free reads**: Uses atomic operations for thread-safe access without blocking +- **Lock-free reads**: For types that don't require drop, reads use a seqlock pattern + that never acquires a lock. Other types fall back to a CAS-based lock - **Zero dependencies** (optional `rapidhash` for faster hashing) - **`no_std` compatible** (with `alloc`) - **Static initialization**: Create caches at compile time with the `static_cache!` macro diff --git a/src/lib.rs b/src/lib.rs index f44b05a..bf6f6a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -99,6 +99,9 @@ impl CacheConfig for DefaultCacheConfig {} /// The cache is safe to share across threads (`Send + Sync`). All operations use atomic /// instructions and never block, making it suitable for high-contention scenarios. /// +/// For `Copy` entry types, reads use a seqlock pattern that avoids acquiring a lock entirely. This +/// makes cache hits completely lock-free. +/// /// # Limitations /// /// - **Eviction on collision**: When two keys hash to the same bucket, the older entry is evicted. From 79bc7b7528906412794b5f71e41a5a1ada865ac5 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:43:09 +0100 Subject: [PATCH 13/16] chore: skip versioning if not copy --- src/lib.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bf6f6a5..4da0d60 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -164,9 +164,7 @@ where C: CacheConfig, { const NEEDS_DROP: bool = Bucket::<(K, V)>::NEEDS_DROP; - - // TODO: Not entirely correct. - const ENTRY_IMPLS_COPY: bool = !Self::NEEDS_DROP; + const ENTRY_IMPLS_COPY: bool = Bucket::<(K, V)>::IMPLS_COPY; /// Create a new cache with the specified number of entries and hasher. /// @@ -420,7 +418,11 @@ where // SAFETY: We hold the lock, so we have exclusive access. unsafe { data.assume_init_drop() }; } - let new_version = prev.wrapping_add(VERSION_INCREMENT) & VERSION_MASK; + let new_version = if Self::ENTRY_IMPLS_COPY { + prev.wrapping_add(VERSION_INCREMENT) & VERSION_MASK + } else { + 0 + }; bucket.unlock(new_version); return Some(v); } @@ -619,6 +621,9 @@ pub struct Bucket { impl Bucket { const NEEDS_DROP: bool = std::mem::needs_drop::(); + // TODO: Not entirely correct. + const IMPLS_COPY: bool = !Self::NEEDS_DROP; + /// Creates a new zeroed bucket. #[inline] pub const fn new() -> Self { @@ -626,7 +631,11 @@ impl Bucket { } #[inline] - fn try_lock_ret(&self, expected: Option, bump_version: bool) -> Option { + fn try_lock_ret(&self, expected: Option, mut bump_version: bool) -> Option { + if !Self::IMPLS_COPY { + bump_version = false; + } + let state = self.tag.load(Ordering::Relaxed); if let Some(expected) = expected { if state & !VERSION_MASK != expected { From 9c1be54e6ed6096e02dbcee276ce479d33eb8848 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:48:20 +0100 Subject: [PATCH 14/16] Revert "chore: skip versioning if not copy" This reverts commit 79bc7b7528906412794b5f71e41a5a1ada865ac5. --- src/lib.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 4da0d60..bf6f6a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -164,7 +164,9 @@ where C: CacheConfig, { const NEEDS_DROP: bool = Bucket::<(K, V)>::NEEDS_DROP; - const ENTRY_IMPLS_COPY: bool = Bucket::<(K, V)>::IMPLS_COPY; + + // TODO: Not entirely correct. + const ENTRY_IMPLS_COPY: bool = !Self::NEEDS_DROP; /// Create a new cache with the specified number of entries and hasher. /// @@ -418,11 +420,7 @@ where // SAFETY: We hold the lock, so we have exclusive access. unsafe { data.assume_init_drop() }; } - let new_version = if Self::ENTRY_IMPLS_COPY { - prev.wrapping_add(VERSION_INCREMENT) & VERSION_MASK - } else { - 0 - }; + let new_version = prev.wrapping_add(VERSION_INCREMENT) & VERSION_MASK; bucket.unlock(new_version); return Some(v); } @@ -621,9 +619,6 @@ pub struct Bucket { impl Bucket { const NEEDS_DROP: bool = std::mem::needs_drop::(); - // TODO: Not entirely correct. - const IMPLS_COPY: bool = !Self::NEEDS_DROP; - /// Creates a new zeroed bucket. #[inline] pub const fn new() -> Self { @@ -631,11 +626,7 @@ impl Bucket { } #[inline] - fn try_lock_ret(&self, expected: Option, mut bump_version: bool) -> Option { - if !Self::IMPLS_COPY { - bump_version = false; - } - + fn try_lock_ret(&self, expected: Option, bump_version: bool) -> Option { let state = self.tag.load(Ordering::Relaxed); if let Some(expected) = expected { if state & !VERSION_MASK != expected { From 0dd1856eee864fc8ddab7704847a6f418de21e41 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 18:50:09 +0100 Subject: [PATCH 15/16] consts --- src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bf6f6a5..74699b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -164,9 +164,7 @@ where C: CacheConfig, { const NEEDS_DROP: bool = Bucket::<(K, V)>::NEEDS_DROP; - - // TODO: Not entirely correct. - const ENTRY_IMPLS_COPY: bool = !Self::NEEDS_DROP; + const ENTRY_IMPLS_COPY: bool = Bucket::<(K, V)>::IMPLS_COPY; /// Create a new cache with the specified number of entries and hasher. /// @@ -619,6 +617,9 @@ pub struct Bucket { impl Bucket { const NEEDS_DROP: bool = std::mem::needs_drop::(); + // TODO: Not entirely correct. + const IMPLS_COPY: bool = !Self::NEEDS_DROP; + /// Creates a new zeroed bucket. #[inline] pub const fn new() -> Self { From 141bf24023c94c7911fbc113a005d3ed645b9c76 Mon Sep 17 00:00:00 2001 From: DaniPopes <57450786+DaniPopes@users.noreply.github.com> Date: Sat, 7 Feb 2026 19:00:01 +0100 Subject: [PATCH 16/16] perf: cold path --- src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 74699b9..3d6f3ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -365,6 +365,8 @@ where } return Some(v); } + + cold_path(); } } else if let Some(prev) = bucket.try_lock_ret(Some(tag), false) { // SAFETY: We hold the lock and bucket is alive, so we have exclusive access. @@ -380,6 +382,7 @@ where bucket.unlock(prev); return Some(v); } + cold_path(); bucket.unlock(prev); } #[cfg(feature = "stats")] @@ -705,6 +708,10 @@ macro_rules! static_cache { }}; } +#[inline(always)] +#[cold] +const fn cold_path() {} + #[cfg(test)] mod tests { use super::*;