From a844c26fc064a4433935dbda6f253bbc8d0e777b Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla <105630300+fsdvh@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:39:10 +0200 Subject: [PATCH 01/12] Add non-blocking methods for sync cache (#1) Add non-blocking API for sync cache --- src/rw_lock.rs | 40 ++++++++ src/sync.rs | 253 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 293 insertions(+) diff --git a/src/rw_lock.rs b/src/rw_lock.rs index 278f5b7..cc8e81c 100644 --- a/src/rw_lock.rs +++ b/src/rw_lock.rs @@ -105,6 +105,46 @@ impl RwLock { }) } + /// Attempts to acquire this `RwLock` with shared read access without blocking. + /// + /// Returns `Some(guard)` if the lock was acquired, or `None` if it is already + /// held by a writer. + #[inline] + pub fn try_read(&self) -> Option> { + #[cfg(feature = "parking_lot")] + { + self.0.try_read().map(RwLockReadGuard) + } + #[cfg(not(feature = "parking_lot"))] + { + match self.0.try_read() { + Ok(guard) => Some(RwLockReadGuard(guard)), + Err(std::sync::TryLockError::WouldBlock) => None, + Err(std::sync::TryLockError::Poisoned(err)) => panic!("{}", err), + } + } + } + + /// Attempts to acquire this `RwLock` with exclusive write access without blocking. + /// + /// Returns `Some(guard)` if the lock was acquired, or `None` if it is already + /// held by any readers or a writer. + #[inline] + pub fn try_write(&self) -> Option> { + #[cfg(feature = "parking_lot")] + { + self.0.try_write().map(RwLockWriteGuard) + } + #[cfg(not(feature = "parking_lot"))] + { + match self.0.try_write() { + Ok(guard) => Some(RwLockWriteGuard(guard)), + Err(std::sync::TryLockError::WouldBlock) => None, + Err(std::sync::TryLockError::Poisoned(err)) => panic!("{}", err), + } + } + } + /// Locks this `RwLock` with exclusive write access, blocking the current /// thread until it can be acquired. /// diff --git a/src/sync.rs b/src/sync.rs index df77b40..e9f6f29 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -18,6 +18,29 @@ use crate::shard::EntryOrPlaceholder; pub use crate::sync_placeholder::{EntryAction, EntryResult, GuardResult, PlaceholderGuard}; use crate::sync_placeholder::{JoinFuture, JoinResult}; +/// The result of a non-blocking cache operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContendedResult { + /// The operation succeeded. For read operations the inner value holds the lookup result; + /// for write operations it holds the lifecycle request state (or `()` for [`Cache::try_insert`]). + Ok(Val), + /// The shard lock could not be acquired without blocking. The operation was not performed. + Contended, +} + +impl ContendedResult { + pub fn ok(self) -> Option { + match self { + ContendedResult::Ok(val) => Some(val), + ContendedResult::Contended => None, + } + } + + pub fn is_contended(&self) -> bool { + matches!(self, ContendedResult::Contended) + } +} + /// A concurrent cache /// /// The concurrent cache is internally composed of equally sized shards, each of which is independently @@ -247,6 +270,23 @@ impl< .is_some_and(|(shard, hash)| shard.read().contains(hash, key)) } + /// Attempts to check if a key exists in the cache without blocking. + /// Returns [`ContendedResult::Ok(true)`] if present, [`ContendedResult::Ok(false)`] if absent, + /// or [`ContendedResult::Contended`] if the shard lock could not be acquired without blocking. + pub fn try_contains_key(&self, key: &Q) -> ContendedResult + where + Q: Hash + Equivalent + ?Sized, + { + let Some((shard, hash)) = self.shard_for(key) else { + return ContendedResult::Ok(false); + }; + + shard + .try_read() + .map(|guard| ContendedResult::Ok(guard.contains(hash, key))) + .unwrap_or(ContendedResult::Contended) + } + /// Fetches an item from the cache whose key is `key`. pub fn get(&self, key: &Q) -> Option where @@ -256,6 +296,22 @@ impl< shard.read().get(hash, key).cloned() } + /// Attempts to fetch an item from the cache whose key is `key`. + /// Returns [`ContendedResult::Ok(Some(val))`] if the key is present, [`ContendedResult::Ok(None)`] if absent, + /// or [`ContendedResult::Contended`] if the shard lock could not be acquired without blocking. + pub fn try_get(&self, key: &Q) -> ContendedResult> + where + Q: Hash + Equivalent + ?Sized, + { + let Some((shard, hash)) = self.shard_for(key) else { + return ContendedResult::Ok(None); + }; + shard + .try_read() + .map(|guard| ContendedResult::Ok(guard.get(hash, key).cloned())) + .unwrap_or(ContendedResult::Contended) + } + /// Peeks an item from the cache whose key is `key`. /// Contrary to gets, peeks don't alter the key "hotness". pub fn peek(&self, key: &Q) -> Option @@ -266,6 +322,23 @@ impl< shard.read().peek(hash, key).cloned() } + /// Attempts to peek an item from the cache whose key is `key`. + /// Contrary to gets, peeks don't alter the key "hotness". + /// Returns [`ContendedResult::Ok(Some(val))`] if the key is present, [`ContendedResult::Ok(None)`] if absent, + /// or [`ContendedResult::Contended`] if the shard lock could not be acquired without blocking. + pub fn try_peek(&self, key: &Q) -> ContendedResult> + where + Q: Hash + Equivalent + ?Sized, + { + let Some((shard, hash)) = self.shard_for(key) else { + return ContendedResult::Ok(None); + }; + shard + .try_read() + .map(|guard| ContendedResult::Ok(guard.peek(hash, key).cloned())) + .unwrap_or(ContendedResult::Contended) + } + /// Remove an item from the cache whose key is `key`. /// Returns the removed entry, if any. pub fn remove(&self, key: &Q) -> Option<(Key, Val)> @@ -276,6 +349,23 @@ impl< shard.write().remove(hash, key) } + /// Attempts to remove an item from the cache whose key is `key`. + /// Returns [`ContendedResult::Ok(Some(entry))`] with the removed entry if present, [`ContendedResult::Ok(None)`] if absent, + /// or [`ContendedResult::Contended`] if the shard lock could not be acquired without blocking. + pub fn try_remove(&self, key: &Q) -> ContendedResult> + where + Q: Hash + Equivalent + ?Sized, + { + let Some((shard, hash)) = self.shard_for(key) else { + return ContendedResult::Ok(None); + }; + + shard + .try_write() + .map(|mut guard| ContendedResult::Ok(guard.remove(hash, key))) + .unwrap_or(ContendedResult::Contended) + } + /// Remove an item from the cache whose key is `key` if `f(&value)` returns `true` for that entry. /// Compared to peek and remove, this method guarantees that no new value was inserted in-between. /// @@ -337,6 +427,18 @@ impl< self.lifecycle.end_request(lcs); } + /// Attempts to insert an item in the cache with key `key`. + /// Returns [`ContendedResult::Ok`] if the item was inserted, or [`ContendedResult::Contended`] if the shard lock was contended. + pub fn try_insert(&self, key: Key, value: Val) -> ContendedResult<()> { + match self.try_insert_with_lifecycle(key, value) { + ContendedResult::Ok(lcs) => { + self.lifecycle.end_request(lcs); + ContendedResult::Ok(()) + } + ContendedResult::Contended => ContendedResult::Contended, + } + } + /// Inserts an item in the cache with key `key`. pub fn insert_with_lifecycle(&self, key: Key, value: Val) -> L::RequestState { let mut lcs = self.lifecycle.begin_request(); @@ -349,6 +451,28 @@ impl< lcs } + /// Attempts to insert an item in the cache with key `key`. + /// Returns [`ContendedResult::Ok`] with the lifecycle request state if the item was inserted, + /// or [`ContendedResult::Contended`] if the shard lock was contended. + pub fn try_insert_with_lifecycle( + &self, + key: Key, + value: Val, + ) -> ContendedResult { + let (shard, hash) = self.shard_for(&key).unwrap(); + + shard + .try_write() + .map(|mut guard| { + let mut lcs = self.lifecycle.begin_request(); + let result = guard.insert(&mut lcs, hash, key, value, InsertStrategy::Insert); + // result cannot err with the Insert strategy + debug_assert!(result.is_ok()); + ContendedResult::Ok(lcs) + }) + .unwrap_or(ContendedResult::Contended) + } + /// Clear all items from the cache pub fn clear(&self) { for s in self.shards.iter() { @@ -1499,4 +1623,133 @@ mod tests { } } } + + // --- Non-blocking method tests --- + + #[test] + fn test_contended_result_helpers() { + let ok: ContendedResult = ContendedResult::Ok(42); + assert!(!ok.is_contended()); + assert_eq!(ok.ok(), Some(42)); + + let contended: ContendedResult = ContendedResult::Contended; + assert!(contended.is_contended()); + assert_eq!(contended.ok(), None); + } + + #[test] + fn test_try_contains_key() { + let cache = Cache::new(100); + cache.insert(1, 10); + + assert_eq!(cache.try_contains_key(&1), ContendedResult::Ok(true)); + assert_eq!(cache.try_contains_key(&2), ContendedResult::Ok(false)); + } + + #[test] + fn test_try_contains_key_contended() { + let cache = Cache::new(100); + cache.insert(1, 10); + // Hold write locks on all shards so try_read is blocked. + let _guards: Vec<_> = cache.shards.iter().map(|s| s.write()).collect(); + assert_eq!(cache.try_contains_key(&1), ContendedResult::Contended); + } + + #[test] + fn test_try_get() { + let cache = Cache::new(100); + cache.insert(1, 10); + + assert_eq!(cache.try_get(&1), ContendedResult::Ok(Some(10))); + assert_eq!(cache.try_get(&2), ContendedResult::Ok(None)); + } + + #[test] + fn test_try_get_contended() { + let cache = Cache::new(100); + cache.insert(1, 10); + let _guards: Vec<_> = cache.shards.iter().map(|s| s.write()).collect(); + assert_eq!(cache.try_get(&1), ContendedResult::Contended); + } + + #[test] + fn test_try_peek() { + let cache = Cache::new(100); + cache.insert(1, 10); + + assert_eq!(cache.try_peek(&1), ContendedResult::Ok(Some(10))); + assert_eq!(cache.try_peek(&2), ContendedResult::Ok(None)); + } + + #[test] + fn test_try_peek_contended() { + let cache = Cache::new(100); + cache.insert(1, 10); + let _guards: Vec<_> = cache.shards.iter().map(|s| s.write()).collect(); + assert_eq!(cache.try_peek(&1), ContendedResult::Contended); + } + + #[test] + fn test_try_remove() { + let cache = Cache::new(100); + cache.insert(1, 10); + + assert_eq!(cache.try_remove(&1), ContendedResult::Ok(Some((1, 10)))); + assert_eq!(cache.try_remove(&1), ContendedResult::Ok(None)); + assert_eq!(cache.try_remove(&99), ContendedResult::Ok(None)); + } + + #[test] + fn test_try_remove_contended() { + let cache = Cache::new(100); + cache.insert(1, 10); + // Hold read locks on all shards so try_write is blocked. + let guards: Vec<_> = cache.shards.iter().map(|s| s.read()).collect(); + assert_eq!(cache.try_remove(&1), ContendedResult::Contended); + drop(guards); + // Item must still be present since the remove did not happen. + assert_eq!(cache.get(&1), Some(10)); + } + + #[test] + fn test_try_insert() { + let cache = Cache::new(100); + + assert_eq!(cache.try_insert(1, 10), ContendedResult::Ok(())); + assert_eq!(cache.get(&1), Some(10)); + + // Insert same key overwrites the previous value. + assert_eq!(cache.try_insert(1, 20), ContendedResult::Ok(())); + assert_eq!(cache.get(&1), Some(20)); + } + + #[test] + fn test_try_insert_contended() { + let cache = Cache::new(100); + let guards: Vec<_> = cache.shards.iter().map(|s| s.read()).collect(); + assert_eq!(cache.try_insert(1, 10), ContendedResult::Contended); + drop(guards); + assert_eq!(cache.get(&1), None); + } + + #[test] + fn test_try_insert_with_lifecycle() { + let cache = Cache::new(100); + + // Successful insert returns the lifecycle request state. + let result = cache.try_insert_with_lifecycle(1, 10); + assert!(!result.is_contended()); + let lcs = result.ok().unwrap(); + cache.lifecycle.end_request(lcs); + assert_eq!(cache.get(&1), Some(10)); + + // Contended when a read lock is held. + let guards: Vec<_> = cache.shards.iter().map(|s| s.read()).collect(); + assert_eq!( + cache.try_insert_with_lifecycle(2, 20), + ContendedResult::Contended + ); + drop(guards); + assert_eq!(cache.get(&2), None); + } } From 1b13f573e24793b5563b59ffc4be520679a9f40a Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla <105630300+fsdvh@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:40:23 +0200 Subject: [PATCH 02/12] Expose shard_index method (#2) Expose shard_index method --- src/sync.rs | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index e9f6f29..0057c64 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -231,6 +231,37 @@ impl< self.shards.iter().map(|s| s.read().hits()).sum() } + #[inline] + fn compute_shard_index(&self, hash: u64) -> u64 { + // Give preference to the bits in the middle of the hash. When choosing the + // shard, rotate the hash by usize::BITS / 2 so we avoid the lower bits and + // the highest 7 bits that hashbrown uses internally for probing, improving + // the real entropy available to each hashbrown shard. + hash.rotate_right(usize::BITS / 2) & self.shards_mask + } + + /// Returns the shard index for the given key. + /// + /// The returned index is guaranteed to be in `[0, num_shards())`. + /// + /// # Use cases + /// + /// - **Batching**: group keys by shard index before acquiring shard locks, so + /// each lock is taken only once per batch instead of once per key. + /// + /// # Notes + /// + /// The mapping from key to shard index depends on the [`BuildHasher`] supplied + /// at construction time. If two `Cache` instances are built with different + /// hashers, the same key may map to different shard indices. + /// + /// [`BuildHasher`]: std::hash::BuildHasher + #[inline] + pub fn shard_index + ?Sized>(&self, key: &Q) -> usize { + let hash = self.hash_builder.hash_one(key); + self.compute_shard_index(hash) as usize + } + #[inline] fn shard_for( &self, @@ -243,11 +274,7 @@ impl< Q: Hash + Equivalent + ?Sized, { let hash = self.hash_builder.hash_one(key); - // When choosing the shard, rotate the hash bits usize::BITS / 2 so that we - // give preference to the bits in the middle of the hash. - // Internally hashbrown uses the lower bits for start of probing + the 7 highest, - // so by picking something else we improve the real entropy available to each hashbrown shard. - let shard_idx = (hash.rotate_right(usize::BITS / 2) & self.shards_mask) as usize; + let shard_idx = self.compute_shard_index(hash) as usize; self.shards.get(shard_idx).map(|s| (s, hash)) } From c76a9db8eacbf8c76a6adbbcf512f43a81ef69e4 Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla <105630300+fsdvh@users.noreply.github.com> Date: Mon, 27 Apr 2026 18:39:00 +0200 Subject: [PATCH 03/12] Add wait-free methods (#3) * Add non-blocking methods * Do not leak lcs * Dedicated result * More * Simplify * Cleanup + some docs * Docs * Use a feature flag * Posion lock * Remove feature and more tests * Update src/sync.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Simplify signature * Simplify * clippy * More docs * Update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update src/sync.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * AI suggestions * Update src/sync.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 1 + src/sync.rs | 188 +++++++++++++++++++++++----------------------------- 2 files changed, 84 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 8c986e8..046d03c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Lightweight and high performance concurrent cache optimized for low cache overhe * Scales well with the number of threads * Atomic operations with `get_or_insert` and `get_value_or_guard` functions * Atomic async operations with `get_or_insert_async` and `get_value_or_guard_async` functions +* Non-blocking `try_get`, `try_insert`, `try_remove`, and related methods that return an error instead of blocking: typically `Err(LockContention)`, or `Err((Key, Val))` for `try_insert`/`try_insert_with_lifecycle` so inputs are preserved * Closure-based `entry` API for atomic inspect-and-act patterns (keep, remove, replace) * Supports item pinning * Iteration and draining diff --git a/src/sync.rs b/src/sync.rs index 0057c64..7f6f5f3 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -18,29 +18,22 @@ use crate::shard::EntryOrPlaceholder; pub use crate::sync_placeholder::{EntryAction, EntryResult, GuardResult, PlaceholderGuard}; use crate::sync_placeholder::{JoinFuture, JoinResult}; -/// The result of a non-blocking cache operation. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ContendedResult { - /// The operation succeeded. For read operations the inner value holds the lookup result; - /// for write operations it holds the lifecycle request state (or `()` for [`Cache::try_insert`]). - Ok(Val), - /// The shard lock could not be acquired without blocking. The operation was not performed. - Contended, -} - -impl ContendedResult { - pub fn ok(self) -> Option { - match self { - ContendedResult::Ok(val) => Some(val), - ContendedResult::Contended => None, - } - } +/// Error returned by non-blocking cache operations that do not consume their +/// inputs when the relevant shard lock could not be acquired immediately. +/// +/// This is used by borrowed-key/read-path operations. Non-blocking operations +/// that consume owned inputs may instead return those inputs on contention. +#[derive(Debug)] +pub struct LockContention; - pub fn is_contended(&self) -> bool { - matches!(self, ContendedResult::Contended) +impl std::fmt::Display for LockContention { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Lock Contention") } } +impl std::error::Error for LockContention {} + /// A concurrent cache /// /// The concurrent cache is internally composed of equally sized shards, each of which is independently @@ -298,20 +291,20 @@ impl< } /// Attempts to check if a key exists in the cache without blocking. - /// Returns [`ContendedResult::Ok(true)`] if present, [`ContendedResult::Ok(false)`] if absent, - /// or [`ContendedResult::Contended`] if the shard lock could not be acquired without blocking. - pub fn try_contains_key(&self, key: &Q) -> ContendedResult + /// Returns `Ok(true)` if present, `Ok(false)` if absent, + /// or `Err(LockContention)` if the shard lock could not be acquired without blocking. + pub fn try_contains_key(&self, key: &Q) -> Result where Q: Hash + Equivalent + ?Sized, { let Some((shard, hash)) = self.shard_for(key) else { - return ContendedResult::Ok(false); + return Ok(false); }; - shard - .try_read() - .map(|guard| ContendedResult::Ok(guard.contains(hash, key))) - .unwrap_or(ContendedResult::Contended) + match shard.try_read() { + Some(guard) => Ok(guard.contains(hash, key)), + None => Err(LockContention), + } } /// Fetches an item from the cache whose key is `key`. @@ -324,19 +317,20 @@ impl< } /// Attempts to fetch an item from the cache whose key is `key`. - /// Returns [`ContendedResult::Ok(Some(val))`] if the key is present, [`ContendedResult::Ok(None)`] if absent, - /// or [`ContendedResult::Contended`] if the shard lock could not be acquired without blocking. - pub fn try_get(&self, key: &Q) -> ContendedResult> + /// Returns `Ok(Some(val))` if the key is present, `Ok(None)` if absent, + /// or `Err(LockContention)` if the shard lock could not be acquired without blocking. + pub fn try_get(&self, key: &Q) -> Result, LockContention> where Q: Hash + Equivalent + ?Sized, { let Some((shard, hash)) = self.shard_for(key) else { - return ContendedResult::Ok(None); + return Ok(None); }; - shard - .try_read() - .map(|guard| ContendedResult::Ok(guard.get(hash, key).cloned())) - .unwrap_or(ContendedResult::Contended) + + match shard.try_read() { + Some(guard) => Ok(guard.get(hash, key).cloned()), + None => Err(LockContention), + } } /// Peeks an item from the cache whose key is `key`. @@ -351,19 +345,19 @@ impl< /// Attempts to peek an item from the cache whose key is `key`. /// Contrary to gets, peeks don't alter the key "hotness". - /// Returns [`ContendedResult::Ok(Some(val))`] if the key is present, [`ContendedResult::Ok(None)`] if absent, - /// or [`ContendedResult::Contended`] if the shard lock could not be acquired without blocking. - pub fn try_peek(&self, key: &Q) -> ContendedResult> + /// Returns `Ok(Some(val))` if the key is present, `Ok(None)` if absent, + /// or `Err(LockContention)` if the shard lock could not be acquired without blocking. + pub fn try_peek(&self, key: &Q) -> Result, LockContention> where Q: Hash + Equivalent + ?Sized, { let Some((shard, hash)) = self.shard_for(key) else { - return ContendedResult::Ok(None); + return Ok(None); }; - shard - .try_read() - .map(|guard| ContendedResult::Ok(guard.peek(hash, key).cloned())) - .unwrap_or(ContendedResult::Contended) + match shard.try_read() { + Some(guard) => Ok(guard.peek(hash, key).cloned()), + None => Err(LockContention), + } } /// Remove an item from the cache whose key is `key`. @@ -377,20 +371,20 @@ impl< } /// Attempts to remove an item from the cache whose key is `key`. - /// Returns [`ContendedResult::Ok(Some(entry))`] with the removed entry if present, [`ContendedResult::Ok(None)`] if absent, - /// or [`ContendedResult::Contended`] if the shard lock could not be acquired without blocking. - pub fn try_remove(&self, key: &Q) -> ContendedResult> + /// Returns `Ok(Some(entry))` with the removed entry if present, `Ok(None)` if absent, + /// or `Err(LockContention)` if the shard lock could not be acquired without blocking. + pub fn try_remove(&self, key: &Q) -> Result, LockContention> where Q: Hash + Equivalent + ?Sized, { let Some((shard, hash)) = self.shard_for(key) else { - return ContendedResult::Ok(None); + return Ok(None); }; - shard - .try_write() - .map(|mut guard| ContendedResult::Ok(guard.remove(hash, key))) - .unwrap_or(ContendedResult::Contended) + match shard.try_write() { + Some(mut guard) => Ok(guard.remove(hash, key)), + None => Err(LockContention), + } } /// Remove an item from the cache whose key is `key` if `f(&value)` returns `true` for that entry. @@ -454,16 +448,13 @@ impl< self.lifecycle.end_request(lcs); } - /// Attempts to insert an item in the cache with key `key`. - /// Returns [`ContendedResult::Ok`] if the item was inserted, or [`ContendedResult::Contended`] if the shard lock was contended. - pub fn try_insert(&self, key: Key, value: Val) -> ContendedResult<()> { - match self.try_insert_with_lifecycle(key, value) { - ContendedResult::Ok(lcs) => { - self.lifecycle.end_request(lcs); - ContendedResult::Ok(()) - } - ContendedResult::Contended => ContendedResult::Contended, - } + /// Attempts to insert an item in the cache with key `key` without blocking. + /// Returns `Ok(())` if the item was inserted, or `Err((key, value))` if the shard lock + /// could not be acquired without blocking. + pub fn try_insert(&self, key: Key, value: Val) -> Result<(), (Key, Val)> { + let lcs = self.try_insert_with_lifecycle(key, value)?; + self.lifecycle.end_request(lcs); + Ok(()) } /// Inserts an item in the cache with key `key`. @@ -478,26 +469,26 @@ impl< lcs } - /// Attempts to insert an item in the cache with key `key`. - /// Returns [`ContendedResult::Ok`] with the lifecycle request state if the item was inserted, - /// or [`ContendedResult::Contended`] if the shard lock was contended. + /// Attempts to insert an item in the cache with key `key` without blocking. + /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, + /// or `Err((key, value))` if the shard lock could not be acquired without blocking. pub fn try_insert_with_lifecycle( &self, key: Key, value: Val, - ) -> ContendedResult { + ) -> Result { let (shard, hash) = self.shard_for(&key).unwrap(); - shard - .try_write() - .map(|mut guard| { + match shard.try_write() { + Some(mut shard) => { let mut lcs = self.lifecycle.begin_request(); - let result = guard.insert(&mut lcs, hash, key, value, InsertStrategy::Insert); + let result = shard.insert(&mut lcs, hash, key, value, InsertStrategy::Insert); // result cannot err with the Insert strategy debug_assert!(result.is_ok()); - ContendedResult::Ok(lcs) - }) - .unwrap_or(ContendedResult::Contended) + Ok(lcs) + } + _ => Err((key, value)), + } } /// Clear all items from the cache @@ -1652,25 +1643,13 @@ mod tests { } // --- Non-blocking method tests --- - - #[test] - fn test_contended_result_helpers() { - let ok: ContendedResult = ContendedResult::Ok(42); - assert!(!ok.is_contended()); - assert_eq!(ok.ok(), Some(42)); - - let contended: ContendedResult = ContendedResult::Contended; - assert!(contended.is_contended()); - assert_eq!(contended.ok(), None); - } - #[test] fn test_try_contains_key() { let cache = Cache::new(100); cache.insert(1, 10); - assert_eq!(cache.try_contains_key(&1), ContendedResult::Ok(true)); - assert_eq!(cache.try_contains_key(&2), ContendedResult::Ok(false)); + assert!(cache.try_contains_key(&1).is_ok_and(|v| v)); + assert!(cache.try_contains_key(&2).is_ok_and(|v| !v)); } #[test] @@ -1679,7 +1658,7 @@ mod tests { cache.insert(1, 10); // Hold write locks on all shards so try_read is blocked. let _guards: Vec<_> = cache.shards.iter().map(|s| s.write()).collect(); - assert_eq!(cache.try_contains_key(&1), ContendedResult::Contended); + assert!(cache.try_contains_key(&1).is_err()); } #[test] @@ -1687,8 +1666,8 @@ mod tests { let cache = Cache::new(100); cache.insert(1, 10); - assert_eq!(cache.try_get(&1), ContendedResult::Ok(Some(10))); - assert_eq!(cache.try_get(&2), ContendedResult::Ok(None)); + assert!(cache.try_get(&1).is_ok_and(|v| matches!(v, Some(10)))); + assert!(cache.try_get(&2).is_ok_and(|v| v.is_none())); } #[test] @@ -1696,7 +1675,7 @@ mod tests { let cache = Cache::new(100); cache.insert(1, 10); let _guards: Vec<_> = cache.shards.iter().map(|s| s.write()).collect(); - assert_eq!(cache.try_get(&1), ContendedResult::Contended); + assert!(cache.try_get(&1).is_err()); } #[test] @@ -1704,8 +1683,8 @@ mod tests { let cache = Cache::new(100); cache.insert(1, 10); - assert_eq!(cache.try_peek(&1), ContendedResult::Ok(Some(10))); - assert_eq!(cache.try_peek(&2), ContendedResult::Ok(None)); + assert!(cache.try_peek(&1).is_ok_and(|v| matches!(v, Some(10)))); + assert!(cache.try_peek(&2).is_ok_and(|v| v.is_none())); } #[test] @@ -1713,7 +1692,7 @@ mod tests { let cache = Cache::new(100); cache.insert(1, 10); let _guards: Vec<_> = cache.shards.iter().map(|s| s.write()).collect(); - assert_eq!(cache.try_peek(&1), ContendedResult::Contended); + assert!(cache.try_peek(&1).is_err()); } #[test] @@ -1721,9 +1700,11 @@ mod tests { let cache = Cache::new(100); cache.insert(1, 10); - assert_eq!(cache.try_remove(&1), ContendedResult::Ok(Some((1, 10)))); - assert_eq!(cache.try_remove(&1), ContendedResult::Ok(None)); - assert_eq!(cache.try_remove(&99), ContendedResult::Ok(None)); + assert!(cache + .try_remove(&1) + .is_ok_and(|v| matches!(v, Some((1, 10))))); + assert!(cache.try_remove(&1).is_ok_and(|v| v.is_none())); + assert!(cache.try_remove(&99).is_ok_and(|v| v.is_none())); } #[test] @@ -1732,7 +1713,7 @@ mod tests { cache.insert(1, 10); // Hold read locks on all shards so try_write is blocked. let guards: Vec<_> = cache.shards.iter().map(|s| s.read()).collect(); - assert_eq!(cache.try_remove(&1), ContendedResult::Contended); + assert!(cache.try_remove(&1).is_err()); drop(guards); // Item must still be present since the remove did not happen. assert_eq!(cache.get(&1), Some(10)); @@ -1742,11 +1723,11 @@ mod tests { fn test_try_insert() { let cache = Cache::new(100); - assert_eq!(cache.try_insert(1, 10), ContendedResult::Ok(())); + assert_eq!(cache.try_insert(1, 10), Ok(())); assert_eq!(cache.get(&1), Some(10)); // Insert same key overwrites the previous value. - assert_eq!(cache.try_insert(1, 20), ContendedResult::Ok(())); + assert_eq!(cache.try_insert(1, 20), Ok(())); assert_eq!(cache.get(&1), Some(20)); } @@ -1754,7 +1735,7 @@ mod tests { fn test_try_insert_contended() { let cache = Cache::new(100); let guards: Vec<_> = cache.shards.iter().map(|s| s.read()).collect(); - assert_eq!(cache.try_insert(1, 10), ContendedResult::Contended); + assert_eq!(cache.try_insert(1, 10), Err((1, 10))); drop(guards); assert_eq!(cache.get(&1), None); } @@ -1765,17 +1746,14 @@ mod tests { // Successful insert returns the lifecycle request state. let result = cache.try_insert_with_lifecycle(1, 10); - assert!(!result.is_contended()); + assert!(result.is_ok()); let lcs = result.ok().unwrap(); cache.lifecycle.end_request(lcs); assert_eq!(cache.get(&1), Some(10)); // Contended when a read lock is held. let guards: Vec<_> = cache.shards.iter().map(|s| s.read()).collect(); - assert_eq!( - cache.try_insert_with_lifecycle(2, 20), - ContendedResult::Contended - ); + assert_eq!(cache.try_insert_with_lifecycle(2, 20), Err((2, 20))); drop(guards); assert_eq!(cache.get(&2), None); } From c370c0462cf6142d96060331b3b74e05f252b7ec Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla <105630300+fsdvh@users.noreply.github.com> Date: Wed, 29 Apr 2026 10:54:12 +0200 Subject: [PATCH 04/12] Move request initialisation outside of the lock (#4) --- src/sync.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 7f6f5f3..55b1580 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -22,8 +22,9 @@ use crate::sync_placeholder::{JoinFuture, JoinResult}; /// inputs when the relevant shard lock could not be acquired immediately. /// /// This is used by borrowed-key/read-path operations. Non-blocking operations -/// that consume owned inputs may instead return those inputs on contention. -#[derive(Debug)] +/// that consume owned inputs (e.g. `try_insert`) instead return those inputs +/// on contention so the caller can retry or discard without losing data. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct LockContention; impl std::fmt::Display for LockContention { @@ -450,7 +451,8 @@ impl< /// Attempts to insert an item in the cache with key `key` without blocking. /// Returns `Ok(())` if the item was inserted, or `Err((key, value))` if the shard lock - /// could not be acquired without blocking. + /// could not be acquired without blocking. Lock contention is the only failure + /// mode: the inputs are returned so the caller can retry or discard them. pub fn try_insert(&self, key: Key, value: Val) -> Result<(), (Key, Val)> { let lcs = self.try_insert_with_lifecycle(key, value)?; self.lifecycle.end_request(lcs); @@ -472,16 +474,20 @@ impl< /// Attempts to insert an item in the cache with key `key` without blocking. /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, /// or `Err((key, value))` if the shard lock could not be acquired without blocking. + /// Lock contention is the only failure mode: the inputs are returned so the + /// caller can retry or discard them. pub fn try_insert_with_lifecycle( &self, key: Key, value: Val, ) -> Result { + // Tradeoff: begin_request is called before acquiring the shard lock to avoid holding + // the lock during potentially expensive lifecycle initialization. + let mut lcs = self.lifecycle.begin_request(); let (shard, hash) = self.shard_for(&key).unwrap(); match shard.try_write() { Some(mut shard) => { - let mut lcs = self.lifecycle.begin_request(); let result = shard.insert(&mut lcs, hash, key, value, InsertStrategy::Insert); // result cannot err with the Insert strategy debug_assert!(result.is_ok()); From 48869188f6a24e75b84a0cee8e290f4e6bef3fa2 Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla <105630300+fsdvh@users.noreply.github.com> Date: Thu, 30 Apr 2026 10:47:35 +0200 Subject: [PATCH 05/12] Dedicated method with state as an input (#5) * Dedicated method with state as an input * Shared state methods * More docs --- src/sync.rs | 104 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 19 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 55b1580..3e0aa34 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -76,6 +76,15 @@ impl Cache { } impl + Clone> Cache { + /// Creates a new cache with a custom [`Weighter`]. + /// + /// - `estimated_items_capacity` — expected number of items the cache will hold, + /// roughly `weight_capacity / average_item_weight`. + /// - `weight_capacity` — total weight the cache may hold across all shards. + /// - `weighter` — determines the weight of each key–value pair. + /// + /// Use [`Cache::new`] when each item has unit weight (i.e. when you only care + /// about item count, not size). pub fn with_weighter( estimated_items_capacity: usize, weight_capacity: u64, @@ -412,11 +421,19 @@ impl< Ok(()) } - /// Inserts an item in the cache, but _only_ if an entry with key `key` already exists. + /// Inserts an item in the cache, but _only_ if an entry with key `key` already exists, + /// returning the lifecycle request state. + /// /// If `soft` is set, the replace operation won't affect the "hotness" of the entry, /// even if the value is replaced. /// - /// Returns `Ok` if the entry was admitted and `Err(_)` if it wasn't. + /// Returns `Ok(lcs)` with the lifecycle request state if the entry was replaced. + /// Returns `Err((key, value))` if no entry existed for `key` (inputs are returned + /// so the caller can retry or discard them). The caller is responsible for passing + /// the returned state to [`Lifecycle::end_request`]. + /// + /// Prefer [`replace`](Self::replace) unless you need manual control over lifecycle + /// request lifetime. pub fn replace_with_lifecycle( &self, key: Key, @@ -449,6 +466,38 @@ impl< self.lifecycle.end_request(lcs); } + /// Inserts an item in the cache with key `key`, returning the lifecycle request state. + /// + /// Unlike [`insert`](Self::insert), the lifecycle request is **not** ended automatically. + /// The caller is responsible for passing the returned state to [`Lifecycle::end_request`] + /// when it is ready to drop any evicted items outside the shard lock. + /// + /// This is useful when coalescing multiple insertions into a single lifecycle request, + /// or when you need to control exactly when evicted items are dropped. + pub fn insert_with_lifecycle(&self, key: Key, value: Val) -> L::RequestState { + let mut lcs = self.lifecycle.begin_request(); + self.insert_with_state(key, value, &mut lcs); + lcs + } + + /// Inserts an item in the cache with key `key` using an existing lifecycle request state. + /// + /// `lcs` must have been obtained from a prior [`Lifecycle::begin_request`] call and + /// must **not** have been passed to [`Lifecycle::end_request`] yet. Any items evicted + /// by this insert are recorded into `lcs` and will be dropped when `end_request` is + /// eventually called. + /// + /// Prefer [`insert`](Self::insert) for the common case where you do not need to + /// manage the lifecycle state manually. + pub fn insert_with_state(&self, key: Key, value: Val, lcs: &mut L::RequestState) { + let (shard, hash) = self.shard_for(&key).unwrap(); + let result = shard + .write() + .insert(lcs, hash, key, value, InsertStrategy::Insert); + // result cannot err with the Insert strategy + debug_assert!(result.is_ok()); + } + /// Attempts to insert an item in the cache with key `key` without blocking. /// Returns `Ok(())` if the item was inserted, or `Err((key, value))` if the shard lock /// could not be acquired without blocking. Lock contention is the only failure @@ -459,18 +508,6 @@ impl< Ok(()) } - /// Inserts an item in the cache with key `key`. - pub fn insert_with_lifecycle(&self, key: Key, value: Val) -> L::RequestState { - let mut lcs = self.lifecycle.begin_request(); - let (shard, hash) = self.shard_for(&key).unwrap(); - let result = shard - .write() - .insert(&mut lcs, hash, key, value, InsertStrategy::Insert); - // result cannot err with the Insert strategy - debug_assert!(result.is_ok()); - lcs - } - /// Attempts to insert an item in the cache with key `key` without blocking. /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, /// or `Err((key, value))` if the shard lock could not be acquired without blocking. @@ -484,14 +521,29 @@ impl< // Tradeoff: begin_request is called before acquiring the shard lock to avoid holding // the lock during potentially expensive lifecycle initialization. let mut lcs = self.lifecycle.begin_request(); + self.try_insert_with_state(key, value, &mut lcs)?; + Ok(lcs) + } + + /// Attempts to insert an item in the cache with key `key` without blocking. + /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, + /// or `Err((key, value))` if the shard lock could not be acquired without blocking. + /// Lock contention is the only failure mode: the inputs are returned so the + /// caller can retry or discard them. + pub fn try_insert_with_state( + &self, + key: Key, + value: Val, + lcs: &mut L::RequestState, + ) -> Result<(), (Key, Val)> { let (shard, hash) = self.shard_for(&key).unwrap(); match shard.try_write() { Some(mut shard) => { - let result = shard.insert(&mut lcs, hash, key, value, InsertStrategy::Insert); + let result = shard.insert(lcs, hash, key, value, InsertStrategy::Insert); // result cannot err with the Insert strategy debug_assert!(result.is_ok()); - Ok(lcs) + Ok(()) } _ => Err((key, value)), } @@ -577,9 +629,18 @@ impl< PlaceholderGuard::join(&self.lifecycle, shard, hash, key, timeout) } - /// Gets or inserts an item in the cache with key `key`. + /// Gets an item from the cache with key `key`, or inserts one produced by `with`. + /// + /// If the key is already present, the cached value is returned without calling `with`. + /// Otherwise, the cache is locked for this key (other callers to `get_value_or_guard` + /// or the `get_or_insert` family will block until the value is populated), `with` is + /// called to produce the value, and the result is inserted and returned. /// - /// See also `get_value_or_guard` and `get_value_or_guard_async`. + /// `with` may return an error, in which case nothing is inserted and the error is + /// propagated. The placeholder is dropped so waiting callers can retry. + /// + /// See also [`get_value_or_guard`](Self::get_value_or_guard) for more control over + /// the placeholder lifecycle. pub fn get_or_insert_with( &self, key: &Q, @@ -630,7 +691,12 @@ impl< } } - /// Gets or inserts an item in the cache with key `key`. + /// Gets an item from the cache with key `key`, or inserts one produced by `with`. + /// + /// Async counterpart of [`get_or_insert_with`](Self::get_or_insert_with). The `with` + /// future is only polled when no value exists for `key`. While `with` is being awaited, + /// other tasks looking up the same key will suspend and resume once the value is + /// inserted (or the guard is dropped without inserting). pub async fn get_or_insert_async( &self, key: &Q, From 54dacff7944109b99c293f1691bd591cb26a51e3 Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla Date: Wed, 24 Jun 2026 13:07:03 +0200 Subject: [PATCH 06/12] Add per item access count --- src/lib.rs | 16 +++++++++++++++ src/shard.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/sync.rs | 34 +++++++++++++++++++++++++++++++ src/unsync.rs | 10 +++++++++ 4 files changed, 116 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 7840608..09b8265 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -225,6 +225,22 @@ impl MemoryUsed { } } +/// Per-item statistics returned by `item_stats`. +/// +/// Only available with the `stats` feature enabled. +#[cfg(feature = "stats")] +#[non_exhaustive] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct ItemStats { + /// Number of times the item has been accessed (read) since it became resident. + /// + /// Incremented on every cache hit (`get`/`get_mut`/`get_value_or_guard`/`entry`). + /// Unlike the internal eviction counter, this is monotonic per residency and is + /// not bounded by the eviction policy. It resets to zero if the slot is reused for + /// a new value (e.g. after eviction and re-insertion). + pub access_count: u64, +} + #[cfg(test)] mod tests { use std::{ diff --git a/src/shard.rs b/src/shard.rs index ec40b94..92be278 100644 --- a/src/shard.rs +++ b/src/shard.rs @@ -79,6 +79,11 @@ pub struct Resident { value: Val, state: ResidentState, referenced: AtomicU16, + /// Number of times this item has been accessed (read) since it became resident. + /// Incremented wherever a cache hit is recorded, unlike `referenced` which is + /// bounded by the eviction policy. Reset whenever the slot is reused for a new key. + #[cfg(feature = "stats")] + access_count: AtomicU64, } impl Clone for Resident { @@ -89,6 +94,8 @@ impl Clone for Resident { value: self.value.clone(), state: self.state, referenced: self.referenced.load(atomic::Ordering::Relaxed).into(), + #[cfg(feature = "stats")] + access_count: self.access_count.load(atomic::Ordering::Relaxed).into(), } } } @@ -195,6 +202,20 @@ macro_rules! record_miss_mut { *$self.misses.get_mut() += 1; }}; } +#[cfg(feature = "stats")] +macro_rules! record_item_hit { + ($resident: expr) => {{ + $resident + .access_count + .fetch_add(1, atomic::Ordering::Relaxed); + }}; +} +#[cfg(feature = "stats")] +macro_rules! record_item_hit_mut { + ($resident: expr) => {{ + *$resident.access_count.get_mut() += 1; + }}; +} #[cfg(not(feature = "stats"))] macro_rules! record_hit { @@ -212,6 +233,14 @@ macro_rules! record_miss { macro_rules! record_miss_mut { ($self: expr) => {{}}; } +#[cfg(not(feature = "stats"))] +macro_rules! record_item_hit { + ($resident: expr) => {{}}; +} +#[cfg(not(feature = "stats"))] +macro_rules! record_item_hit_mut { + ($resident: expr) => {{}}; +} impl CacheShard { pub fn remove_placeholder(&mut self, placeholder: &Plh) { @@ -563,6 +592,7 @@ impl< resident.referenced.fetch_add(1, atomic::Ordering::Relaxed); } record_hit!(self); + record_item_hit!(resident); Some((&resident.key, &resident.value)) } else { record_miss!(self); @@ -594,6 +624,7 @@ impl< *resident.referenced.get_mut() += 1; } record_hit_mut!(self); + record_item_hit_mut!(resident); let old_weight = self.weighter.weight(&resident.key, &resident.value); Some(RefMut { @@ -640,6 +671,19 @@ impl< Some(&resident.value) } + /// Returns per-item statistics for a resident key without affecting its hotness + /// or access count. Returns `None` if the key is not resident. + #[cfg(feature = "stats")] + pub fn item_stats(&self, hash: u64, key: &Q) -> Option + where + Q: Hash + Equivalent + ?Sized, + { + let (_, resident) = self.search_resident(hash, key)?; + Some(crate::ItemStats { + access_count: resident.access_count.load(atomic::Ordering::Relaxed), + }) + } + pub fn peek_mut(&mut self, hash: u64, key: &Q) -> Option> where Q: Hash + Equivalent + ?Sized, @@ -900,6 +944,8 @@ impl< value, state: enter_state, referenced: referenced.into(), + #[cfg(feature = "stats")] + access_count: Default::default(), }), ); match evicted { @@ -1020,6 +1066,8 @@ impl< value, state: placeholder_hot, referenced: (referenced as u16).into(), + #[cfg(feature = "stats")] + access_count: Default::default(), }); let list_head = if placeholder_hot == ResidentState::Hot { @@ -1102,6 +1150,8 @@ impl< value, state, referenced: Default::default(), + #[cfg(feature = "stats")] + access_count: Default::default(), })); if weight != 0 { *list_head = Some(self.entries.link(idx, *list_head)); @@ -1147,6 +1197,7 @@ impl< *resident.referenced.get_mut() += 1; } record_hit_mut!(self); + record_item_hit_mut!(resident); unsafe { // Rustc gets insanely confused returning references from mut borrows // Safety: value will have the same lifetime as `resident` @@ -1205,6 +1256,7 @@ impl< if *resident.referenced.get_mut() < MAX_F { *resident.referenced.get_mut() += 1; } + record_item_hit_mut!(resident); EntryOrPlaceholder::Kept(t) } EntryAction::Remove => { @@ -1404,8 +1456,12 @@ impl, B, L, Plh: SharedPlaceholder> #[cfg(test)] mod tests { + #[cfg(not(feature = "stats"))] use super::*; + // The tight entry overhead is only guaranteed without the `stats` feature, + // which adds a per-item `access_count` field to each `Resident`. + #[cfg(not(feature = "stats"))] #[test] fn entry_overhead() { use std::mem::size_of; diff --git a/src/sync.rs b/src/sync.rs index 3e0aa34..ab426ad 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -370,6 +370,17 @@ impl< } } + /// Returns per-item statistics for `key`, or `None` if the key is not present. + /// Like peeks, this does not alter the key "hotness" or its access count. + #[cfg(feature = "stats")] + pub fn item_stats(&self, key: &Q) -> Option + where + Q: Hash + Equivalent + ?Sized, + { + let (shard, hash) = self.shard_for(key)?; + shard.read().item_stats(hash, key) + } + /// Remove an item from the cache whose key is `key`. /// Returns the removed entry, if any. pub fn remove(&self, key: &Q) -> Option<(Key, Val)> @@ -1767,6 +1778,29 @@ mod tests { assert!(cache.try_peek(&1).is_err()); } + #[cfg(feature = "stats")] + #[test] + fn test_item_stats() { + let cache = Cache::new(100); + // Missing key has no stats. + assert!(cache.item_stats(&1).is_none()); + + cache.insert(1, 10); + // Insert alone is not a hit. + assert_eq!(cache.item_stats(&1).map(|s| s.access_count), Some(0)); + + // Each get increments the per-item access count. + cache.get(&1); + cache.get(&1); + cache.get(&1); + assert_eq!(cache.item_stats(&1).map(|s| s.access_count), Some(3)); + + // Peeking (including item_stats itself) does not alter the count. + cache.peek(&1); + let _ = cache.item_stats(&1); + assert_eq!(cache.item_stats(&1).map(|s| s.access_count), Some(3)); + } + #[test] fn test_try_remove() { let cache = Cache::new(100); diff --git a/src/unsync.rs b/src/unsync.rs index 66ee954..a187105 100644 --- a/src/unsync.rs +++ b/src/unsync.rs @@ -175,6 +175,16 @@ impl, B: BuildHasher, L: Lifecycle(&self, key: &Q) -> Option + where + Q: Hash + Equivalent + ?Sized, + { + self.shard.item_stats(self.shard.hash(key), key) + } + /// Peeks an item from the cache. Contrary to gets, peeks don't alter the key "hotness". /// /// Note: Leaking the returned RefMut might cause cache weight tracking to be inaccurate. From 6555ffba87acb2385d261bfebc6d42e7b0002d5f Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla Date: Thu, 25 Jun 2026 15:03:29 +0200 Subject: [PATCH 07/12] cleanup --- src/sync.rs | 100 +++++++--------------------------------------------- 1 file changed, 12 insertions(+), 88 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 624b517..118b9ca 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -76,15 +76,6 @@ impl Cache { } impl + Clone> Cache { - /// Creates a new cache with a custom [`Weighter`]. - /// - /// - `estimated_items_capacity` — expected number of items the cache will hold, - /// roughly `weight_capacity / average_item_weight`. - /// - `weight_capacity` — total weight the cache may hold across all shards. - /// - `weighter` — determines the weight of each key–value pair. - /// - /// Use [`Cache::new`] when each item has unit weight (i.e. when you only care - /// about item count, not size). pub fn with_weighter( estimated_items_capacity: usize, weight_capacity: u64, @@ -432,19 +423,11 @@ impl< Ok(()) } - /// Inserts an item in the cache, but _only_ if an entry with key `key` already exists, - /// returning the lifecycle request state. - /// + /// Inserts an item in the cache, but _only_ if an entry with key `key` already exists. /// If `soft` is set, the replace operation won't affect the "hotness" of the entry, /// even if the value is replaced. /// - /// Returns `Ok(lcs)` with the lifecycle request state if the entry was replaced. - /// Returns `Err((key, value))` if no entry existed for `key` (inputs are returned - /// so the caller can retry or discard them). The caller is responsible for passing - /// the returned state to [`Lifecycle::end_request`]. - /// - /// Prefer [`replace`](Self::replace) unless you need manual control over lifecycle - /// request lifetime. + /// Returns `Ok` if the entry was admitted and `Err(_)` if it wasn't. pub fn replace_with_lifecycle( &self, key: Key, @@ -487,10 +470,16 @@ impl< Ok(()) } + /// Inserts an item in the cache with key `key`. /// Inserts an item in the cache with key `key`. pub fn insert_with_lifecycle(&self, key: Key, value: Val) -> L::RequestState { let mut lcs = self.lifecycle.begin_request(); - self.insert_with_state(key, value, &mut lcs); + let (shard, hash) = self.shard_for(&key).unwrap(); + let result = shard + .write() + .insert(&mut lcs, hash, key, value, InsertStrategy::Insert); + // result cannot err with the Insert strategy + debug_assert!(result.is_ok()); lcs } @@ -512,57 +501,6 @@ impl< debug_assert!(result.is_ok()); } - /// Attempts to insert an item in the cache with key `key` without blocking. - /// Returns `Ok(())` if the item was inserted, or `Err((key, value))` if the shard lock - /// could not be acquired without blocking. Lock contention is the only failure - /// mode: the inputs are returned so the caller can retry or discard them. - pub fn try_insert(&self, key: Key, value: Val) -> Result<(), (Key, Val)> { - let lcs = self.try_insert_with_lifecycle(key, value)?; - self.lifecycle.end_request(lcs); - Ok(()) - } - - /// Attempts to insert an item in the cache with key `key` without blocking. - /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, - /// or `Err((key, value))` if the shard lock could not be acquired without blocking. - /// Lock contention is the only failure mode: the inputs are returned so the - /// caller can retry or discard them. - pub fn try_insert_with_lifecycle( - &self, - key: Key, - value: Val, - ) -> Result { - // Tradeoff: begin_request is called before acquiring the shard lock to avoid holding - // the lock during potentially expensive lifecycle initialization. - let mut lcs = self.lifecycle.begin_request(); - self.try_insert_with_state(key, value, &mut lcs)?; - Ok(lcs) - } - - /// Attempts to insert an item in the cache with key `key` without blocking. - /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, - /// or `Err((key, value))` if the shard lock could not be acquired without blocking. - /// Lock contention is the only failure mode: the inputs are returned so the - /// caller can retry or discard them. - pub fn try_insert_with_state( - &self, - key: Key, - value: Val, - lcs: &mut L::RequestState, - ) -> Result<(), (Key, Val)> { - let (shard, hash) = self.shard_for(&key).unwrap(); - - match shard.try_write() { - Some(mut shard) => { - let result = shard.insert(lcs, hash, key, value, InsertStrategy::Insert); - // result cannot err with the Insert strategy - debug_assert!(result.is_ok()); - Ok(()) - } - _ => Err((key, value)), - } - } - /// Attempts to insert an item in the cache with key `key` without blocking. /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, /// or `Err((key, value))` if the shard lock could not be acquired without blocking. @@ -669,18 +607,9 @@ impl< PlaceholderGuard::join(&self.lifecycle, shard, hash, key, timeout) } - /// Gets an item from the cache with key `key`, or inserts one produced by `with`. + /// Gets or inserts an item in the cache with key `key`. /// - /// If the key is already present, the cached value is returned without calling `with`. - /// Otherwise, the cache is locked for this key (other callers to `get_value_or_guard` - /// or the `get_or_insert` family will block until the value is populated), `with` is - /// called to produce the value, and the result is inserted and returned. - /// - /// `with` may return an error, in which case nothing is inserted and the error is - /// propagated. The placeholder is dropped so waiting callers can retry. - /// - /// See also [`get_value_or_guard`](Self::get_value_or_guard) for more control over - /// the placeholder lifecycle. + /// See also `get_value_or_guard` and `get_value_or_guard_async`. pub fn get_or_insert_with( &self, key: &Q, @@ -731,12 +660,7 @@ impl< } } - /// Gets an item from the cache with key `key`, or inserts one produced by `with`. - /// - /// Async counterpart of [`get_or_insert_with`](Self::get_or_insert_with). The `with` - /// future is only polled when no value exists for `key`. While `with` is being awaited, - /// other tasks looking up the same key will suspend and resume once the value is - /// inserted (or the guard is dropped without inserting). + /// Gets or inserts an item in the cache with key `key`. pub async fn get_or_insert_async( &self, key: &Q, From e3bb3d013bd0f21444b30e68497a82648f207a44 Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla Date: Thu, 25 Jun 2026 15:08:13 +0200 Subject: [PATCH 08/12] cleanup --- src/sync.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 118b9ca..38daf84 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -483,24 +483,6 @@ impl< lcs } - /// Inserts an item in the cache with key `key` using an existing lifecycle request state. - /// - /// `lcs` must have been obtained from a prior [`Lifecycle::begin_request`] call and - /// must **not** have been passed to [`Lifecycle::end_request`] yet. Any items evicted - /// by this insert are recorded into `lcs` and will be dropped when `end_request` is - /// eventually called. - /// - /// Prefer [`insert`](Self::insert) for the common case where you do not need to - /// manage the lifecycle state manually. - pub fn insert_with_state(&self, key: Key, value: Val, lcs: &mut L::RequestState) { - let (shard, hash) = self.shard_for(&key).unwrap(); - let result = shard - .write() - .insert(lcs, hash, key, value, InsertStrategy::Insert); - // result cannot err with the Insert strategy - debug_assert!(result.is_ok()); - } - /// Attempts to insert an item in the cache with key `key` without blocking. /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, /// or `Err((key, value))` if the shard lock could not be acquired without blocking. From 313c4a107060abc21e6b4f25ce5abcbe0b675c1c Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla Date: Thu, 25 Jun 2026 16:00:41 +0200 Subject: [PATCH 09/12] empty commit --- src/sync.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sync.rs b/src/sync.rs index 38daf84..b1b99ab 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -363,6 +363,7 @@ impl< /// Returns per-item statistics for `key`, or `None` if the key is not present. /// Like peeks, this does not alter the key "hotness" or its access count. + /// Only available when the `stats` feature is enabled. #[cfg(feature = "stats")] pub fn item_stats(&self, key: &Q) -> Option where From 7ad7fa40582ee47aef482ce461fbc0006c77dbc8 Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla Date: Fri, 26 Jun 2026 14:08:18 +0200 Subject: [PATCH 10/12] add try_* method --- src/shard.rs | 2 ++ src/sync.rs | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/shard.rs b/src/shard.rs index 0e4c2d5..2d2e52c 100644 --- a/src/shard.rs +++ b/src/shard.rs @@ -1474,6 +1474,8 @@ impl, B, L, Plh: SharedPlaceholder> #[cfg(test)] mod tests { + use crate::shard::Entry; + #[cfg(not(feature = "stats"))] use super::*; diff --git a/src/sync.rs b/src/sync.rs index b1b99ab..65ee4e5 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -363,7 +363,6 @@ impl< /// Returns per-item statistics for `key`, or `None` if the key is not present. /// Like peeks, this does not alter the key "hotness" or its access count. - /// Only available when the `stats` feature is enabled. #[cfg(feature = "stats")] pub fn item_stats(&self, key: &Q) -> Option where @@ -373,6 +372,24 @@ impl< shard.read().item_stats(hash, key) } + /// Attempts to return per-item statistics for `key`. + /// Like peeks, this does not alter the key "hotness" or its access count. + /// Returns `Ok(Some(stats))` if the key is present, `Ok(None)` if absent, + /// or `Err(LockContention)` if the shard lock could not be acquired without blocking. + #[cfg(feature = "stats")] + pub fn try_item_stats(&self, key: &Q) -> Result, LockContention> + where + Q: Hash + Equivalent + ?Sized, + { + let Some((shard, hash)) = self.shard_for(key) else { + return Ok(None); + }; + match shard.try_read() { + Some(guard) => Ok(guard.item_stats(hash, key)), + None => Err(LockContention), + } + } + /// Remove an item from the cache whose key is `key`. /// Returns the removed entry, if any. pub fn remove(&self, key: &Q) -> Option<(Key, Val)> From 6eb8e7c48af2f4cf79d332a46777659cedb0a10d Mon Sep 17 00:00:00 2001 From: Faiaz Sanaulla Date: Fri, 26 Jun 2026 14:22:07 +0200 Subject: [PATCH 11/12] cleanup --- src/shard.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/shard.rs b/src/shard.rs index 1256f9d..ee0a014 100644 --- a/src/shard.rs +++ b/src/shard.rs @@ -1470,9 +1470,6 @@ impl, B, L, Plh: SharedPlaceholder> #[cfg(test)] mod tests { - use crate::shard::Entry; - - #[cfg(not(feature = "stats"))] use super::*; // The tight entry overhead is only guaranteed without the `stats` feature, From 344004abca93831b2133f15d15ca46e5b7bcc06c Mon Sep 17 00:00:00 2001 From: Arthur Silva Date: Fri, 26 Jun 2026 19:00:03 +0200 Subject: [PATCH 12/12] Address review: unify stats hit macros, fix entry_overhead under stats, document stats overhead - Fold record_item_hit{,_mut} into arity-overloaded record_hit{,_mut} arms so each resident hit-site records both counters in one call. - Re-enable entry_overhead under the stats feature; assert the real per-cache sizes (sync unchanged, unsync +8 via discriminant niche). - Document the stats feature's per-entry size and per-hit/miss cost. --- src/lib.rs | 7 ++++-- src/shard.rs | 63 +++++++++++++++++++++++----------------------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5f08c72..f706b8b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -79,7 +79,7 @@ //! | `parking_lot` | ✓ | Use [parking_lot](https://crates.io/crates/parking_lot) for synchronization primitives. Mutually exclusive with `sharded-lock`. | //! | `sharded-lock` | | Use [`crossbeam_utils::sync::ShardedLock`](https://docs.rs/crossbeam-utils/latest/crossbeam_utils/sync/struct.ShardedLock.html) for synchronization primitives. Mutually exclusive with `parking_lot`. | //! | `shuttle` | | Enable [shuttle](https://crates.io/crates/shuttle) testing support for concurrency testing. | -//! | `stats` | | Enable cache statistics tracking via the `hits()` and `misses()` methods. | +//! | `stats` | | Enable cache statistics tracking via the `hits()`, `misses()`, and per-item `item_stats()` methods. Overhead: adds an 8-byte per-item access counter (`AtomicU64`) to each resident item — raising per-entry memory by up to 8 bytes, depending on layout — and performs two atomic increments per cache hit and one per miss. | #![allow(clippy::type_complexity)] #![cfg_attr(docsrs, feature(doc_cfg))] @@ -273,7 +273,10 @@ impl MemoryUsed { /// Per-item statistics returned by `item_stats`. /// -/// Only available with the `stats` feature enabled. +/// Only available with the `stats` feature enabled. Enabling that feature adds an +/// 8-byte per-item access counter to each resident item (raising per-entry memory +/// by up to 8 bytes, depending on layout) and performs two atomic increments per +/// cache hit and one per miss. #[cfg(feature = "stats")] #[non_exhaustive] #[derive(Debug, Copy, Clone, PartialEq, Eq)] diff --git a/src/shard.rs b/src/shard.rs index ee0a014..7b110a1 100644 --- a/src/shard.rs +++ b/src/shard.rs @@ -183,12 +183,22 @@ macro_rules! record_hit { ($self: expr) => {{ $self.hits.fetch_add(1, atomic::Ordering::Relaxed); }}; + ($self: expr, $resident: expr) => {{ + $self.hits.fetch_add(1, atomic::Ordering::Relaxed); + $resident + .access_count + .fetch_add(1, atomic::Ordering::Relaxed); + }}; } #[cfg(feature = "stats")] macro_rules! record_hit_mut { ($self: expr) => {{ *$self.hits.get_mut() += 1; }}; + ($self: expr, $resident: expr) => {{ + *$self.hits.get_mut() += 1; + *$resident.access_count.get_mut() += 1; + }}; } #[cfg(feature = "stats")] macro_rules! record_miss { @@ -202,28 +212,15 @@ macro_rules! record_miss_mut { *$self.misses.get_mut() += 1; }}; } -#[cfg(feature = "stats")] -macro_rules! record_item_hit { - ($resident: expr) => {{ - $resident - .access_count - .fetch_add(1, atomic::Ordering::Relaxed); - }}; -} -#[cfg(feature = "stats")] -macro_rules! record_item_hit_mut { - ($resident: expr) => {{ - *$resident.access_count.get_mut() += 1; - }}; -} - #[cfg(not(feature = "stats"))] macro_rules! record_hit { ($self: expr) => {{}}; + ($self: expr, $resident: expr) => {{}}; } #[cfg(not(feature = "stats"))] macro_rules! record_hit_mut { ($self: expr) => {{}}; + ($self: expr, $resident: expr) => {{}}; } #[cfg(not(feature = "stats"))] macro_rules! record_miss { @@ -233,14 +230,6 @@ macro_rules! record_miss { macro_rules! record_miss_mut { ($self: expr) => {{}}; } -#[cfg(not(feature = "stats"))] -macro_rules! record_item_hit { - ($resident: expr) => {{}}; -} -#[cfg(not(feature = "stats"))] -macro_rules! record_item_hit_mut { - ($resident: expr) => {{}}; -} impl CacheShard { pub fn remove_placeholder(&mut self, placeholder: &Plh) { @@ -596,8 +585,7 @@ impl< // Even if that happens there's no impact correctness wise. resident.referenced.fetch_add(1, atomic::Ordering::Relaxed); } - record_hit!(self); - record_item_hit!(resident); + record_hit!(self, resident); Some((&resident.key, &resident.value)) } else { record_miss!(self); @@ -628,8 +616,7 @@ impl< if *resident.referenced.get_mut() < MAX_F { *resident.referenced.get_mut() += 1; } - record_hit_mut!(self); - record_item_hit_mut!(resident); + record_hit_mut!(self, resident); let old_weight = self.weighter.weight(&resident.key, &resident.value); Some(RefMut { @@ -1214,8 +1201,7 @@ impl< if *resident.referenced.get_mut() < MAX_F { *resident.referenced.get_mut() += 1; } - record_hit_mut!(self); - record_item_hit_mut!(resident); + record_hit_mut!(self, resident); unsafe { // Rustc gets insanely confused returning references from mut borrows // Safety: value will have the same lifetime as `resident` @@ -1266,7 +1252,6 @@ impl< return match action { EntryAction::Retain(t) => { - record_hit_mut!(self); let Some((Entry::Resident(resident), _)) = self.entries.get_mut(idx) else { // SAFETY: we had a mut reference to the Resident under `idx` until the previous line unsafe { unreachable_unchecked() }; @@ -1274,7 +1259,7 @@ impl< if *resident.referenced.get_mut() < MAX_F { *resident.referenced.get_mut() += 1; } - record_item_hit_mut!(resident); + record_hit_mut!(self, resident); EntryOrPlaceholder::Kept(t) } EntryAction::Remove => { @@ -1472,8 +1457,6 @@ impl, B, L, Plh: SharedPlaceholder> mod tests { use super::*; - // The tight entry overhead is only guaranteed without the `stats` feature, - // which adds a per-item `access_count` field to each `Resident`. #[cfg(not(feature = "stats"))] #[test] fn reserve_caps_ghost_headroom() { @@ -1510,14 +1493,24 @@ mod tests { #[test] fn entry_overhead() { use std::mem::size_of; + // 8 bytes from the linked slab, 8 bytes from the entry enum. + // `stats` adds an 8-byte `access_count` to each `Resident` (24 -> 32 bytes). + // Whether the slab entry grows then depends on enum discriminant/niche layout: + // the sync entry's discriminant moves into the `Arc` niche, cancelling the + // growth (stays 32), while the unsync entry loses its niche and grows by 8. + // (Layout-dependent and not guaranteed stable across rustc versions.) assert_eq!( size_of::>>() - size_of::<[u64; 2]>(), - 16 // 8 bytes from linked slab, 8 bytes from entry + 16 ); + #[cfg(not(feature = "stats"))] + let unsync_overhead = 16; + #[cfg(feature = "stats")] + let unsync_overhead = 24; assert_eq!( size_of::>() - size_of::<[u64; 2]>(), - 16 // 8 bytes from linked slab, 8 bytes from entry + unsync_overhead ); } }