From 0b632cd07b624e2862d09fb311f9f2e3cf172f85 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 6 Oct 2026 19:27:32 -0500 Subject: [PATCH 1/7] Move ListSnapshot into its own internal file Pure move, no behavior change: ListSnapshot and IListSnapshot were private nested types of InMemoryCacheClient and are now internal types in InMemoryListSnapshot.cs so they can be tested directly. --- src/Foundatio/Caching/InMemoryCacheClient.cs | 230 ----------------- src/Foundatio/Caching/InMemoryListSnapshot.cs | 237 ++++++++++++++++++ 2 files changed, 237 insertions(+), 230 deletions(-) create mode 100644 src/Foundatio/Caching/InMemoryListSnapshot.cs diff --git a/src/Foundatio/Caching/InMemoryCacheClient.cs b/src/Foundatio/Caching/InMemoryCacheClient.cs index 58141b000..566973c23 100644 --- a/src/Foundatio/Caching/InMemoryCacheClient.cs +++ b/src/Foundatio/Caching/InMemoryCacheClient.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Collections.Immutable; using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; using System.Linq; @@ -1364,235 +1363,6 @@ public virtual void Dispose() return new CacheEntry(value, expiresAt, _timeProvider, _shouldClone, size); } - private interface IListSnapshot - { - object GetSnapshot(); - } - - // Updates copy one bounded block and the block directory, not the entire list. The immutable - // key index shares unchanged nodes. Raw dictionary reads retain their existing public shape. - private sealed class ListSnapshot : IListSnapshot where T : notnull - { - public const int BlockSize = 32; - private ImmutableDictionary? _indices; - private readonly Dictionary? _initialValues; - private readonly Item[][] _blocks; - private readonly ImmutableSortedDictionary _expirations; - private readonly ImmutableStack _freeIndices; - private readonly int _nextIndex; - private readonly int _permanentCount; - private Dictionary? _snapshot; - - private readonly record struct Item(T Key, DateTime? Expiration, bool Present); - - public ListSnapshot(Dictionary values) - { - _initialValues = values; - var expirations = ImmutableSortedDictionary.CreateBuilder(); - _blocks = new Item[(values.Count + BlockSize - 1) / BlockSize][]; - for (int i = 0; i < _blocks.Length; i++) - _blocks[i] = new Item[BlockSize]; - foreach (var pair in values) - { - _blocks[_nextIndex / BlockSize][_nextIndex % BlockSize] = new Item(pair.Key, pair.Value, true); - _nextIndex++; - if (pair.Value is { } expiration) - expirations[expiration] = expirations.GetValueOrDefault(expiration) + 1; - else - _permanentCount++; - } - _expirations = expirations.ToImmutable(); - _freeIndices = ImmutableStack.Empty; - } - - private ListSnapshot(ImmutableDictionary indices, Item[][] blocks, - ImmutableSortedDictionary expirations, ImmutableStack freeIndices, int nextIndex, int permanentCount) - { - _indices = indices; - _blocks = blocks; - _expirations = expirations; - _freeIndices = freeIndices; - _nextIndex = nextIndex; - _permanentCount = permanentCount; - } - - public int Count => _indices?.Count ?? _initialValues!.Count; - private ImmutableDictionary Indices - { - get - { - if (Volatile.Read(ref _indices) is { } existing) - return existing; - - var builder = ImmutableDictionary.CreateBuilder(_initialValues!.Comparer); - int index = 0; - foreach (var key in _initialValues.Keys) - builder.Add(key, index++); - var indices = builder.ToImmutable(); - return Interlocked.CompareExchange(ref _indices, indices, null) ?? indices; - } - } - - public Dictionary? Snapshot => Volatile.Read(ref _snapshot); - public DateTime? ExpiresAt => _permanentCount > 0 || _expirations.IsEmpty ? null : _expirations.Last().Key; - - public T[] Read(DateTime utcNow) - { - var result = new T[Count]; - int count = 0; - foreach (var block in _blocks) - foreach (var item in block) - if (item.Present && (item.Expiration is null || item.Expiration >= utcNow)) - result[count++] = item.Key; - if (count != result.Length) - Array.Resize(ref result, count); - return result; - } - - public object GetSnapshot() - { - if (Snapshot is { } snapshot) - return snapshot; - - if (_initialValues is not null) - return Interlocked.CompareExchange(ref _snapshot, _initialValues, null) ?? _initialValues; - - var dictionary = CopyValues(); - return Interlocked.CompareExchange(ref _snapshot, dictionary, null) ?? dictionary; - } - - private Dictionary CopyValues() - { - if (_initialValues is not null) - return new Dictionary(_initialValues, _initialValues.Comparer); - - var dictionary = new Dictionary(Count, Indices.KeyComparer); - foreach (var block in _blocks) - foreach (var item in block) - if (item.Present) - dictionary.Add(item.Key, item.Expiration); - return dictionary; - } - - public ListSnapshot Update(ICollection items, DateTime? expiration, DateTime utcNow, bool remove, out long removed) - { - removed = 0; - bool prune = !_expirations.IsEmpty && _expirations.First().Key < utcNow; - if (remove && !prune && !items.Any(key => _initialValues?.ContainsKey(key) ?? Indices.ContainsKey(key))) - return this; - - // When a batch could copy every block, one dictionary copy is cheaper than - // rebuilding many index paths. It also avoids indexing short-lived bulk lists. - if (items.Count >= Math.Max(1, Count / BlockSize)) - { - var dictionary = CopyValues(); - if (prune) - foreach (var key in dictionary.Where(pair => pair.Value < utcNow).Select(pair => pair.Key).ToArray()) - dictionary.Remove(key); - if (remove) - removed = items.Count(dictionary.Remove); - else - foreach (var key in items) - dictionary[key] = expiration; - return new ListSnapshot(dictionary); - } - - var indices = Indices.ToBuilder(); - int capacity = Math.Max(_nextIndex, Count + (remove ? 0 : items.Count)); - var blocks = new Item[(capacity + BlockSize - 1) / BlockSize][]; - Array.Copy(_blocks, blocks, _blocks.Length); - var copied = new bool[blocks.Length]; - ImmutableSortedDictionary.Builder? expirations = null; - var freeIndices = _freeIndices; - int nextIndex = _nextIndex; - int permanentCount = _permanentCount; - bool changed = false; - - if (prune) - foreach (var block in _blocks) - foreach (var item in block) - if (item.Present && item.Expiration < utcNow) - Remove(item.Key); - - foreach (var item in items) - { - if (remove) - { - if (Remove(item)) - removed++; - continue; - } - - T key = item; - bool exists = indices.TryGetValue(key, out int index); - if (exists) - { - var previous = blocks[index / BlockSize][index % BlockSize]; - if (previous.Expiration == expiration) - continue; - key = previous.Key; - AdjustExpiration(previous.Expiration, -1); - } - else if (!freeIndices.IsEmpty) - { - index = freeIndices.Peek(); - freeIndices = freeIndices.Pop(); - } - else - index = nextIndex++; - - CopyBlock(index); - blocks[index / BlockSize][index % BlockSize] = new Item(key, expiration, true); - indices[key] = index; - AdjustExpiration(expiration, 1); - } - - if (!changed) - return this; - - int blockCount = (nextIndex + BlockSize - 1) / BlockSize; - if (blocks.Length != blockCount) - Array.Resize(ref blocks, blockCount); - return new ListSnapshot(indices.ToImmutable(), blocks, expirations?.ToImmutable() ?? _expirations, freeIndices, nextIndex, permanentCount); - - bool Remove(T key) - { - if (!indices.TryGetValue(key, out int index)) - return false; - CopyBlock(index); - AdjustExpiration(blocks[index / BlockSize][index % BlockSize].Expiration, -1); - blocks[index / BlockSize][index % BlockSize] = default; - indices.Remove(key); - freeIndices = freeIndices.Push(index); - return true; - } - - void CopyBlock(int index) - { - int blockIndex = index / BlockSize; - if (copied[blockIndex]) - return; - blocks[blockIndex] = blockIndex < _blocks.Length ? (Item[])_blocks[blockIndex].Clone() : new Item[BlockSize]; - copied[blockIndex] = true; - changed = true; - } - - void AdjustExpiration(DateTime? value, int delta) - { - if (value is not { } time) - { - permanentCount += delta; - return; - } - expirations ??= _expirations.ToBuilder(); - int count = expirations.GetValueOrDefault(time) + delta; - if (count == 0) - expirations.Remove(time); - else - expirations[time] = count; - } - } - } /// /// A cache entry's value, expiration and size never change after it is published to the cache dictionary; diff --git a/src/Foundatio/Caching/InMemoryListSnapshot.cs b/src/Foundatio/Caching/InMemoryListSnapshot.cs new file mode 100644 index 000000000..306915975 --- /dev/null +++ b/src/Foundatio/Caching/InMemoryListSnapshot.cs @@ -0,0 +1,237 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; + +namespace Foundatio.Caching; + +internal interface IListSnapshot +{ + object GetSnapshot(); +} + +// Updates copy one bounded block and the block directory, not the entire list. The immutable +// key index shares unchanged nodes. Raw dictionary reads retain their existing public shape. +internal sealed class ListSnapshot : IListSnapshot where T : notnull +{ + public const int BlockSize = 32; + private ImmutableDictionary? _indices; + private readonly Dictionary? _initialValues; + private readonly Item[][] _blocks; + private readonly ImmutableSortedDictionary _expirations; + private readonly ImmutableStack _freeIndices; + private readonly int _nextIndex; + private readonly int _permanentCount; + private Dictionary? _snapshot; + + private readonly record struct Item(T Key, DateTime? Expiration, bool Present); + + public ListSnapshot(Dictionary values) + { + _initialValues = values; + var expirations = ImmutableSortedDictionary.CreateBuilder(); + _blocks = new Item[(values.Count + BlockSize - 1) / BlockSize][]; + for (int i = 0; i < _blocks.Length; i++) + _blocks[i] = new Item[BlockSize]; + foreach (var pair in values) + { + _blocks[_nextIndex / BlockSize][_nextIndex % BlockSize] = new Item(pair.Key, pair.Value, true); + _nextIndex++; + if (pair.Value is { } expiration) + expirations[expiration] = expirations.GetValueOrDefault(expiration) + 1; + else + _permanentCount++; + } + _expirations = expirations.ToImmutable(); + _freeIndices = ImmutableStack.Empty; + } + + private ListSnapshot(ImmutableDictionary indices, Item[][] blocks, + ImmutableSortedDictionary expirations, ImmutableStack freeIndices, int nextIndex, int permanentCount) + { + _indices = indices; + _blocks = blocks; + _expirations = expirations; + _freeIndices = freeIndices; + _nextIndex = nextIndex; + _permanentCount = permanentCount; + } + + public int Count => _indices?.Count ?? _initialValues!.Count; + private ImmutableDictionary Indices + { + get + { + if (Volatile.Read(ref _indices) is { } existing) + return existing; + + var builder = ImmutableDictionary.CreateBuilder(_initialValues!.Comparer); + int index = 0; + foreach (var key in _initialValues.Keys) + builder.Add(key, index++); + var indices = builder.ToImmutable(); + return Interlocked.CompareExchange(ref _indices, indices, null) ?? indices; + } + } + + public Dictionary? Snapshot => Volatile.Read(ref _snapshot); + public DateTime? ExpiresAt => _permanentCount > 0 || _expirations.IsEmpty ? null : _expirations.Last().Key; + + public T[] Read(DateTime utcNow) + { + var result = new T[Count]; + int count = 0; + foreach (var block in _blocks) + foreach (var item in block) + if (item.Present && (item.Expiration is null || item.Expiration >= utcNow)) + result[count++] = item.Key; + if (count != result.Length) + Array.Resize(ref result, count); + return result; + } + + public object GetSnapshot() + { + if (Snapshot is { } snapshot) + return snapshot; + + if (_initialValues is not null) + return Interlocked.CompareExchange(ref _snapshot, _initialValues, null) ?? _initialValues; + + var dictionary = CopyValues(); + return Interlocked.CompareExchange(ref _snapshot, dictionary, null) ?? dictionary; + } + + private Dictionary CopyValues() + { + if (_initialValues is not null) + return new Dictionary(_initialValues, _initialValues.Comparer); + + var dictionary = new Dictionary(Count, Indices.KeyComparer); + foreach (var block in _blocks) + foreach (var item in block) + if (item.Present) + dictionary.Add(item.Key, item.Expiration); + return dictionary; + } + + public ListSnapshot Update(ICollection items, DateTime? expiration, DateTime utcNow, bool remove, out long removed) + { + removed = 0; + bool prune = !_expirations.IsEmpty && _expirations.First().Key < utcNow; + if (remove && !prune && !items.Any(key => _initialValues?.ContainsKey(key) ?? Indices.ContainsKey(key))) + return this; + + // When a batch could copy every block, one dictionary copy is cheaper than + // rebuilding many index paths. It also avoids indexing short-lived bulk lists. + if (items.Count >= Math.Max(1, Count / BlockSize)) + { + var dictionary = CopyValues(); + if (prune) + foreach (var key in dictionary.Where(pair => pair.Value < utcNow).Select(pair => pair.Key).ToArray()) + dictionary.Remove(key); + if (remove) + removed = items.Count(dictionary.Remove); + else + foreach (var key in items) + dictionary[key] = expiration; + return new ListSnapshot(dictionary); + } + + var indices = Indices.ToBuilder(); + int capacity = Math.Max(_nextIndex, Count + (remove ? 0 : items.Count)); + var blocks = new Item[(capacity + BlockSize - 1) / BlockSize][]; + Array.Copy(_blocks, blocks, _blocks.Length); + var copied = new bool[blocks.Length]; + ImmutableSortedDictionary.Builder? expirations = null; + var freeIndices = _freeIndices; + int nextIndex = _nextIndex; + int permanentCount = _permanentCount; + bool changed = false; + + if (prune) + foreach (var block in _blocks) + foreach (var item in block) + if (item.Present && item.Expiration < utcNow) + Remove(item.Key); + + foreach (var item in items) + { + if (remove) + { + if (Remove(item)) + removed++; + continue; + } + + T key = item; + bool exists = indices.TryGetValue(key, out int index); + if (exists) + { + var previous = blocks[index / BlockSize][index % BlockSize]; + if (previous.Expiration == expiration) + continue; + key = previous.Key; + AdjustExpiration(previous.Expiration, -1); + } + else if (!freeIndices.IsEmpty) + { + index = freeIndices.Peek(); + freeIndices = freeIndices.Pop(); + } + else + index = nextIndex++; + + CopyBlock(index); + blocks[index / BlockSize][index % BlockSize] = new Item(key, expiration, true); + indices[key] = index; + AdjustExpiration(expiration, 1); + } + + if (!changed) + return this; + + int blockCount = (nextIndex + BlockSize - 1) / BlockSize; + if (blocks.Length != blockCount) + Array.Resize(ref blocks, blockCount); + return new ListSnapshot(indices.ToImmutable(), blocks, expirations?.ToImmutable() ?? _expirations, freeIndices, nextIndex, permanentCount); + + bool Remove(T key) + { + if (!indices.TryGetValue(key, out int index)) + return false; + CopyBlock(index); + AdjustExpiration(blocks[index / BlockSize][index % BlockSize].Expiration, -1); + blocks[index / BlockSize][index % BlockSize] = default; + indices.Remove(key); + freeIndices = freeIndices.Push(index); + return true; + } + + void CopyBlock(int index) + { + int blockIndex = index / BlockSize; + if (copied[blockIndex]) + return; + blocks[blockIndex] = blockIndex < _blocks.Length ? (Item[])_blocks[blockIndex].Clone() : new Item[BlockSize]; + copied[blockIndex] = true; + changed = true; + } + + void AdjustExpiration(DateTime? value, int delta) + { + if (value is not { } time) + { + permanentCount += delta; + return; + } + expirations ??= _expirations.ToBuilder(); + int count = expirations.GetValueOrDefault(time) + delta; + if (count == 0) + expirations.Remove(time); + else + expirations[time] = count; + } + } +} From bbc1c36665603d499bd9515bdbec4707d41e2bd6 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 6 Oct 2026 19:48:51 -0500 Subject: [PATCH 2/7] Make list snapshots cheap to create and to change for the first time - A snapshot built from a dictionary keeps only the dictionary and its expiration bounds; blocks and the key index are built on the first small update, so bulk writes are one dictionary copy (fast copy constructor, no rehash). - Replace the ImmutableDictionary key index with small hash-partitioned dictionaries that are copied on write. The first small change on a 1,000-item list went from ~129 us / 90 KB (building the AVL tree) to a single pass. - Track min/max expiration per block instead of an ImmutableSortedDictionary; pruning skips blocks with nothing expired. - Add AsReadOnlyDictionary(), a read-only IDictionary/ICollection view used to size lists without copying them. - Add InMemoryListSnapshotTests, including a 3,000-step randomized comparison against a Dictionary model. --- src/Foundatio/Caching/InMemoryListSnapshot.cs | 521 +++++++++++++----- .../Caching/InMemoryListSnapshotTests.cs | 206 +++++++ 2 files changed, 588 insertions(+), 139 deletions(-) create mode 100644 tests/Foundatio.Tests/Caching/InMemoryListSnapshotTests.cs diff --git a/src/Foundatio/Caching/InMemoryListSnapshot.cs b/src/Foundatio/Caching/InMemoryListSnapshot.cs index 306915975..aaaabcafa 100644 --- a/src/Foundatio/Caching/InMemoryListSnapshot.cs +++ b/src/Foundatio/Caching/InMemoryListSnapshot.cs @@ -1,7 +1,10 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; +using System.Collections.ObjectModel; using System.Linq; +using System.Numerics; using System.Threading; namespace Foundatio.Caching; @@ -11,81 +14,84 @@ internal interface IListSnapshot object GetSnapshot(); } -// Updates copy one bounded block and the block directory, not the entire list. The immutable -// key index shares unchanged nodes. Raw dictionary reads retain their existing public shape. +/// +/// An immutable list value: each update returns a new snapshot and never changes this one, so readers and +/// compare-and-swap writers can share it without locks. +/// +/// +/// A snapshot starts out backed by a plain dictionary, which makes bulk writes a single dictionary copy. The first +/// small update builds an indexed layout: items in fixed-size blocks that keep insertion order and their own +/// expiration bounds, plus a key index split into small hash partitions. Small updates then copy only the blocks and +/// partitions they touch. +/// internal sealed class ListSnapshot : IListSnapshot where T : notnull { public const int BlockSize = 32; - private ImmutableDictionary? _indices; - private readonly Dictionary? _initialValues; - private readonly Item[][] _blocks; - private readonly ImmutableSortedDictionary _expirations; - private readonly ImmutableStack _freeIndices; - private readonly int _nextIndex; + private const int PartitionSize = 32; + private const long Permanent = long.MaxValue; + + private readonly Dictionary? _values; + private readonly IEqualityComparer _comparer; private readonly int _permanentCount; + private readonly long _minExpirationTicks; + private readonly long _maxExpirationTicks; + private Layout? _layout; private Dictionary? _snapshot; - private readonly record struct Item(T Key, DateTime? Expiration, bool Present); - public ListSnapshot(Dictionary values) { - _initialValues = values; - var expirations = ImmutableSortedDictionary.CreateBuilder(); - _blocks = new Item[(values.Count + BlockSize - 1) / BlockSize][]; - for (int i = 0; i < _blocks.Length; i++) - _blocks[i] = new Item[BlockSize]; - foreach (var pair in values) - { - _blocks[_nextIndex / BlockSize][_nextIndex % BlockSize] = new Item(pair.Key, pair.Value, true); - _nextIndex++; - if (pair.Value is { } expiration) - expirations[expiration] = expirations.GetValueOrDefault(expiration) + 1; - else - _permanentCount++; - } - _expirations = expirations.ToImmutable(); - _freeIndices = ImmutableStack.Empty; + _values = values; + _comparer = values.Comparer; + Count = values.Count; + _minExpirationTicks = long.MaxValue; + foreach (var expiration in values.Values) + Track(ToTicks(expiration), ref _permanentCount, ref _minExpirationTicks, ref _maxExpirationTicks); } - private ListSnapshot(ImmutableDictionary indices, Item[][] blocks, - ImmutableSortedDictionary expirations, ImmutableStack freeIndices, int nextIndex, int permanentCount) + private ListSnapshot(Layout layout, IEqualityComparer comparer, int count) { - _indices = indices; - _blocks = blocks; - _expirations = expirations; - _freeIndices = freeIndices; - _nextIndex = nextIndex; - _permanentCount = permanentCount; - } - - public int Count => _indices?.Count ?? _initialValues!.Count; - private ImmutableDictionary Indices - { - get + _layout = layout; + _comparer = comparer; + Count = count; + _minExpirationTicks = long.MaxValue; + foreach (var block in layout.Blocks) { - if (Volatile.Read(ref _indices) is { } existing) - return existing; - - var builder = ImmutableDictionary.CreateBuilder(_initialValues!.Comparer); - int index = 0; - foreach (var key in _initialValues.Keys) - builder.Add(key, index++); - var indices = builder.ToImmutable(); - return Interlocked.CompareExchange(ref _indices, indices, null) ?? indices; + _permanentCount += block.PermanentCount; + _minExpirationTicks = Math.Min(_minExpirationTicks, block.MinExpirationTicks); + _maxExpirationTicks = Math.Max(_maxExpirationTicks, block.MaxExpirationTicks); } } + public int Count { get; } + + /// The materialized dictionary, once a raw read has requested it; afterwards it is the source of truth. public Dictionary? Snapshot => Volatile.Read(ref _snapshot); - public DateTime? ExpiresAt => _permanentCount > 0 || _expirations.IsEmpty ? null : _expirations.Last().Key; + + /// The latest item expiration, or null when any item is permanent or the list is empty. + public DateTime? ExpiresAt => _permanentCount > 0 || Count == 0 ? null : FromTicks(_maxExpirationTicks); + + /// A read-only dictionary over the current items, without copying them. + public IDictionary AsReadOnlyDictionary() => new ReadOnlyView(this); public T[] Read(DateTime utcNow) { var result = new T[Count]; int count = 0; - foreach (var block in _blocks) - foreach (var item in block) - if (item.Present && (item.Expiration is null || item.Expiration >= utcNow)) - result[count++] = item.Key; + long nowTicks = utcNow.Ticks; + if (Volatile.Read(ref _layout) is null && _values is not null) + { + foreach (var pair in _values) + if (pair.Value is null || pair.Value.Value.Ticks >= nowTicks) + result[count++] = pair.Key; + } + else + { + foreach (var block in GetLayout().Blocks) + foreach (var item in block.Items) + if (item.Present && item.ExpirationTicks >= nowTicks) + result[count++] = item.Key; + } + if (count != result.Length) Array.Resize(ref result, count); return result; @@ -96,40 +102,25 @@ public object GetSnapshot() if (Snapshot is { } snapshot) return snapshot; - if (_initialValues is not null) - return Interlocked.CompareExchange(ref _snapshot, _initialValues, null) ?? _initialValues; - - var dictionary = CopyValues(); + var dictionary = _values ?? CopyValues(); return Interlocked.CompareExchange(ref _snapshot, dictionary, null) ?? dictionary; } - private Dictionary CopyValues() - { - if (_initialValues is not null) - return new Dictionary(_initialValues, _initialValues.Comparer); - - var dictionary = new Dictionary(Count, Indices.KeyComparer); - foreach (var block in _blocks) - foreach (var item in block) - if (item.Present) - dictionary.Add(item.Key, item.Expiration); - return dictionary; - } - public ListSnapshot Update(ICollection items, DateTime? expiration, DateTime utcNow, bool remove, out long removed) { removed = 0; - bool prune = !_expirations.IsEmpty && _expirations.First().Key < utcNow; - if (remove && !prune && !items.Any(key => _initialValues?.ContainsKey(key) ?? Indices.ContainsKey(key))) + long nowTicks = utcNow.Ticks; + bool prune = _minExpirationTicks < nowTicks; + if (remove && !prune && !items.Any(ContainsKey)) return this; - // When a batch could copy every block, one dictionary copy is cheaper than - // rebuilding many index paths. It also avoids indexing short-lived bulk lists. + // When a batch could touch every block, one dictionary copy is cheaper than updating blocks and partitions, + // and it avoids indexing short-lived bulk lists. if (items.Count >= Math.Max(1, Count / BlockSize)) { var dictionary = CopyValues(); if (prune) - foreach (var key in dictionary.Where(pair => pair.Value < utcNow).Select(pair => pair.Key).ToArray()) + foreach (var key in dictionary.Where(pair => pair.Value?.Ticks < nowTicks).Select(pair => pair.Key).ToArray()) dictionary.Remove(key); if (remove) removed = items.Count(dictionary.Remove); @@ -139,99 +130,351 @@ public ListSnapshot Update(ICollection items, DateTime? expiration, DateTi return new ListSnapshot(dictionary); } - var indices = Indices.ToBuilder(); - int capacity = Math.Max(_nextIndex, Count + (remove ? 0 : items.Count)); - var blocks = new Item[(capacity + BlockSize - 1) / BlockSize][]; - Array.Copy(_blocks, blocks, _blocks.Length); - var copied = new bool[blocks.Length]; - ImmutableSortedDictionary.Builder? expirations = null; - var freeIndices = _freeIndices; - int nextIndex = _nextIndex; - int permanentCount = _permanentCount; - bool changed = false; - + var update = new LayoutUpdate(GetLayout(), _comparer, Count); if (prune) - foreach (var block in _blocks) - foreach (var item in block) - if (item.Present && item.Expiration < utcNow) - Remove(item.Key); + update.RemoveExpired(nowTicks); + long ticks = ToTicks(expiration); foreach (var item in items) { if (remove) { - if (Remove(item)) + if (update.Remove(item)) removed++; - continue; } + else + { + update.Set(item, ticks); + } + } + + return update.Changed ? new ListSnapshot(update.Build(), _comparer, update.Count) : this; + } + + private bool ContainsKey(T key) => TryGetExpiration(key, out _); + + private bool TryGetExpiration(T key, out DateTime? expiration) + { + if (Volatile.Read(ref _layout) is null && _values is not null) + return _values.TryGetValue(key, out expiration); + + var layout = GetLayout(); + if (layout.Partitions[layout.PartitionOf(key, _comparer)].TryGetValue(key, out int index)) + { + expiration = FromTicks(layout.Blocks[index / BlockSize].Items[index % BlockSize].ExpirationTicks); + return true; + } + + expiration = null; + return false; + } + + private IEnumerable> EnumeratePairs() + { + if (Volatile.Read(ref _layout) is null && _values is not null) + { + foreach (var pair in _values) + yield return pair; + yield break; + } + + foreach (var block in GetLayout().Blocks) + foreach (var item in block.Items) + if (item.Present) + yield return new KeyValuePair(item.Key, FromTicks(item.ExpirationTicks)); + } + + private Dictionary CopyValues() + { + if (_values is not null) + return new Dictionary(_values, _comparer); + + var dictionary = new Dictionary(Count, _comparer); + foreach (var pair in EnumeratePairs()) + dictionary.Add(pair.Key, pair.Value); + return dictionary; + } + + private Layout GetLayout() + { + if (Volatile.Read(ref _layout) is { } existing) + return existing; + + var layout = Layout.Create(_values!, _comparer); + return Interlocked.CompareExchange(ref _layout, layout, null) ?? layout; + } + + private static long ToTicks(DateTime? expiration) => expiration?.Ticks ?? Permanent; + + private static DateTime? FromTicks(long ticks) => ticks == Permanent ? null : new DateTime(ticks, DateTimeKind.Utc); + + private static void Track(long ticks, ref int permanentCount, ref long minTicks, ref long maxTicks) + { + if (ticks == Permanent) + { + permanentCount++; + return; + } + + minTicks = Math.Min(minTicks, ticks); + maxTicks = Math.Max(maxTicks, ticks); + } + + // ExpirationTicks is Permanent for an item with no expiration; Present is false for a free slot + private readonly record struct Item(T Key, long ExpirationTicks, bool Present = true); + + private sealed class Block + { + public Block(Item[] items) + { + Items = items; + MinExpirationTicks = long.MaxValue; + foreach (var item in items) + if (item.Present) + Track(item.ExpirationTicks, ref PermanentCount, ref MinExpirationTicks, ref MaxExpirationTicks); + } - T key = item; - bool exists = indices.TryGetValue(key, out int index); - if (exists) + public readonly Item[] Items; + public readonly int PermanentCount; + public readonly long MinExpirationTicks; + public readonly long MaxExpirationTicks; + } + + private sealed class Layout(Block[] blocks, Dictionary[] partitions, ImmutableStack freeSlots, int nextSlot) + { + public Block[] Blocks { get; } = blocks; + public Dictionary[] Partitions { get; } = partitions; + public ImmutableStack FreeSlots { get; } = freeSlots; + public int NextSlot { get; } = nextSlot; + + public static Layout Create(Dictionary values, IEqualityComparer comparer) + { + var blocks = new Block[(values.Count + BlockSize - 1) / BlockSize]; + var items = new Item[BlockSize]; + int slot = 0; + foreach (var pair in values) { - var previous = blocks[index / BlockSize][index % BlockSize]; - if (previous.Expiration == expiration) - continue; - key = previous.Key; - AdjustExpiration(previous.Expiration, -1); + items[slot % BlockSize] = new Item(pair.Key, ToTicks(pair.Value)); + if (++slot % BlockSize == 0 || slot == values.Count) + { + blocks[(slot - 1) / BlockSize] = new Block(items); + items = new Item[BlockSize]; + } } - else if (!freeIndices.IsEmpty) + + return new Layout(blocks, BuildPartitions(blocks, values.Count, comparer), ImmutableStack.Empty, values.Count); + } + + public int PartitionOf(T key, IEqualityComparer comparer) => PartitionOf(key, comparer, Partitions.Length); + + public static Dictionary[] BuildPartitions(Block[] blocks, int count, IEqualityComparer comparer) + { + int partitionCount = (int)BitOperations.RoundUpToPowerOf2((uint)Math.Max(1, count / PartitionSize)); + var partitions = new Dictionary[partitionCount]; + for (int i = 0; i < partitions.Length; i++) + partitions[i] = new Dictionary(PartitionSize + PartitionSize / 2, comparer); + + for (int blockIndex = 0; blockIndex < blocks.Length; blockIndex++) { - index = freeIndices.Peek(); - freeIndices = freeIndices.Pop(); + var blockItems = blocks[blockIndex].Items; + for (int i = 0; i < blockItems.Length; i++) + if (blockItems[i].Present) + partitions[PartitionOf(blockItems[i].Key, comparer, partitionCount)].Add(blockItems[i].Key, blockIndex * BlockSize + i); } - else - index = nextIndex++; - CopyBlock(index); - blocks[index / BlockSize][index % BlockSize] = new Item(key, expiration, true); - indices[key] = index; - AdjustExpiration(expiration, 1); + return partitions; } - if (!changed) - return this; + private static int PartitionOf(T key, IEqualityComparer comparer, int partitionCount) + { + if (partitionCount == 1) + return 0; + + // Fibonacci hashing spreads keys whose hash codes share low bits (sequential or aligned values) + uint hash = (uint)comparer.GetHashCode(key) * 0x9E3779B9u; + return (int)(hash >> (32 - BitOperations.Log2((uint)partitionCount))); + } + } - int blockCount = (nextIndex + BlockSize - 1) / BlockSize; - if (blocks.Length != blockCount) - Array.Resize(ref blocks, blockCount); - return new ListSnapshot(indices.ToImmutable(), blocks, expirations?.ToImmutable() ?? _expirations, freeIndices, nextIndex, permanentCount); + /// + /// Applies changes to a copy of a layout, copying each block and partition the first time it is touched. + /// + private sealed class LayoutUpdate + { + private readonly Layout _source; + private readonly IEqualityComparer _comparer; + private Block[] _blocks; + private Item[]?[] _copiedBlocks; + private readonly Dictionary[] _partitions; + private readonly bool[] _copiedPartitions; + private ImmutableStack _freeSlots; + private int _nextSlot; + + public LayoutUpdate(Layout source, IEqualityComparer comparer, int count) + { + _source = source; + _comparer = comparer; + _blocks = source.Blocks; + _copiedBlocks = new Item[]?[source.Blocks.Length]; + _partitions = (Dictionary[])source.Partitions.Clone(); + _copiedPartitions = new bool[_partitions.Length]; + _freeSlots = source.FreeSlots; + _nextSlot = source.NextSlot; + Count = count; + } + + public int Count { get; private set; } + public bool Changed { get; private set; } + + public void RemoveExpired(long nowTicks) + { + for (int blockIndex = 0; blockIndex < _source.Blocks.Length; blockIndex++) + { + var block = _source.Blocks[blockIndex]; + if (block.MinExpirationTicks >= nowTicks) + continue; - bool Remove(T key) + for (int i = 0; i < block.Items.Length; i++) + { + long ticks = block.Items[i].ExpirationTicks; + if (block.Items[i].Present && ticks < nowTicks) + Remove(block.Items[i].Key); + } + } + } + + public bool Remove(T key) { - if (!indices.TryGetValue(key, out int index)) + var partition = _partitions[_source.PartitionOf(key, _comparer)]; + if (!partition.TryGetValue(key, out int slot)) return false; - CopyBlock(index); - AdjustExpiration(blocks[index / BlockSize][index % BlockSize].Expiration, -1); - blocks[index / BlockSize][index % BlockSize] = default; - indices.Remove(key); - freeIndices = freeIndices.Push(index); + + WritablePartition(key).Remove(key); + WritableBlock(slot)[slot % BlockSize] = default; + _freeSlots = _freeSlots.Push(slot); + Count--; return true; } - void CopyBlock(int index) + public void Set(T key, long expirationTicks) { - int blockIndex = index / BlockSize; - if (copied[blockIndex]) + if (_partitions[_source.PartitionOf(key, _comparer)].TryGetValue(key, out int slot)) + { + ref readonly var existing = ref ReadSlot(slot); + if (existing.ExpirationTicks == expirationTicks) + return; + + WritableBlock(slot)[slot % BlockSize] = new Item(existing.Key, expirationTicks); return; - blocks[blockIndex] = blockIndex < _blocks.Length ? (Item[])_blocks[blockIndex].Clone() : new Item[BlockSize]; - copied[blockIndex] = true; - changed = true; + } + + if (!_freeSlots.IsEmpty) + _freeSlots = _freeSlots.Pop(out slot); + else + slot = _nextSlot++; + + WritableBlock(slot)[slot % BlockSize] = new Item(key, expirationTicks); + WritablePartition(key).Add(key, slot); + Count++; + } + + public Layout Build() + { + for (int i = 0; i < _blocks.Length; i++) + if (_copiedBlocks[i] is { } items) + _blocks[i] = new Block(items); + + // Keep partitions small as the list grows, so a change copies a bounded amount of index + var partitions = Count > _partitions.Length * PartitionSize * 2 + ? Layout.BuildPartitions(_blocks, Count, _comparer) + : _partitions; + return new Layout(_blocks, partitions, _freeSlots, _nextSlot); } - void AdjustExpiration(DateTime? value, int delta) + private ref readonly Item ReadSlot(int slot) { - if (value is not { } time) + int blockIndex = slot / BlockSize; + return ref (_copiedBlocks[blockIndex] ?? _blocks[blockIndex].Items)[slot % BlockSize]; + } + + private Item[] WritableBlock(int slot) + { + int blockIndex = slot / BlockSize; + if (blockIndex >= _blocks.Length) { - permanentCount += delta; - return; + // Slots are handed out in order, so a new slot needs at most one more block + Array.Resize(ref _blocks, blockIndex + 1); + Array.Resize(ref _copiedBlocks, blockIndex + 1); } - expirations ??= _expirations.ToBuilder(); - int count = expirations.GetValueOrDefault(time) + delta; - if (count == 0) - expirations.Remove(time); - else - expirations[time] = count; + + if (_copiedBlocks[blockIndex] is { } copied) + return copied; + + if (ReferenceEquals(_blocks, _source.Blocks)) + _blocks = (Block[])_source.Blocks.Clone(); + + Changed = true; + var items = _blocks[blockIndex] is { } block ? (Item[])block.Items.Clone() : new Item[BlockSize]; + _copiedBlocks[blockIndex] = items; + return items; } + + private Dictionary WritablePartition(T key) + { + int index = _source.PartitionOf(key, _comparer); + if (!_copiedPartitions[index]) + { + _partitions[index] = new Dictionary(_partitions[index], _comparer); + _copiedPartitions[index] = true; + } + + return _partitions[index]; + } + } + + /// A read-only over a snapshot, used to size lists without copying them. + private sealed class ReadOnlyView(ListSnapshot snapshot) : IDictionary, IReadOnlyDictionary, ICollection + { + public int Count => snapshot.Count; + public bool IsReadOnly => true; + bool ICollection.IsSynchronized => false; + object ICollection.SyncRoot => snapshot; + + public DateTime? this[T key] + { + get => snapshot.TryGetExpiration(key, out var expiration) ? expiration : throw new KeyNotFoundException($"The key '{key}' was not found in the list."); + set => throw ReadOnly(); + } + + public ICollection Keys => new ReadOnlyCollection(snapshot.EnumeratePairs().Select(pair => pair.Key).ToArray()); + public ICollection Values => new ReadOnlyCollection(snapshot.EnumeratePairs().Select(pair => pair.Value).ToArray()); + IEnumerable IReadOnlyDictionary.Keys => Keys; + IEnumerable IReadOnlyDictionary.Values => Values; + + public bool ContainsKey(T key) => snapshot.ContainsKey(key); + public bool TryGetValue(T key, out DateTime? value) => snapshot.TryGetExpiration(key, out value); + public bool Contains(KeyValuePair item) => snapshot.TryGetExpiration(item.Key, out var value) && value == item.Value; + public IEnumerator> GetEnumerator() => snapshot.EnumeratePairs().GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public void CopyTo(KeyValuePair[] array, int arrayIndex) + { + foreach (var pair in snapshot.EnumeratePairs()) + array[arrayIndex++] = pair; + } + + void ICollection.CopyTo(Array array, int index) + { + foreach (var pair in snapshot.EnumeratePairs()) + array.SetValue(pair, index++); + } + + public void Add(T key, DateTime? value) => throw ReadOnly(); + public void Add(KeyValuePair item) => throw ReadOnly(); + public bool Remove(T key) => throw ReadOnly(); + public bool Remove(KeyValuePair item) => throw ReadOnly(); + public void Clear() => throw ReadOnly(); + + private static NotSupportedException ReadOnly() => new("Cached list values are read-only."); } } diff --git a/tests/Foundatio.Tests/Caching/InMemoryListSnapshotTests.cs b/tests/Foundatio.Tests/Caching/InMemoryListSnapshotTests.cs new file mode 100644 index 000000000..8cc41f39c --- /dev/null +++ b/tests/Foundatio.Tests/Caching/InMemoryListSnapshotTests.cs @@ -0,0 +1,206 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Foundatio.Caching; +using Xunit; + +namespace Foundatio.Tests.Caching; + +public class InMemoryListSnapshotTests +{ + private static readonly DateTime Now = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Update_OnDictionaryBackedSnapshot_AppliesSingleItemChange(bool remove) + { + // Arrange + var snapshot = new ListSnapshot(CreateValues(1000)); + int item = remove ? 500 : -1; + + // Act + var updated = snapshot.Update([item], null, Now, remove, out long removed); + + // Assert + var expected = Enumerable.Range(0, 1000).Where(i => i != 500 || !remove).ToList(); + if (!remove) + expected.Add(-1); + Assert.Equal(remove ? 1 : 0, removed); + Assert.Equal(expected.Count, updated.Count); + Assert.Equal(expected.Order(), updated.Read(Now).Order()); + } + + [Fact] + public void Update_WhenGrowingPastPartitionCount_KeepsAllValues() + { + // Arrange + var snapshot = new ListSnapshot(CreateValues(64)); + + // Act + for (int i = 64; i < 2064; i++) + snapshot = snapshot.Update([i], null, Now, remove: false, out _); + for (int i = 0; i < 2064; i += 2) + snapshot = snapshot.Update([i], null, Now, remove: true, out _); + + // Assert + var expected = Enumerable.Range(0, 2064).Where(i => i % 2 == 1).ToArray(); + Assert.Equal(expected.Length, snapshot.Count); + Assert.Equal(expected, snapshot.Read(Now).Order()); + Assert.All(expected, i => Assert.True(snapshot.AsReadOnlyDictionary().ContainsKey(i))); + Assert.False(snapshot.AsReadOnlyDictionary().ContainsKey(0)); + } + + [Fact] + public void ExpiresAt_WithMixedPermanentAndExpiringItems_ReturnsLatestOrNull() + { + // Arrange + var values = CreateValues(100, i => Now.AddMinutes(1 + i % 3)); + values[1000] = null; + var snapshot = new ListSnapshot(values); + + // Act + var withoutPermanent = snapshot.Update([1000], null, Now, remove: true, out _); + var withoutLatest = withoutPermanent.Update(values.Where(kvp => kvp.Value == Now.AddMinutes(3)).Select(kvp => kvp.Key).ToArray(), null, Now, remove: true, out _); + var singleChange = withoutLatest.Update([2000], Now.AddMinutes(10), Now, remove: false, out _); + + // Assert + Assert.Null(snapshot.ExpiresAt); + Assert.Equal(Now.AddMinutes(3), withoutPermanent.ExpiresAt); + Assert.Equal(Now.AddMinutes(2), withoutLatest.ExpiresAt); + Assert.Equal(Now.AddMinutes(10), singleChange.ExpiresAt); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Update_WithExpiredItems_PrunesThem(bool indexed) + { + // Arrange + var values = CreateValues(1000, i => i < 100 ? Now.AddMinutes(-1) : null); + var snapshot = new ListSnapshot(values); + if (indexed) + snapshot = snapshot.Update([5000], null, Now.AddMinutes(-2), remove: false, out _); + + // Act + var updated = snapshot.Update([-1], null, Now, remove: false, out _); + + // Assert + var expected = Enumerable.Range(100, 900).Append(-1).Concat(indexed ? [5000] : []).Order().ToArray(); + Assert.Equal(expected.Length, updated.Count); + Assert.Equal(expected, updated.Read(Now).Order()); + Assert.False(updated.AsReadOnlyDictionary().ContainsKey(0)); + } + + [Theory] + [InlineData("single-add")] + [InlineData("single-remove")] + [InlineData("batch-add")] + [InlineData("batch-remove")] + [InlineData("prune")] + public void Update_DoesNotMutateSourceSnapshot(string operation) + { + // Arrange + var values = CreateValues(1000, i => i < 10 ? Now.AddMinutes(-1) : null); + foreach (bool indexed in new[] { false, true }) + { + var source = new ListSnapshot(values); + if (indexed) + source = source.Update([5000], null, Now.AddMinutes(-2), remove: false, out _); + var before = source.Read(Now.AddMinutes(-2)).Order().ToArray(); + var expiresBefore = source.ExpiresAt; + + // Act + _ = operation switch + { + "single-add" => source.Update([-1], null, Now.AddMinutes(-2), remove: false, out _), + "single-remove" => source.Update([500], null, Now.AddMinutes(-2), remove: true, out _), + "batch-add" => source.Update(Enumerable.Range(2000, 600).ToArray(), null, Now.AddMinutes(-2), remove: false, out _), + "batch-remove" => source.Update(Enumerable.Range(0, 600).ToArray(), null, Now.AddMinutes(-2), remove: true, out _), + _ => source.Update([-1], null, Now, remove: false, out _) + }; + + // Assert + Assert.Equal(before, source.Read(Now.AddMinutes(-2)).Order()); + Assert.Equal(before.Length, source.Count); + Assert.Equal(expiresBefore, source.ExpiresAt); + Assert.Equal(before.Length, source.AsReadOnlyDictionary().Count); + } + } + + [Fact] + public void Update_WithRandomOperations_MatchesDictionaryModel() + { + // Arrange + var random = new Random(570); + var model = CreateValues(200); + var snapshot = new ListSnapshot(new Dictionary(model)); + var utcNow = Now; + + for (int step = 0; step < 3000; step++) + { + // Act + utcNow = utcNow.AddSeconds(random.Next(0, 3)); + int batchSize = random.Next(10) == 0 ? random.Next(20, 400) : random.Next(1, 3); + var items = Enumerable.Range(0, batchSize).Select(_ => random.Next(0, 1500)).Distinct().ToArray(); + bool remove = random.Next(2) == 0; + DateTime? expiration = random.Next(4) == 0 ? utcNow.AddSeconds(random.Next(1, 30)) : null; + snapshot = snapshot.Update(items, remove ? null : expiration, utcNow, remove, out long removed); + + foreach (int key in model.Where(kvp => kvp.Value < utcNow).Select(kvp => kvp.Key).ToArray()) + model.Remove(key); + long expectedRemoved = 0; + foreach (int item in items) + { + if (remove) + expectedRemoved += model.Remove(item) ? 1 : 0; + else + model[item] = expiration; + } + + // Assert + Assert.Equal(expectedRemoved, removed); + Assert.Equal(model.Count, snapshot.Count); + Assert.Equal(model.Keys.Order(), snapshot.Read(utcNow).Order()); + Assert.Equal(model.Values.Contains(null) || model.Count == 0 ? null : model.Values.Max(), snapshot.ExpiresAt); + var view = snapshot.AsReadOnlyDictionary(); + Assert.Equal(model.Count, view.Count); + Assert.Equal(model.OrderBy(kvp => kvp.Key), view.OrderBy(kvp => kvp.Key)); + } + } + + [Fact] + public void AsReadOnlyDictionary_ExposesCurrentItemsAndRejectsChanges() + { + // Arrange + var values = CreateValues(100, i => i == 7 ? Now.AddMinutes(5) : null); + var snapshot = new ListSnapshot(values).Update([500], null, Now, remove: false, out _); + + // Act + var view = snapshot.AsReadOnlyDictionary(); + + // Assert + Assert.Equal(101, view.Count); + Assert.Equal(101, ((ICollection)view).Count); + Assert.True(view.IsReadOnly); + Assert.True(view.ContainsKey(500)); + Assert.True(view.TryGetValue(7, out var expiration)); + Assert.Equal(Now.AddMinutes(5), expiration); + Assert.Null(view[0]); + Assert.False(view.TryGetValue(-1, out _)); + Assert.Throws(() => view[-1]); + Assert.Equal(Enumerable.Range(0, 100).Append(500).Order(), view.Keys.Order()); + Assert.Equal(101, view.Values.Count); + Assert.Contains(new KeyValuePair(7, Now.AddMinutes(5)), view); + Assert.Throws(() => view.Add(-1, null)); + Assert.Throws(() => view.Remove(0)); + Assert.Throws(() => view[0] = Now); + Assert.Throws(view.Clear); + } + + private static Dictionary CreateValues(int count, Func? expiration = null) + { + return Enumerable.Range(0, count).ToDictionary(i => i, i => expiration?.Invoke(i)); + } +} From 4513933b21fd2095d100dfe87281c74869561b0c Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 6 Oct 2026 19:48:51 -0500 Subject: [PATCH 3/7] Size shared lists through a read-only view instead of copying them - Lists are shared (no full copy per write) whenever cloning is off, in every sizing mode; previously only no sizing and WithFixedSizing shared them. Snapshot updates size the whole list through ListSnapshot.AsReadOnlyDictionary(). - ListAddAsync no longer sizes and clones the incoming batch on its own when the key already holds a shared list. - Behavior change: a custom SizeCalculator receives list values as a read-only IDictionary rather than a Dictionary. Documented in the SizeCalculator contract, the caching guide and the skill. - Remove FixedSizeCalculator, which existed only for the sharing check; WithFixedSizing is back to main's lambda. --- .agents/skills/foundatio/SKILL.md | 1 + docs/guide/caching.md | 4 +- src/Foundatio/Caching/InMemoryCacheClient.cs | 48 ++++++++++++++----- .../Caching/InMemoryCacheClientOptions.cs | 10 ++-- .../Caching/InMemoryCacheClientTests.cs | 29 ++++++++++- 5 files changed, 71 insertions(+), 21 deletions(-) diff --git a/.agents/skills/foundatio/SKILL.md b/.agents/skills/foundatio/SKILL.md index 518d58f3a..ef4d1a298 100644 --- a/.agents/skills/foundatio/SKILL.md +++ b/.agents/skills/foundatio/SKILL.md @@ -313,6 +313,7 @@ public class OrderServiceTests : TestLoggerBase - **JobWithLockBase vs manual locking**: Use `JobWithLockBase` when the entire run must be single-instance (leader election). Use manual `ILockProvider.AcquireAsync` inside `JobBase` for finer-grained locking within a job. - **JobContext.RenewLockAsync**: Call in long-running jobs (both `JobBase` and `QueueJobBase`) to prevent lock expiration mid-processing. - **In-memory expired entries**: Writes treat an expired entry as missing before maintenance sweeps it (as Redis does): increments and `SetIfHigher`/`SetIfLower` start fresh, `ReplaceAsync`, `SetExpirationAsync`, `RemoveIfEqualAsync` and `ReplaceIfEqualAsync` miss. The write reclaims the expired entry and raises `ItemExpired` once. Successful live conditional removal and empty-list removal free memory without that event. Size-rejected updates preserve value, TTL, and tracked size. +- **Custom `SizeCalculator` and lists**: list values arrive as a read-only `IDictionary`, not `Dictionary`. Match on the interface; casting to the concrete type throws. - **In-memory numeric TTLs**: `SetIfHigherAsync` and `SetIfLowerAsync` apply the requested expiration even when the numeric value is unchanged. A zero return value does not mean the TTL was preserved; `null` clears it. Size-rejected updates remain unchanged. - **In-memory lists and cloning**: List writes preserve supported dictionary types and comparers; copying an unsupported `IDictionary` throws `NotSupportedException` without changing the entry. Batch large updates because cloning, custom sizing, and raw dictionary access can require whole-list copies. `Items` honors `CloneValues` without affecting LRU. Collection snapshots do not isolate shared mutable elements when cloning is disabled. See the [caching guide](https://foundatio.readthedocs.io/guide/caching.html#in-memory-update-behavior) for details. - **Register as singletons**: All infrastructure services (`ICacheClient`, `IMessageBus`, `IQueue`, `IFileStorage`, `ILockProvider`) maintain internal state and connections -- always register as singletons. diff --git a/docs/guide/caching.md b/docs/guide/caching.md index 191c4975b..e1a79faf2 100644 --- a/docs/guide/caching.md +++ b/docs/guide/caching.md @@ -273,7 +273,9 @@ List updates preserve the collection structure of previously returned snapshots. `Items` returns cached values without changing eviction order. Both `Items` and ordinary reads honor `CloneValues`. With cloning disabled, mutable payloads and list elements can remain shared: a collection snapshot is not a deep copy. Do not mutate shared objects concurrently with cache operations. ::: tip Large list updates -Batch values into a single `ListAddAsync` or `ListRemoveAsync` call. Cloning, custom size calculation, and raw dictionary access can require copies proportional to the list size. Benchmark representative list sizes and read/write patterns before choosing these options; atomic updates do not make every workload allocation-free. +Batch values into a single `ListAddAsync` or `ListRemoveAsync` call. Lists over 32 items are stored as immutable snapshots, so a small change copies only the parts it touches, with or without a size calculator. Cloning and raw dictionary access (`GetAsync>`) can still require copies proportional to the list size. Benchmark representative list sizes and read/write patterns before choosing these options; atomic updates do not make every workload allocation-free. + +A custom `SizeCalculator` receives list values as a read-only `IDictionary` (item to expiration), not a concrete `Dictionary`. Check for the interface, not the concrete type, and don't modify it. ::: ### HybridCacheClient diff --git a/src/Foundatio/Caching/InMemoryCacheClient.cs b/src/Foundatio/Caching/InMemoryCacheClient.cs index 566973c23..a02318e7f 100644 --- a/src/Foundatio/Caching/InMemoryCacheClient.cs +++ b/src/Foundatio/Caching/InMemoryCacheClient.cs @@ -72,7 +72,7 @@ public InMemoryCacheClient(InMemoryCacheClientOptions? options = null) _sizeCalculator = options.SizeCalculator; _hasSizeCalculator = _sizeCalculator is not null; - _canShareListValues = !_shouldClone && (!_hasSizeCalculator || _sizeCalculator?.Target is InMemoryCacheClientOptionsBuilder.FixedSizeCalculator); + _canShareListValues = !_shouldClone; _shouldTrackMemory = _hasSizeCalculator && _maxMemorySize.HasValue; _memory = new ConcurrentDictionary(); @@ -601,25 +601,34 @@ private long ListAdd(string key, Dictionary items, DateTime? ex if (items.Count is 0) return 0; - var entry = CreateEntry(items, expiresAt); - if (entry is null) - return 0; - Interlocked.Increment(ref _writes); + // Sized and cloned only when a path needs the new items on their own, then reused across retries + CacheEntry? entry = null; + bool entryCreated = false; return UpdateEntry(key, current => { + if (current?.StoredValue is ListSnapshot { Snapshot: null } stored) + { + var list = stored.Update(items.Keys, expiresAt, _timeProvider.GetUtcNow().UtcDateTime, remove: false, out _); + long listSize = CalculateListSize(list); + return listSize < 0 ? (current, 0L) : (current.WithValue(list, list.ExpiresAt, listSize), (long)items.Count); + } + + if (!entryCreated) + { + entry = CreateEntry(items, expiresAt); + entryCreated = true; + } + + if (entry is null) + return (current, 0L); // Entry exceeds limits + if (current is null) return (_canShareListValues && items.Count > ListSnapshot.BlockSize ? entry.WithValue(new ListSnapshot(items), expiresAt, entry.Size) : entry, (long)items.Count); - if (current.StoredValue is ListSnapshot { Snapshot: null } stored) - { - var list = stored.Update(items.Keys, expiresAt, _timeProvider.GetUtcNow().UtcDateTime, remove: false, out _); - return (current.WithValue(list, list.ExpiresAt, entry.Size), (long)items.Count); - } - if (CopyListValues(current) is not { } dictionary) throw new InvalidOperationException($"Unable to add value for key: {key}. Cache value does not contain a set"); @@ -635,6 +644,14 @@ private long ListAdd(string key, Dictionary items, DateTime? ex }); } + /// + /// Sizes a shared list through a read-only view, so the calculator sees every item without the list being copied. + /// + private long CalculateListSize(ListSnapshot list) where T : notnull + { + return _hasSizeCalculator ? CalculateEntrySize(list.AsReadOnlyDictionary()) : 0; + } + private CacheEntry WithListValues(CacheEntry current, IDictionary dictionary, long size) where T : notnull { object value = _canShareListValues && dictionary is Dictionary values && values.Count > ListSnapshot.BlockSize @@ -704,8 +721,13 @@ private long ListRemove(string key, HashSet items) where T : notnull if (current?.StoredValue is ListSnapshot { Snapshot: null } list) { var replacement = list.Update(items, null, _timeProvider.GetUtcNow().UtcDateTime, remove: true, out long count); - return ReferenceEquals(list, replacement) ? (current, 0L) - : (replacement.Count == 0 ? null : current.WithValue(replacement, replacement.ExpiresAt, current.Size), count); + if (ReferenceEquals(list, replacement)) + return (current, 0L); + if (replacement.Count == 0) + return (null, count); + + long listSize = CalculateListSize(replacement); + return listSize < 0 ? (current, 0L) : (current.WithValue(replacement, replacement.ExpiresAt, listSize), count); } object? value = current?.StoredValue is IListSnapshot snapshot ? snapshot.GetSnapshot() : current?.StoredValue; diff --git a/src/Foundatio/Caching/InMemoryCacheClientOptions.cs b/src/Foundatio/Caching/InMemoryCacheClientOptions.cs index a7b6ad688..9752f3014 100644 --- a/src/Foundatio/Caching/InMemoryCacheClientOptions.cs +++ b/src/Foundatio/Caching/InMemoryCacheClientOptions.cs @@ -35,6 +35,9 @@ public class InMemoryCacheClientOptions : SharedOptions /// Negative return values will cause the entry to be skipped (not cached) with a warning logged /// Should be thread-safe as it may be called concurrently /// Should handle null values gracefully (typically return 8 bytes for a null reference) + /// List values (from ListAddAsync) are passed as a read-only + /// of item to expiration, not a concrete , so the list is not copied on each write. + /// Check for the interface, and do not modify it /// /// /// @@ -101,7 +104,7 @@ public InMemoryCacheClientOptionsBuilder SizeCalculator(Func sizeC public InMemoryCacheClientOptionsBuilder WithFixedSizing(long maxMemorySize, long averageEntrySize) { Target.MaxMemorySize = maxMemorySize; - Target.SizeCalculator = new FixedSizeCalculator(averageEntrySize).Calculate; + Target.SizeCalculator = _ => averageEntrySize; return this; } @@ -171,9 +174,4 @@ public InMemoryCacheClientOptionsBuilder ShouldThrowOnSerializationError(bool sh Target.ShouldThrowOnSerializationError = shouldThrow; return this; } - - internal sealed class FixedSizeCalculator(long size) - { - public long Calculate(object value) => size; - } } diff --git a/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs b/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs index 6369273b2..cab8e5b73 100644 --- a/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs +++ b/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs @@ -645,7 +645,7 @@ public async Task ListAddAsync_WithOverriddenFixedSizer_ValidatesReplacement() { // Arrange using var cache = new InMemoryCacheClient(o => o.WithFixedSizing(1000, 1) - .SizeCalculator(value => ((Dictionary)value).Count).MaxEntrySize(64)); + .SizeCalculator(value => ((IDictionary)value).Count).MaxEntrySize(64)); await cache.ListAddAsync("set", Enumerable.Range(0, 64)); // Act @@ -786,6 +786,33 @@ public override Task ListRemoveAsync_WithInvalidInputs_ThrowsAppropriateExceptio return base.ListRemoveAsync_WithInvalidInputs_ThrowsAppropriateException(); } + [Fact] + public async Task ListAddAsync_WithCustomSizeCalculator_ReceivesReadOnlyDictionaryOfCurrentItems() + { + // Arrange + var seen = new List<(int Count, bool IsReadOnly, bool HasNewItem, bool HasRemovedItem)>(); + using var cache = new InMemoryCacheClient(o => o.CloneValues(false).MaxMemorySize(100_000_000).SizeCalculator(value => + { + if (value is IDictionary list) + seen.Add((list.Count, list.IsReadOnly, list.ContainsKey(-1), list.ContainsKey(0))); + return 100; + })); + await cache.ListAddAsync("list", Enumerable.Range(0, 1000)); + seen.Clear(); + + // Act + await cache.ListAddAsync("list", [-1]); + await cache.ListRemoveAsync("list", [0]); + + // Assert + Assert.Equal([(1001, true, true, true), (1000, true, true, false)], seen); + var list = await cache.GetListAsync("list"); + Assert.Equal(1000, list.Value!.Count); + var dictionary = await cache.GetAsync>("list"); + Assert.IsType>(dictionary.Value); + Assert.Equal(1000, dictionary.Value!.Count); + } + [Fact] public async Task ListRemoveAsync_WithMissingValue_DoesNotAllocateInProportionToListSize() { From a98b04da5570c6c160cfa2fc71ccfd52fa24229f Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 6 Oct 2026 22:35:51 -0500 Subject: [PATCH 4/7] Bound wasted list work: copy before indexing, serialize after conflicts, share immutable items under cloning - The first small change to an unindexed list copies its dictionary (~3 us at 1,000 items) and only a list that keeps changing builds the block and partition index, so a list written once or twice never pays for indexing. - ListAddAsync and ListRemoveAsync take a lock striped by key (64 stripes) after a lost compare-and-swap, so writers queue instead of each redoing an O(n) copy. Publishing still uses compare-and-swap; other writers stay lock-free. - With CloneValues on, lists whose items never need cloning (numbers, strings, value types) also keep a shared snapshot. Raw dictionary reads still deep-clone the materialized dictionary; reference-type items keep copying. --- src/Foundatio/Caching/InMemoryCacheClient.cs | 90 ++++++++++++++----- src/Foundatio/Caching/InMemoryListSnapshot.cs | 20 ++++- .../Caching/InMemoryCacheClientTests.cs | 34 +++++++ .../Caching/InMemoryListSnapshotTests.cs | 23 +++-- 4 files changed, 137 insertions(+), 30 deletions(-) diff --git a/src/Foundatio/Caching/InMemoryCacheClient.cs b/src/Foundatio/Caching/InMemoryCacheClient.cs index a02318e7f..2c4df68ba 100644 --- a/src/Foundatio/Caching/InMemoryCacheClient.cs +++ b/src/Foundatio/Caching/InMemoryCacheClient.cs @@ -25,7 +25,8 @@ public class InMemoryCacheClient : IMemoryCacheClient, IHaveTimeProvider, IHaveL private readonly int? _maxItems; private readonly long? _maxMemorySize; private readonly bool _hasSizeCalculator; - private readonly bool _canShareListValues; + private const int ConflictLockCount = 64; + private object[]? _conflictLocks; private readonly bool _shouldTrackMemory; private Func? _sizeCalculator; private readonly long? _maxEntrySize; @@ -72,7 +73,6 @@ public InMemoryCacheClient(InMemoryCacheClientOptions? options = null) _sizeCalculator = options.SizeCalculator; _hasSizeCalculator = _sizeCalculator is not null; - _canShareListValues = !_shouldClone; _shouldTrackMemory = _hasSizeCalculator && _maxMemorySize.HasValue; _memory = new ConcurrentDictionary(); @@ -357,29 +357,65 @@ internal long RemoveExpiredKey(string key, bool sendNotification = true) /// null to remove it. It runs again with the latest entry when another writer changes the key first, /// so it must not have side effects; only the result of the attempt that was published is returned. /// - internal TResult UpdateEntry(string key, Func update) + /// + /// After a lost compare-and-swap, take a lock striped by key for the remaining attempts, so writers whose callback + /// is expensive (copying a large list) queue instead of each redoing that work. Publishing still uses + /// compare-and-swap, so writers that don't take the lock stay correct. + /// + internal TResult UpdateEntry(string key, Func update, bool serializeAfterConflict = false) { - // Retries only when another writer changed the key first, so some writer always makes progress; Dispose clears - // the store, which lets in-flight updates finish on the next attempt. - while (true) + object? stripe = null; + bool locked = false; + try { - _memory.TryGetValue(key, out var stored); - bool expired = stored is { IsExpired: true }; - var (desired, result) = update(expired ? null : stored); - if (ReferenceEquals(desired, stored)) - return result; + // Retries only when another writer changed the key first, so some writer always makes progress; Dispose + // clears the store, which lets in-flight updates finish on the next attempt. + while (true) + { + _memory.TryGetValue(key, out var stored); + bool expired = stored is { IsExpired: true }; + var (desired, result) = update(expired ? null : stored); + if (ReferenceEquals(desired, stored)) + return result; - if (!TryPublish(key, stored, desired)) - continue; + if (!TryPublish(key, stored, desired)) + { + if (serializeAfterConflict && !locked) + { + stripe = GetConflictLock(key); + Monitor.Enter(stripe, ref locked); + } + + continue; + } - // An entry that expires while the callback runs (a lease expiring mid-comparison) is also reclaimed - if (expired || (desired is null && stored is { IsExpired: true })) - OnItemExpired(key); + // An entry that expires while the callback runs (a lease expiring mid-comparison) is also reclaimed + if (expired || (desired is null && stored is { IsExpired: true })) + OnItemExpired(key); - return result; + return result; + } + } + finally + { + if (locked) + Monitor.Exit(stripe!); } } + private object GetConflictLock(string key) + { + var locks = LazyInitializer.EnsureInitialized(ref _conflictLocks, static () => + { + var created = new object[ConflictLockCount]; + for (int i = 0; i < created.Length; i++) + created[i] = new object(); + return created; + }); + + return locks[(int)((uint)key.GetHashCode() % ConflictLockCount)]; + } + /// /// Like , but the callback sees the stored entry even when it has expired. /// Only maintenance, compaction and explicit removal need this view. @@ -625,7 +661,7 @@ private long ListAdd(string key, Dictionary items, DateTime? ex return (current, 0L); // Entry exceeds limits if (current is null) - return (_canShareListValues && items.Count > ListSnapshot.BlockSize + return (CanShareListValues() && items.Count > ListSnapshot.BlockSize ? entry.WithValue(new ListSnapshot(items), expiresAt, entry.Size) : entry, (long)items.Count); @@ -641,7 +677,19 @@ private long ListAdd(string key, Dictionary items, DateTime? ex return size < 0 ? (current, 0L) : (WithListValues(current, dictionary, size), (long)items.Count); - }); + }, serializeAfterConflict: true); + } + + /// + /// Whether list writes can keep an immutable instead of copying the whole list. + /// With cloning on, only items that never need cloning (numbers, strings, other value types) can be shared: + /// raw dictionary reads still clone the materialized dictionary, and items can't be changed through it. + /// + private bool CanShareListValues() where T : notnull => !_shouldClone || !ListItemCloning.Required; + + private static class ListItemCloning + { + public static readonly bool Required = TypeRequiresCloning(typeof(T)); } /// @@ -654,7 +702,7 @@ private long CalculateListSize(ListSnapshot list) where T : notnull private CacheEntry WithListValues(CacheEntry current, IDictionary dictionary, long size) where T : notnull { - object value = _canShareListValues && dictionary is Dictionary values && values.Count > ListSnapshot.BlockSize + object value = CanShareListValues() && dictionary is Dictionary values && values.Count > ListSnapshot.BlockSize ? new ListSnapshot(values) : dictionary; return current.WithValue(value, GetListExpiration(dictionary), size); @@ -753,7 +801,7 @@ private long ListRemove(string key, HashSet items) where T : notnull return size < 0 ? (current, 0L) : (WithListValues(current, dictionary, size), removedCount); - }); + }, serializeAfterConflict: true); if (removed > 0) _logger.LogTrace("Removed value from set with cache key: {Key}", key); diff --git a/src/Foundatio/Caching/InMemoryListSnapshot.cs b/src/Foundatio/Caching/InMemoryListSnapshot.cs index aaaabcafa..3b17ccb90 100644 --- a/src/Foundatio/Caching/InMemoryListSnapshot.cs +++ b/src/Foundatio/Caching/InMemoryListSnapshot.cs @@ -31,6 +31,7 @@ internal sealed class ListSnapshot : IListSnapshot where T : notnull private const long Permanent = long.MaxValue; private readonly Dictionary? _values; + private readonly bool _indexOnNextChange; private readonly IEqualityComparer _comparer; private readonly int _permanentCount; private readonly long _minExpirationTicks; @@ -38,9 +39,15 @@ internal sealed class ListSnapshot : IListSnapshot where T : notnull private Layout? _layout; private Dictionary? _snapshot; - public ListSnapshot(Dictionary values) + /// The items, owned by the snapshot from now on. + /// + /// true when this dictionary came from a small change, so the list is being modified incrementally and the + /// next small change should build the index; a list written once (or only in bulk) never pays for indexing. + /// + public ListSnapshot(Dictionary values, bool indexOnNextChange = false) { _values = values; + _indexOnNextChange = indexOnNextChange; _comparer = values.Comparer; Count = values.Count; _minExpirationTicks = long.MaxValue; @@ -64,6 +71,9 @@ private ListSnapshot(Layout layout, IEqualityComparer comparer, int count) public int Count { get; } + /// Whether small updates use the block and partition index rather than copying the dictionary. + public bool IsIndexed => Volatile.Read(ref _layout) is not null; + /// The materialized dictionary, once a raw read has requested it; afterwards it is the source of truth. public Dictionary? Snapshot => Volatile.Read(ref _snapshot); @@ -115,8 +125,10 @@ public ListSnapshot Update(ICollection items, DateTime? expiration, DateTi return this; // When a batch could touch every block, one dictionary copy is cheaper than updating blocks and partitions, - // and it avoids indexing short-lived bulk lists. - if (items.Count >= Math.Max(1, Count / BlockSize)) + // and it avoids indexing short-lived bulk lists. The first small change to an unindexed list also copies, + // since a copy (~3 us at 1,000 items) is far cheaper than building the index for a list that may not change again. + bool batch = items.Count >= Math.Max(1, Count / BlockSize); + if (batch || (Volatile.Read(ref _layout) is null && _values is not null && !_indexOnNextChange)) { var dictionary = CopyValues(); if (prune) @@ -127,7 +139,7 @@ public ListSnapshot Update(ICollection items, DateTime? expiration, DateTi else foreach (var key in items) dictionary[key] = expiration; - return new ListSnapshot(dictionary); + return new ListSnapshot(dictionary, indexOnNextChange: !batch); } var update = new LayoutUpdate(GetLayout(), _comparer, Count); diff --git a/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs b/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs index cab8e5b73..8eb84608b 100644 --- a/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs +++ b/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs @@ -497,6 +497,40 @@ public async Task ListAddAsync_WithCloneValues_StoresCopiesOfAddedValues() Assert.DoesNotContain(stored.Keys, k => ReferenceEquals(k, added)); } + [Fact] + public async Task ListAddAsync_WithCloneValuesAndImmutableItems_SharesSnapshotAndIsolatesReads() + { + // Arrange + using var cache = new InMemoryCacheClient(o => o.CloneValues(true).LoggerFactory(Log)); + await cache.ListAddAsync("set", Enumerable.Range(0, 100)); + + // Act + await cache.ListAddAsync("set", [-1]); + var read = await cache.GetAsync>("set"); + read.Value!.Clear(); + await cache.ListRemoveAsync("set", [0]); + + // Assert + Assert.Equal(Enumerable.Range(1, 99).Append(-1).Order(), (await cache.GetListAsync("set")).Value!.Order()); + var reread = await cache.GetAsync>("set"); + Assert.Equal(100, reread.Value!.Count); + Assert.NotSame(read.Value, reread.Value); + } + + [Fact] + public async Task ListAddAsync_WithCloneValuesAndImmutableItems_UsesSharedSnapshot() + { + // Arrange + using var cache = new InMemoryCacheClient(o => o.CloneValues(true)); + await cache.ListAddAsync("set", Enumerable.Range(0, 100)); + + // Act + await cache.ListAddAsync("set", [-1]); + + // Assert + Assert.IsType>(GetEntry(cache, "set").StoredValue); + } + [Theory] [InlineData(false)] [InlineData(true)] diff --git a/tests/Foundatio.Tests/Caching/InMemoryListSnapshotTests.cs b/tests/Foundatio.Tests/Caching/InMemoryListSnapshotTests.cs index 8cc41f39c..3c6667870 100644 --- a/tests/Foundatio.Tests/Caching/InMemoryListSnapshotTests.cs +++ b/tests/Foundatio.Tests/Caching/InMemoryListSnapshotTests.cs @@ -30,6 +30,11 @@ public void Update_OnDictionaryBackedSnapshot_AppliesSingleItemChange(bool remov Assert.Equal(remove ? 1 : 0, removed); Assert.Equal(expected.Count, updated.Count); Assert.Equal(expected.Order(), updated.Read(Now).Order()); + Assert.False(updated.IsIndexed); + + var second = updated.Update([-2], null, Now, remove: false, out _); + Assert.True(second.IsIndexed); + Assert.Equal(expected.Append(-2).Order(), second.Read(Now).Order()); } [Fact] @@ -81,7 +86,7 @@ public void Update_WithExpiredItems_PrunesThem(bool indexed) var values = CreateValues(1000, i => i < 100 ? Now.AddMinutes(-1) : null); var snapshot = new ListSnapshot(values); if (indexed) - snapshot = snapshot.Update([5000], null, Now.AddMinutes(-2), remove: false, out _); + snapshot = Indexed(snapshot, Now.AddMinutes(-2)); // Act var updated = snapshot.Update([-1], null, Now, remove: false, out _); @@ -107,7 +112,7 @@ public void Update_DoesNotMutateSourceSnapshot(string operation) { var source = new ListSnapshot(values); if (indexed) - source = source.Update([5000], null, Now.AddMinutes(-2), remove: false, out _); + source = Indexed(source, Now.AddMinutes(-2)); var before = source.Read(Now.AddMinutes(-2)).Order().ToArray(); var expiresBefore = source.ExpiresAt; @@ -175,7 +180,7 @@ public void AsReadOnlyDictionary_ExposesCurrentItemsAndRejectsChanges() { // Arrange var values = CreateValues(100, i => i == 7 ? Now.AddMinutes(5) : null); - var snapshot = new ListSnapshot(values).Update([500], null, Now, remove: false, out _); + var snapshot = Indexed(new ListSnapshot(values), Now); // Act var view = snapshot.AsReadOnlyDictionary(); @@ -184,13 +189,13 @@ public void AsReadOnlyDictionary_ExposesCurrentItemsAndRejectsChanges() Assert.Equal(101, view.Count); Assert.Equal(101, ((ICollection)view).Count); Assert.True(view.IsReadOnly); - Assert.True(view.ContainsKey(500)); + Assert.True(view.ContainsKey(5000)); Assert.True(view.TryGetValue(7, out var expiration)); Assert.Equal(Now.AddMinutes(5), expiration); Assert.Null(view[0]); Assert.False(view.TryGetValue(-1, out _)); Assert.Throws(() => view[-1]); - Assert.Equal(Enumerable.Range(0, 100).Append(500).Order(), view.Keys.Order()); + Assert.Equal(Enumerable.Range(0, 100).Append(5000).Order(), view.Keys.Order()); Assert.Equal(101, view.Values.Count); Assert.Contains(new KeyValuePair(7, Now.AddMinutes(5)), view); Assert.Throws(() => view.Add(-1, null)); @@ -199,6 +204,14 @@ public void AsReadOnlyDictionary_ExposesCurrentItemsAndRejectsChanges() Assert.Throws(view.Clear); } + /// Adds item 5000 and makes the next small change build the index, the way repeated small writes do. + private static ListSnapshot Indexed(ListSnapshot snapshot, DateTime utcNow) + { + var indexed = snapshot.Update([5000], null, utcNow, remove: false, out _).Update([5000], null, utcNow, remove: false, out _); + Assert.True(indexed.IsIndexed); + return indexed; + } + private static Dictionary CreateValues(int count, Func? expiration = null) { return Enumerable.Range(0, count).ToDictionary(i => i, i => expiration?.Invoke(i)); From 736871fce908fb891ac5fa66dce9cfaba1dda0d8 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 6 Oct 2026 23:18:41 -0500 Subject: [PATCH 5/7] Document list snapshot sharing under CloneValues --- docs/guide/caching.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/caching.md b/docs/guide/caching.md index e1a79faf2..a7e899774 100644 --- a/docs/guide/caching.md +++ b/docs/guide/caching.md @@ -273,7 +273,7 @@ List updates preserve the collection structure of previously returned snapshots. `Items` returns cached values without changing eviction order. Both `Items` and ordinary reads honor `CloneValues`. With cloning disabled, mutable payloads and list elements can remain shared: a collection snapshot is not a deep copy. Do not mutate shared objects concurrently with cache operations. ::: tip Large list updates -Batch values into a single `ListAddAsync` or `ListRemoveAsync` call. Lists over 32 items are stored as immutable snapshots, so a small change copies only the parts it touches, with or without a size calculator. Cloning and raw dictionary access (`GetAsync>`) can still require copies proportional to the list size. Benchmark representative list sizes and read/write patterns before choosing these options; atomic updates do not make every workload allocation-free. +Batch values into a single `ListAddAsync` or `ListRemoveAsync` call. Lists over 32 items are stored as immutable snapshots, so a small change copies only the parts it touches, with or without a size calculator. With `CloneValues` on, that applies to lists of numbers, strings and other value types; lists of reference types are copied on each write. Raw dictionary access (`GetAsync>`) materializes a copy the first time it's read. Benchmark representative list sizes and read/write patterns before choosing these options; atomic updates do not make every workload allocation-free. A custom `SizeCalculator` receives list values as a read-only `IDictionary` (item to expiration), not a concrete `Dictionary`. Check for the interface, not the concrete type, and don't modify it. ::: From 9e52c84ede3d2f96d8170d67b3c62776d1e27419 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 7 Oct 2026 16:27:46 -0500 Subject: [PATCH 6/7] Test that a lost compare-and-swap retries under the stripe lock on each target's lock type --- src/Foundatio/Caching/InMemoryCacheClient.cs | 12 ++++++++- .../Caching/InMemoryCacheClientTests.cs | 27 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/Foundatio/Caching/InMemoryCacheClient.cs b/src/Foundatio/Caching/InMemoryCacheClient.cs index da743dafd..f805d9642 100644 --- a/src/Foundatio/Caching/InMemoryCacheClient.cs +++ b/src/Foundatio/Caching/InMemoryCacheClient.cs @@ -395,7 +395,7 @@ private TResult UpdateStoredEntry(string key, Func(string key, Func _conflictLocks[key.GetHashCode() & (ConflictLockCount - 1)]; + + internal bool IsConflictLockHeld(string key) => GetConflictLock(key).IsHeldByCurrentThread; +#else + private object GetConflictLock(string key) => _conflictLocks[key.GetHashCode() & (ConflictLockCount - 1)]; + + internal bool IsConflictLockHeld(string key) => Monitor.IsEntered(GetConflictLock(key)); +#endif + private static T[] CreateConflictLocks() where T : new() { var locks = new T[ConflictLockCount]; diff --git a/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs b/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs index 006d04f4b..64d55930f 100644 --- a/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs +++ b/tests/Foundatio.Tests/Caching/InMemoryCacheClientTests.cs @@ -582,6 +582,33 @@ await Parallel.ForEachAsync(Enumerable.Range(0, 4), TestCancellationToken, async Assert.Equal(original, remaining.OrderBy(value => value)); } + [Fact] + public async Task UpdateEntry_WithSerializeAfterConflict_RetriesUnderConflictLock() + { + // Arrange + // Runs on every target framework, so both System.Threading.Lock (net9+) and Monitor (net8) are exercised + var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow); + using var cache = new InMemoryCacheClient(o => o.TimeProvider(timeProvider).CloneValues(false).LoggerFactory(Log)); + await cache.ListAddAsync("set", Enumerable.Range(0, 100)); + var heldDuringAttempt = new List(); + int attempt = 0; + + // Act + long result = cache.UpdateEntry(key: "set", current => + { + heldDuringAttempt.Add(cache.IsConflictLockHeld("set")); + if (attempt++ == 0) + Publish(cache, "set", CreateEntry("concurrent", timeProvider)); + return (CreateEntry("mine", timeProvider), 1L); + }, serializeAfterConflict: true); + + // Assert + Assert.Equal(1, result); + Assert.Equal([false, true], heldDuringAttempt); + Assert.False(cache.IsConflictLockHeld("set")); + Assert.Equal("mine", (await cache.GetAsync("set")).Value); + } + [Fact] public override Task ListAddAsync_WithConcurrentRequests_DoesNotLoseValues() { From 2be63b1a1fa322d9e4b17b1a9b19ee60230cedb2 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Wed, 7 Oct 2026 16:55:43 -0500 Subject: [PATCH 7/7] Copy indexed lists for custom size calculators without the pair iterator --- src/Foundatio/Caching/InMemoryListSnapshot.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Foundatio/Caching/InMemoryListSnapshot.cs b/src/Foundatio/Caching/InMemoryListSnapshot.cs index 30d71c173..c16e743c3 100644 --- a/src/Foundatio/Caching/InMemoryListSnapshot.cs +++ b/src/Foundatio/Caching/InMemoryListSnapshot.cs @@ -94,8 +94,10 @@ private ListSnapshot(Layout layout, IEqualityComparer comparer, int count) return new Dictionary(values, _comparer); var dictionary = new Dictionary(Count, _comparer); - foreach (var pair in EnumeratePairs()) - dictionary.Add(pair.Key, pair.Value); + foreach (var block in _layout!.Blocks) + foreach (var item in block.Items) + if (item.Present) + dictionary.Add(item.Key, FromUtcTicks(item.ExpiresAtUtcTicks)); return dictionary; }