From bb0d88f27b4ca329733c8d9512bd3ec018db5d38 Mon Sep 17 00:00:00 2001 From: Arad Alvand Date: Thu, 19 Nov 2026 21:14:23 +0330 Subject: [PATCH] Pub/Sub: don't orphan a subscribe that races the last unsubscribe of the same channel Subscribing (looking up the channel's subscription, then adding the handler) and unsubscribing the last handler (removing it, removing the now-empty subscription from the multiplexer, then unsubscribing from the server) were not atomic with respect to each other. Interleaved, the handler could be added to a subscription that was being removed: nothing routes messages to it anymore, and the server is told to stop sending them; yet the subscribe completes successfully, so the handler silently never fires. A per-subscription registration lock now makes adding a handler and retiring the subscription mutually exclusive; a subscribe that finds the subscription retired looks up a fresh one instead. The retiring side also issues the unsubscribe command to the server (and only then removes the subscription from the multiplexer) while holding the lock, so that the racing subscribe's command is issued after it, rather than being able to overtake it and leave the server unsubscribed while the client considers itself subscribed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XUNs4qcAQaGCRpkwmr8LnY --- src/StackExchange.Redis/RedisSubscriber.cs | 133 +++++++++++++----- src/StackExchange.Redis/Subscription.cs | 44 +++++- .../SubscriptionRegistrationRaceUnitTests.cs | 58 ++++++++ 3 files changed, 196 insertions(+), 39 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs diff --git a/src/StackExchange.Redis/RedisSubscriber.cs b/src/StackExchange.Redis/RedisSubscriber.cs index d0741e5f5..c6c1998a1 100644 --- a/src/StackExchange.Redis/RedisSubscriber.cs +++ b/src/StackExchange.Redis/RedisSubscriber.cs @@ -32,6 +32,22 @@ internal Subscription GetOrAddSubscription(in RedisChannel channel, CommandFlags return sub; } } + + /// + /// Adds a handler and/or queue to the channel's subscription, creating one if there is none; never to one that + /// is being retired by a concurrent unsubscribe, see . + /// + internal Subscription AddToSubscription(in RedisChannel channel, CommandFlags flags, Action? handler, ChannelMessageQueue? queue) + { + while (true) + { + var sub = GetOrAddSubscription(channel, flags); + if (sub.TryAdd(handler, queue)) return sub; + + // it was retired (and so removed) concurrently; looking it up again yields a fresh one + } + } + internal bool TryGetSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) => subscriptions.TryGetValue(channel, out sub); internal bool TryRemoveSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) { @@ -248,8 +264,7 @@ private int Subscribe(RedisChannel channel, Action? ha ThrowIfNull(channel); if (handler == null && queue == null) { return 0; } - var sub = multiplexer.GetOrAddSubscription(channel, flags); - sub.Add(handler, queue); + var sub = multiplexer.AddToSubscription(channel, flags, handler, queue); return sub.EnsureSubscribedToServer(this, channel, flags, false); } @@ -291,8 +306,7 @@ private Task SubscribeAsync(RedisChannel channel, Action.Default(null); } - var sub = multiplexer.GetOrAddSubscription(channel, flags); - sub.Add(handler, queue); + var sub = multiplexer.AddToSubscription(channel, flags, handler, queue); return sub.EnsureSubscribedToServerAsync(this, channel, flags, false, server); } @@ -304,11 +318,17 @@ void ISubscriber.Unsubscribe(RedisChannel channel, Action? handler, ChannelMessageQueue? queue, CommandFlags flags) { ThrowIfNull(channel); - // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server - // ReSharper disable once SimplifyConditionalTernaryExpression - return UnregisterSubscription(channel, handler, queue, out var sub) - ? sub.UnsubscribeFromServer(this, channel, flags, false) - : true; + if (!multiplexer.TryGetSubscription(channel, out var sub)) return true; + + // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server; + // the latter while still holding the lock, see RetireIfEmpty + lock (sub.RegistrationLock) + { + // ReSharper disable once SimplifyConditionalTernaryExpression + return RetireIfEmpty(sub, handler, queue) + ? UnsubscribeFromServerThenRemove(channel, sub, flags) + : true; + } } Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action? handler, CommandFlags flags) @@ -317,36 +337,69 @@ Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action UnsubscribeAsync(in RedisChannel channel, Action? handler, ChannelMessageQueue? queue, CommandFlags flags) { ThrowIfNull(channel); - // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server - return UnregisterSubscription(channel, handler, queue, out var sub) - ? sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false) - : CompletedTask.Default(asyncState); + if (!multiplexer.TryGetSubscription(channel, out var sub)) return CompletedTask.Default(asyncState); + + // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server; + // the latter is issued (though not awaited) while still holding the lock, see RetireIfEmpty + lock (sub.RegistrationLock) + { + return RetireIfEmpty(sub, handler, queue) + ? UnsubscribeFromServerThenRemoveAsync(channel, sub, flags) + : CompletedTask.Default(asyncState); + } } /// - /// Unregisters a handler or queue and returns if we should remove it from the server. + /// Unregisters a handler or queue, and retires the subscription if that was its last one; must be called while + /// holding the subscription's . If it returns , the + /// caller must then - still holding the lock - issue the unsubscribe command to the server, and only *then* remove + /// the subscription from the multiplexer (see ). That way, a concurrent + /// subscribe to the same channel still finds this subscription until the unsubscribe command has been issued, and + /// so has to wait for the lock (only to find it retired, and to look up a fresh one); so its subscribe command is + /// issued after the unsubscribe command, rather than being able to overtake it and leave the server unsubscribed. /// /// if we should remove the subscription from the server, otherwise. - private bool UnregisterSubscription(in RedisChannel channel, Action? handler, ChannelMessageQueue? queue, [NotNullWhen(true)] out Subscription? sub) + private static bool RetireIfEmpty(Subscription sub, Action? handler, ChannelMessageQueue? queue) { - ThrowIfNull(channel); - if (multiplexer.TryGetSubscription(channel, out sub)) + if (sub.IsRetired) return false; // already retired (and removed) concurrently + + if ((handler == null) & (queue == null)) { - if ((handler == null) & (queue == null)) - { - // This was a blanket wipe, so clear it completely - sub.MarkCompleted(); - multiplexer.TryRemoveSubscription(channel, out _); - return true; - } - else if (sub.Remove(handler, queue)) - { - // Or this was the last handler and/or queue, which also means unsubscribe - multiplexer.TryRemoveSubscription(channel, out _); - return true; - } + // This was a blanket wipe, so clear it completely + sub.MarkCompleted(); + } + else if (!sub.Remove(handler, queue)) + { + // Not the last handler and/or queue, so the subscription stays + return false; + } + + sub.MarkRetired(); + return true; + } + + private bool UnsubscribeFromServerThenRemove(in RedisChannel channel, Subscription sub, CommandFlags flags) + { + try + { + return sub.UnsubscribeFromServer(this, channel, flags, false); + } + finally + { + multiplexer.TryRemoveSubscription(channel, out _); + } + } + + private Task UnsubscribeFromServerThenRemoveAsync(in RedisChannel channel, Subscription sub, CommandFlags flags) + { + try + { + return sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false); + } + finally + { + multiplexer.TryRemoveSubscription(channel, out _); } - return false; } // TODO: We need a new api to support SUNSUBSCRIBE all. Calling this now would unsubscribe both sharded and unsharded channels. @@ -356,10 +409,13 @@ public void UnsubscribeAll(CommandFlags flags = CommandFlags.None) var subs = multiplexer.GetSubscriptions(); foreach (var pair in subs) { - if (subs.TryRemove(pair.Key, out var sub)) + var sub = pair.Value; + lock (sub.RegistrationLock) { - sub.MarkCompleted(); - sub.UnsubscribeFromServer(this, pair.Key, flags, false); + if (RetireIfEmpty(sub, null, null)) + { + UnsubscribeFromServerThenRemove(pair.Key, sub, flags); + } } } } @@ -371,10 +427,13 @@ public Task UnsubscribeAllAsync(CommandFlags flags = CommandFlags.None) var subs = multiplexer.GetSubscriptions(); foreach (var pair in subs) { - if (subs.TryRemove(pair.Key, out var sub)) + var sub = pair.Value; + lock (sub.RegistrationLock) { - sub.MarkCompleted(); - last = sub.UnsubscribeFromServerAsync(this, pair.Key, flags, asyncState, false); + if (RetireIfEmpty(sub, null, null)) + { + last = UnsubscribeFromServerThenRemoveAsync(pair.Key, sub, flags); + } } } return last ?? CompletedTask.Default(asyncState); diff --git a/src/StackExchange.Redis/Subscription.cs b/src/StackExchange.Redis/Subscription.cs index 767ab0a14..a23f10ef5 100644 --- a/src/StackExchange.Redis/Subscription.cs +++ b/src/StackExchange.Redis/Subscription.cs @@ -1,6 +1,7 @@ using System; using System.Buffers; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -19,6 +20,38 @@ internal abstract class Subscription private Action? _handlers; private readonly object _handlersLock = new(); private ChannelMessageQueue? _queues; + + // Guards this subscription's registration: handlers/queues being added to it, versus it being retired once its + // last one is removed (at which point it is also removed from the multiplexer, and unsubscribed from the server). + // Without it, a handler could be added to a subscription that is concurrently being retired; that handler would + // then never be invoked, since nothing routes messages to a retired subscription (and the server has been told + // to stop sending them). + private readonly object _registrationLock = new(); + private bool _isRetired; + + /// + /// Held while retiring this subscription, see . + /// + internal object RegistrationLock => _registrationLock; + + internal bool IsRetired => _isRetired; // only meaningful while holding RegistrationLock + + internal void MarkRetired() => _isRetired = true; // only to be called while holding RegistrationLock + + /// + /// Adds a handler and/or queue, unless this subscription has been retired (in which case the caller should + /// look it up again, which yields a fresh one). + /// + internal bool TryAdd(Action? handler, ChannelMessageQueue? queue) + { + lock (_registrationLock) + { + if (_isRetired) return false; + Add(handler, queue); + return true; + } + } + public CommandFlags Flags { get; } public ResultProcessor.TrackSubscriptionsProcessor Processor { get; } @@ -506,11 +539,18 @@ internal override async Task UnsubscribeFromServerAsync( object? asyncState, bool internalCall) { - bool any = false; + // issue all of them before awaiting any, so that they are all issued synchronously (see RegistrationLock) + var pending = new List>(); foreach (var server in _servers) { var message = GetSubscriptionMessage(channel, SubscriptionAction.Unsubscribe, flags, internalCall); - any |= await subscriber.ExecuteAsync(message, Processor, server.Value).ForAwait(); + pending.Add(subscriber.ExecuteAsync(message, Processor, server.Value)); + } + + bool any = false; + foreach (var task in pending) + { + any |= await task.ForAwait(); } return any; diff --git a/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs b/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs new file mode 100644 index 000000000..752602584 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs @@ -0,0 +1,58 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class SubscriptionRegistrationRaceUnitTests(ITestOutputHelper output) +{ + /// + /// Removing a channel's last handler retires its subscription (locally, and on the server); a handler added to the + /// same channel concurrently must end up on a live subscription, rather than on the retired one - which nothing + /// routes messages to anymore, and which the server is unsubscribed from. + /// + [Fact] + public async Task SubscribeRacingLastUnsubscribeStillReceivesMessages() + { + using var server = new InProcessTestServer(output); + await using var conn = await ConnectionMultiplexer.ConnectAsync(server.GetClientConfig()); + var subscriber = conn.GetSubscriber(); + + // the race is narrow, so it takes a number of attempts to provoke it; each uses a fresh channel + const int Attempts = 300; + var missed = 0; + for (var i = 0; i < Attempts; i++) + { + var channel = RedisChannel.Literal($"{nameof(SubscribeRacingLastUnsubscribeStillReceivesMessages)}:{i}"); + Action leaving = static (_, _) => { }; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Action joining = (_, _) => received.TrySetResult(true); + + await subscriber.SubscribeAsync(channel, leaving); + + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var unsubscribe = Task.Run(async () => + { + await start.Task; + await subscriber.UnsubscribeAsync(channel, leaving); + }); + var subscribe = Task.Run(async () => + { + await start.Task; + await subscriber.SubscribeAsync(channel, joining); + }); + start.SetResult(true); + await Task.WhenAll(unsubscribe, subscribe); + + await subscriber.PublishAsync(channel, "hello"); + if (await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(2))) != received.Task) + { + missed++; + } + + await subscriber.UnsubscribeAsync(channel, joining); + } + + Assert.Equal(0, missed); + } +}