diff --git a/src/StackExchange.Redis/RedisSubscriber.cs b/src/StackExchange.Redis/RedisSubscriber.cs index d0741e5f5..c6c1998a1 100644 --- a/src/StackExchange.Redis/RedisSubscriber.cs +++ b/src/StackExchange.Redis/RedisSubscriber.cs @@ -32,6 +32,22 @@ internal Subscription GetOrAddSubscription(in RedisChannel channel, CommandFlags return sub; } } + + /// + /// Adds a handler and/or queue to the channel's subscription, creating one if there is none; never to one that + /// is being retired by a concurrent unsubscribe, see . + /// + internal Subscription AddToSubscription(in RedisChannel channel, CommandFlags flags, Action? handler, ChannelMessageQueue? queue) + { + while (true) + { + var sub = GetOrAddSubscription(channel, flags); + if (sub.TryAdd(handler, queue)) return sub; + + // it was retired (and so removed) concurrently; looking it up again yields a fresh one + } + } + internal bool TryGetSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) => subscriptions.TryGetValue(channel, out sub); internal bool TryRemoveSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) { @@ -248,8 +264,7 @@ private int Subscribe(RedisChannel channel, Action? ha ThrowIfNull(channel); if (handler == null && queue == null) { return 0; } - var sub = multiplexer.GetOrAddSubscription(channel, flags); - sub.Add(handler, queue); + var sub = multiplexer.AddToSubscription(channel, flags, handler, queue); return sub.EnsureSubscribedToServer(this, channel, flags, false); } @@ -291,8 +306,7 @@ private Task SubscribeAsync(RedisChannel channel, Action.Default(null); } - var sub = multiplexer.GetOrAddSubscription(channel, flags); - sub.Add(handler, queue); + var sub = multiplexer.AddToSubscription(channel, flags, handler, queue); return sub.EnsureSubscribedToServerAsync(this, channel, flags, false, server); } @@ -304,11 +318,17 @@ void ISubscriber.Unsubscribe(RedisChannel channel, Action? handler, ChannelMessageQueue? queue, CommandFlags flags) { ThrowIfNull(channel); - // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server - // ReSharper disable once SimplifyConditionalTernaryExpression - return UnregisterSubscription(channel, handler, queue, out var sub) - ? sub.UnsubscribeFromServer(this, channel, flags, false) - : true; + if (!multiplexer.TryGetSubscription(channel, out var sub)) return true; + + // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server; + // the latter while still holding the lock, see RetireIfEmpty + lock (sub.RegistrationLock) + { + // ReSharper disable once SimplifyConditionalTernaryExpression + return RetireIfEmpty(sub, handler, queue) + ? UnsubscribeFromServerThenRemove(channel, sub, flags) + : true; + } } Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action? handler, CommandFlags flags) @@ -317,36 +337,69 @@ Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action UnsubscribeAsync(in RedisChannel channel, Action? handler, ChannelMessageQueue? queue, CommandFlags flags) { ThrowIfNull(channel); - // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server - return UnregisterSubscription(channel, handler, queue, out var sub) - ? sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false) - : CompletedTask.Default(asyncState); + if (!multiplexer.TryGetSubscription(channel, out var sub)) return CompletedTask.Default(asyncState); + + // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server; + // the latter is issued (though not awaited) while still holding the lock, see RetireIfEmpty + lock (sub.RegistrationLock) + { + return RetireIfEmpty(sub, handler, queue) + ? UnsubscribeFromServerThenRemoveAsync(channel, sub, flags) + : CompletedTask.Default(asyncState); + } } /// - /// Unregisters a handler or queue and returns if we should remove it from the server. + /// Unregisters a handler or queue, and retires the subscription if that was its last one; must be called while + /// holding the subscription's . If it returns , the + /// caller must then - still holding the lock - issue the unsubscribe command to the server, and only *then* remove + /// the subscription from the multiplexer (see ). That way, a concurrent + /// subscribe to the same channel still finds this subscription until the unsubscribe command has been issued, and + /// so has to wait for the lock (only to find it retired, and to look up a fresh one); so its subscribe command is + /// issued after the unsubscribe command, rather than being able to overtake it and leave the server unsubscribed. /// /// if we should remove the subscription from the server, otherwise. - private bool UnregisterSubscription(in RedisChannel channel, Action? handler, ChannelMessageQueue? queue, [NotNullWhen(true)] out Subscription? sub) + private static bool RetireIfEmpty(Subscription sub, Action? handler, ChannelMessageQueue? queue) { - ThrowIfNull(channel); - if (multiplexer.TryGetSubscription(channel, out sub)) + if (sub.IsRetired) return false; // already retired (and removed) concurrently + + if ((handler == null) & (queue == null)) { - if ((handler == null) & (queue == null)) - { - // This was a blanket wipe, so clear it completely - sub.MarkCompleted(); - multiplexer.TryRemoveSubscription(channel, out _); - return true; - } - else if (sub.Remove(handler, queue)) - { - // Or this was the last handler and/or queue, which also means unsubscribe - multiplexer.TryRemoveSubscription(channel, out _); - return true; - } + // This was a blanket wipe, so clear it completely + sub.MarkCompleted(); + } + else if (!sub.Remove(handler, queue)) + { + // Not the last handler and/or queue, so the subscription stays + return false; + } + + sub.MarkRetired(); + return true; + } + + private bool UnsubscribeFromServerThenRemove(in RedisChannel channel, Subscription sub, CommandFlags flags) + { + try + { + return sub.UnsubscribeFromServer(this, channel, flags, false); + } + finally + { + multiplexer.TryRemoveSubscription(channel, out _); + } + } + + private Task UnsubscribeFromServerThenRemoveAsync(in RedisChannel channel, Subscription sub, CommandFlags flags) + { + try + { + return sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false); + } + finally + { + multiplexer.TryRemoveSubscription(channel, out _); } - return false; } // TODO: We need a new api to support SUNSUBSCRIBE all. Calling this now would unsubscribe both sharded and unsharded channels. @@ -356,10 +409,13 @@ public void UnsubscribeAll(CommandFlags flags = CommandFlags.None) var subs = multiplexer.GetSubscriptions(); foreach (var pair in subs) { - if (subs.TryRemove(pair.Key, out var sub)) + var sub = pair.Value; + lock (sub.RegistrationLock) { - sub.MarkCompleted(); - sub.UnsubscribeFromServer(this, pair.Key, flags, false); + if (RetireIfEmpty(sub, null, null)) + { + UnsubscribeFromServerThenRemove(pair.Key, sub, flags); + } } } } @@ -371,10 +427,13 @@ public Task UnsubscribeAllAsync(CommandFlags flags = CommandFlags.None) var subs = multiplexer.GetSubscriptions(); foreach (var pair in subs) { - if (subs.TryRemove(pair.Key, out var sub)) + var sub = pair.Value; + lock (sub.RegistrationLock) { - sub.MarkCompleted(); - last = sub.UnsubscribeFromServerAsync(this, pair.Key, flags, asyncState, false); + if (RetireIfEmpty(sub, null, null)) + { + last = UnsubscribeFromServerThenRemoveAsync(pair.Key, sub, flags); + } } } return last ?? CompletedTask.Default(asyncState); diff --git a/src/StackExchange.Redis/Subscription.cs b/src/StackExchange.Redis/Subscription.cs index 767ab0a14..a23f10ef5 100644 --- a/src/StackExchange.Redis/Subscription.cs +++ b/src/StackExchange.Redis/Subscription.cs @@ -1,6 +1,7 @@ using System; using System.Buffers; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -19,6 +20,38 @@ internal abstract class Subscription private Action? _handlers; private readonly object _handlersLock = new(); private ChannelMessageQueue? _queues; + + // Guards this subscription's registration: handlers/queues being added to it, versus it being retired once its + // last one is removed (at which point it is also removed from the multiplexer, and unsubscribed from the server). + // Without it, a handler could be added to a subscription that is concurrently being retired; that handler would + // then never be invoked, since nothing routes messages to a retired subscription (and the server has been told + // to stop sending them). + private readonly object _registrationLock = new(); + private bool _isRetired; + + /// + /// Held while retiring this subscription, see . + /// + internal object RegistrationLock => _registrationLock; + + internal bool IsRetired => _isRetired; // only meaningful while holding RegistrationLock + + internal void MarkRetired() => _isRetired = true; // only to be called while holding RegistrationLock + + /// + /// Adds a handler and/or queue, unless this subscription has been retired (in which case the caller should + /// look it up again, which yields a fresh one). + /// + internal bool TryAdd(Action? handler, ChannelMessageQueue? queue) + { + lock (_registrationLock) + { + if (_isRetired) return false; + Add(handler, queue); + return true; + } + } + public CommandFlags Flags { get; } public ResultProcessor.TrackSubscriptionsProcessor Processor { get; } @@ -506,11 +539,18 @@ internal override async Task UnsubscribeFromServerAsync( object? asyncState, bool internalCall) { - bool any = false; + // issue all of them before awaiting any, so that they are all issued synchronously (see RegistrationLock) + var pending = new List>(); foreach (var server in _servers) { var message = GetSubscriptionMessage(channel, SubscriptionAction.Unsubscribe, flags, internalCall); - any |= await subscriber.ExecuteAsync(message, Processor, server.Value).ForAwait(); + pending.Add(subscriber.ExecuteAsync(message, Processor, server.Value)); + } + + bool any = false; + foreach (var task in pending) + { + any |= await task.ForAwait(); } return any; diff --git a/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs b/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs new file mode 100644 index 000000000..752602584 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs @@ -0,0 +1,58 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class SubscriptionRegistrationRaceUnitTests(ITestOutputHelper output) +{ + /// + /// Removing a channel's last handler retires its subscription (locally, and on the server); a handler added to the + /// same channel concurrently must end up on a live subscription, rather than on the retired one - which nothing + /// routes messages to anymore, and which the server is unsubscribed from. + /// + [Fact] + public async Task SubscribeRacingLastUnsubscribeStillReceivesMessages() + { + using var server = new InProcessTestServer(output); + await using var conn = await ConnectionMultiplexer.ConnectAsync(server.GetClientConfig()); + var subscriber = conn.GetSubscriber(); + + // the race is narrow, so it takes a number of attempts to provoke it; each uses a fresh channel + const int Attempts = 300; + var missed = 0; + for (var i = 0; i < Attempts; i++) + { + var channel = RedisChannel.Literal($"{nameof(SubscribeRacingLastUnsubscribeStillReceivesMessages)}:{i}"); + Action leaving = static (_, _) => { }; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Action joining = (_, _) => received.TrySetResult(true); + + await subscriber.SubscribeAsync(channel, leaving); + + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var unsubscribe = Task.Run(async () => + { + await start.Task; + await subscriber.UnsubscribeAsync(channel, leaving); + }); + var subscribe = Task.Run(async () => + { + await start.Task; + await subscriber.SubscribeAsync(channel, joining); + }); + start.SetResult(true); + await Task.WhenAll(unsubscribe, subscribe); + + await subscriber.PublishAsync(channel, "hello"); + if (await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(2))) != received.Task) + { + missed++; + } + + await subscriber.UnsubscribeAsync(channel, joining); + } + + Assert.Equal(0, missed); + } +}