From bb0d88f27b4ca329733c8d9512bd3ec018db5d38 Mon Sep 17 00:00:00 2001 From: Arad Alvand Date: Thu, 19 Nov 2026 21:14:23 +0330 Subject: [PATCH 1/2] Pub/Sub: don't orphan a subscribe that races the last unsubscribe of the same channel Subscribing (looking up the channel's subscription, then adding the handler) and unsubscribing the last handler (removing it, removing the now-empty subscription from the multiplexer, then unsubscribing from the server) were not atomic with respect to each other. Interleaved, the handler could be added to a subscription that was being removed: nothing routes messages to it anymore, and the server is told to stop sending them; yet the subscribe completes successfully, so the handler silently never fires. A per-subscription registration lock now makes adding a handler and retiring the subscription mutually exclusive; a subscribe that finds the subscription retired looks up a fresh one instead. The retiring side also issues the unsubscribe command to the server (and only then removes the subscription from the multiplexer) while holding the lock, so that the racing subscribe's command is issued after it, rather than being able to overtake it and leave the server unsubscribed while the client considers itself subscribed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XUNs4qcAQaGCRpkwmr8LnY --- src/StackExchange.Redis/RedisSubscriber.cs | 133 +++++++++++++----- src/StackExchange.Redis/Subscription.cs | 44 +++++- .../SubscriptionRegistrationRaceUnitTests.cs | 58 ++++++++ 3 files changed, 196 insertions(+), 39 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs diff --git a/src/StackExchange.Redis/RedisSubscriber.cs b/src/StackExchange.Redis/RedisSubscriber.cs index d0741e5f5..c6c1998a1 100644 --- a/src/StackExchange.Redis/RedisSubscriber.cs +++ b/src/StackExchange.Redis/RedisSubscriber.cs @@ -32,6 +32,22 @@ internal Subscription GetOrAddSubscription(in RedisChannel channel, CommandFlags return sub; } } + + /// + /// Adds a handler and/or queue to the channel's subscription, creating one if there is none; never to one that + /// is being retired by a concurrent unsubscribe, see . + /// + internal Subscription AddToSubscription(in RedisChannel channel, CommandFlags flags, Action? handler, ChannelMessageQueue? queue) + { + while (true) + { + var sub = GetOrAddSubscription(channel, flags); + if (sub.TryAdd(handler, queue)) return sub; + + // it was retired (and so removed) concurrently; looking it up again yields a fresh one + } + } + internal bool TryGetSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) => subscriptions.TryGetValue(channel, out sub); internal bool TryRemoveSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) { @@ -248,8 +264,7 @@ private int Subscribe(RedisChannel channel, Action? ha ThrowIfNull(channel); if (handler == null && queue == null) { return 0; } - var sub = multiplexer.GetOrAddSubscription(channel, flags); - sub.Add(handler, queue); + var sub = multiplexer.AddToSubscription(channel, flags, handler, queue); return sub.EnsureSubscribedToServer(this, channel, flags, false); } @@ -291,8 +306,7 @@ private Task SubscribeAsync(RedisChannel channel, Action.Default(null); } - var sub = multiplexer.GetOrAddSubscription(channel, flags); - sub.Add(handler, queue); + var sub = multiplexer.AddToSubscription(channel, flags, handler, queue); return sub.EnsureSubscribedToServerAsync(this, channel, flags, false, server); } @@ -304,11 +318,17 @@ void ISubscriber.Unsubscribe(RedisChannel channel, Action? handler, ChannelMessageQueue? queue, CommandFlags flags) { ThrowIfNull(channel); - // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server - // ReSharper disable once SimplifyConditionalTernaryExpression - return UnregisterSubscription(channel, handler, queue, out var sub) - ? sub.UnsubscribeFromServer(this, channel, flags, false) - : true; + if (!multiplexer.TryGetSubscription(channel, out var sub)) return true; + + // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server; + // the latter while still holding the lock, see RetireIfEmpty + lock (sub.RegistrationLock) + { + // ReSharper disable once SimplifyConditionalTernaryExpression + return RetireIfEmpty(sub, handler, queue) + ? UnsubscribeFromServerThenRemove(channel, sub, flags) + : true; + } } Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action? handler, CommandFlags flags) @@ -317,36 +337,69 @@ Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action UnsubscribeAsync(in RedisChannel channel, Action? handler, ChannelMessageQueue? queue, CommandFlags flags) { ThrowIfNull(channel); - // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server - return UnregisterSubscription(channel, handler, queue, out var sub) - ? sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false) - : CompletedTask.Default(asyncState); + if (!multiplexer.TryGetSubscription(channel, out var sub)) return CompletedTask.Default(asyncState); + + // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server; + // the latter is issued (though not awaited) while still holding the lock, see RetireIfEmpty + lock (sub.RegistrationLock) + { + return RetireIfEmpty(sub, handler, queue) + ? UnsubscribeFromServerThenRemoveAsync(channel, sub, flags) + : CompletedTask.Default(asyncState); + } } /// - /// Unregisters a handler or queue and returns if we should remove it from the server. + /// Unregisters a handler or queue, and retires the subscription if that was its last one; must be called while + /// holding the subscription's . If it returns , the + /// caller must then - still holding the lock - issue the unsubscribe command to the server, and only *then* remove + /// the subscription from the multiplexer (see ). That way, a concurrent + /// subscribe to the same channel still finds this subscription until the unsubscribe command has been issued, and + /// so has to wait for the lock (only to find it retired, and to look up a fresh one); so its subscribe command is + /// issued after the unsubscribe command, rather than being able to overtake it and leave the server unsubscribed. /// /// if we should remove the subscription from the server, otherwise. - private bool UnregisterSubscription(in RedisChannel channel, Action? handler, ChannelMessageQueue? queue, [NotNullWhen(true)] out Subscription? sub) + private static bool RetireIfEmpty(Subscription sub, Action? handler, ChannelMessageQueue? queue) { - ThrowIfNull(channel); - if (multiplexer.TryGetSubscription(channel, out sub)) + if (sub.IsRetired) return false; // already retired (and removed) concurrently + + if ((handler == null) & (queue == null)) { - if ((handler == null) & (queue == null)) - { - // This was a blanket wipe, so clear it completely - sub.MarkCompleted(); - multiplexer.TryRemoveSubscription(channel, out _); - return true; - } - else if (sub.Remove(handler, queue)) - { - // Or this was the last handler and/or queue, which also means unsubscribe - multiplexer.TryRemoveSubscription(channel, out _); - return true; - } + // This was a blanket wipe, so clear it completely + sub.MarkCompleted(); + } + else if (!sub.Remove(handler, queue)) + { + // Not the last handler and/or queue, so the subscription stays + return false; + } + + sub.MarkRetired(); + return true; + } + + private bool UnsubscribeFromServerThenRemove(in RedisChannel channel, Subscription sub, CommandFlags flags) + { + try + { + return sub.UnsubscribeFromServer(this, channel, flags, false); + } + finally + { + multiplexer.TryRemoveSubscription(channel, out _); + } + } + + private Task UnsubscribeFromServerThenRemoveAsync(in RedisChannel channel, Subscription sub, CommandFlags flags) + { + try + { + return sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false); + } + finally + { + multiplexer.TryRemoveSubscription(channel, out _); } - return false; } // TODO: We need a new api to support SUNSUBSCRIBE all. Calling this now would unsubscribe both sharded and unsharded channels. @@ -356,10 +409,13 @@ public void UnsubscribeAll(CommandFlags flags = CommandFlags.None) var subs = multiplexer.GetSubscriptions(); foreach (var pair in subs) { - if (subs.TryRemove(pair.Key, out var sub)) + var sub = pair.Value; + lock (sub.RegistrationLock) { - sub.MarkCompleted(); - sub.UnsubscribeFromServer(this, pair.Key, flags, false); + if (RetireIfEmpty(sub, null, null)) + { + UnsubscribeFromServerThenRemove(pair.Key, sub, flags); + } } } } @@ -371,10 +427,13 @@ public Task UnsubscribeAllAsync(CommandFlags flags = CommandFlags.None) var subs = multiplexer.GetSubscriptions(); foreach (var pair in subs) { - if (subs.TryRemove(pair.Key, out var sub)) + var sub = pair.Value; + lock (sub.RegistrationLock) { - sub.MarkCompleted(); - last = sub.UnsubscribeFromServerAsync(this, pair.Key, flags, asyncState, false); + if (RetireIfEmpty(sub, null, null)) + { + last = UnsubscribeFromServerThenRemoveAsync(pair.Key, sub, flags); + } } } return last ?? CompletedTask.Default(asyncState); diff --git a/src/StackExchange.Redis/Subscription.cs b/src/StackExchange.Redis/Subscription.cs index 767ab0a14..a23f10ef5 100644 --- a/src/StackExchange.Redis/Subscription.cs +++ b/src/StackExchange.Redis/Subscription.cs @@ -1,6 +1,7 @@ using System; using System.Buffers; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -19,6 +20,38 @@ internal abstract class Subscription private Action? _handlers; private readonly object _handlersLock = new(); private ChannelMessageQueue? _queues; + + // Guards this subscription's registration: handlers/queues being added to it, versus it being retired once its + // last one is removed (at which point it is also removed from the multiplexer, and unsubscribed from the server). + // Without it, a handler could be added to a subscription that is concurrently being retired; that handler would + // then never be invoked, since nothing routes messages to a retired subscription (and the server has been told + // to stop sending them). + private readonly object _registrationLock = new(); + private bool _isRetired; + + /// + /// Held while retiring this subscription, see . + /// + internal object RegistrationLock => _registrationLock; + + internal bool IsRetired => _isRetired; // only meaningful while holding RegistrationLock + + internal void MarkRetired() => _isRetired = true; // only to be called while holding RegistrationLock + + /// + /// Adds a handler and/or queue, unless this subscription has been retired (in which case the caller should + /// look it up again, which yields a fresh one). + /// + internal bool TryAdd(Action? handler, ChannelMessageQueue? queue) + { + lock (_registrationLock) + { + if (_isRetired) return false; + Add(handler, queue); + return true; + } + } + public CommandFlags Flags { get; } public ResultProcessor.TrackSubscriptionsProcessor Processor { get; } @@ -506,11 +539,18 @@ internal override async Task UnsubscribeFromServerAsync( object? asyncState, bool internalCall) { - bool any = false; + // issue all of them before awaiting any, so that they are all issued synchronously (see RegistrationLock) + var pending = new List>(); foreach (var server in _servers) { var message = GetSubscriptionMessage(channel, SubscriptionAction.Unsubscribe, flags, internalCall); - any |= await subscriber.ExecuteAsync(message, Processor, server.Value).ForAwait(); + pending.Add(subscriber.ExecuteAsync(message, Processor, server.Value)); + } + + bool any = false; + foreach (var task in pending) + { + any |= await task.ForAwait(); } return any; diff --git a/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs b/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs new file mode 100644 index 000000000..752602584 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs @@ -0,0 +1,58 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class SubscriptionRegistrationRaceUnitTests(ITestOutputHelper output) +{ + /// + /// Removing a channel's last handler retires its subscription (locally, and on the server); a handler added to the + /// same channel concurrently must end up on a live subscription, rather than on the retired one - which nothing + /// routes messages to anymore, and which the server is unsubscribed from. + /// + [Fact] + public async Task SubscribeRacingLastUnsubscribeStillReceivesMessages() + { + using var server = new InProcessTestServer(output); + await using var conn = await ConnectionMultiplexer.ConnectAsync(server.GetClientConfig()); + var subscriber = conn.GetSubscriber(); + + // the race is narrow, so it takes a number of attempts to provoke it; each uses a fresh channel + const int Attempts = 300; + var missed = 0; + for (var i = 0; i < Attempts; i++) + { + var channel = RedisChannel.Literal($"{nameof(SubscribeRacingLastUnsubscribeStillReceivesMessages)}:{i}"); + Action leaving = static (_, _) => { }; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Action joining = (_, _) => received.TrySetResult(true); + + await subscriber.SubscribeAsync(channel, leaving); + + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var unsubscribe = Task.Run(async () => + { + await start.Task; + await subscriber.UnsubscribeAsync(channel, leaving); + }); + var subscribe = Task.Run(async () => + { + await start.Task; + await subscriber.SubscribeAsync(channel, joining); + }); + start.SetResult(true); + await Task.WhenAll(unsubscribe, subscribe); + + await subscriber.PublishAsync(channel, "hello"); + if (await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(2))) != received.Task) + { + missed++; + } + + await subscriber.UnsubscribeAsync(channel, joining); + } + + Assert.Equal(0, missed); + } +} From ebe21901f254d9383e34da376cb3324c7b56ddae Mon Sep 17 00:00:00 2001 From: Arad Alvand Date: Fri, 20 Nov 2026 09:21:01 +0330 Subject: [PATCH 2/2] Pub/Sub: retry a last unsubscribe the server declined, rather than leaving it subscribed Unsubscribing a channel's last handler retires its subscription on the client whether or not the server-side unsubscribe succeeds. When the server declined to run it (e.g. a BUSY reply while a script runs), the server stayed subscribed, pushing messages that nothing receives, with nothing left on the client to unsubscribe - until the connection was replaced. Now such an unsubscribe is re-issued after a backoff, unless the channel has been subscribed to again by then (at which point client and server agree once more); each re-issue happens while holding the same lock that adding a subscription takes, so it is ordered before any subscribe that follows and can't undo a newer subscription. The failure is noticed where it can always be seen, rather than through the caller's task: TrackSubscriptionsProcessor sees every error reply - including those of fire-and-forget and synchronous unsubscribes - and the returned task is only observed for failures with no reply at all (the command never having been written). The caller still observes the original failure. Fixes #3367. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XUNs4qcAQaGCRpkwmr8LnY --- .../Availability/FaultContext.cs | 2 +- src/StackExchange.Redis/RedisSubscriber.cs | 99 +++++++++++++- src/StackExchange.Redis/ResultProcessor.cs | 20 +++ src/StackExchange.Redis/Subscription.cs | 9 ++ .../FailedUnsubscribeUnitTests.cs | 126 ++++++++++++++++++ 5 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/FailedUnsubscribeUnitTests.cs diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index 7ec643082..0aeb96f51 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -130,7 +130,7 @@ public FaultContext(Exception fault) /// public Maintenance.MaintenanceNotificationType MaintenanceType { get; } - private static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status) + internal static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status) { // the client never handed it to the socket, so the server cannot have seen it if (status is CommandStatus.WaitingToBeSent or CommandStatus.WaitingInBacklog) return true; diff --git a/src/StackExchange.Redis/RedisSubscriber.cs b/src/StackExchange.Redis/RedisSubscriber.cs index c6c1998a1..79394b29d 100644 --- a/src/StackExchange.Redis/RedisSubscriber.cs +++ b/src/StackExchange.Redis/RedisSubscriber.cs @@ -57,6 +57,96 @@ internal bool TryRemoveSubscription(in RedisChannel channel, [NotNullWhen(true)] } } + private static readonly TimeSpan UnsubscribeRetryInitialDelay = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan UnsubscribeRetryMaxDelay = TimeSpan.FromSeconds(5); + private const int UnsubscribeRetryMaxAttempts = 30; + + /// + /// Retiring a subscription removes it from the multiplexer whether or not the server-side unsubscribe succeeds. When + /// the server never ran that unsubscribe (e.g. it replied BUSY), it is still subscribed, and would keep pushing + /// messages that nothing receives, with nothing left on the client to unsubscribe; so this re-issues the unsubscribe, + /// after a backoff, unless the channel has been subscribed to again by then (at which point client and server agree + /// once more). It is called for each such failure - including those of the re-issued unsubscribes - from wherever the + /// failure is observed: for error replies (whether or not + /// anyone awaits the command, e.g. ), and + /// for failures with no reply at all (the command never having been written). + /// + internal void OnUnsubscribeNotApplied(in RedisChannel channel, Subscription retired, CommandFlags flags) + { + var attempt = retired.RecordDeclinedUnsubscribe(); + if (attempt > UnsubscribeRetryMaxAttempts) return; + + var delay = TimeSpan.FromTicks(Math.Min( + UnsubscribeRetryInitialDelay.Ticks << Math.Min(attempt - 1, 16), + UnsubscribeRetryMaxDelay.Ticks)); + _ = RetryUnsubscribeAsync(channel, retired, flags & Message.UserSelectableFlags, delay); + } + + private async Task RetryUnsubscribeAsync(RedisChannel channel, Subscription retired, CommandFlags flags, TimeSpan delay) + { + try + { + await Task.Delay(delay).ForAwait(); + if (_isDisposed) return; + + var subscriber = (RedisSubscriber)GetSubscriber(); + // only while the channel has no subscription again, and in order with any that's added afterwards; see + // TryIssueWhileUnsubscribed (otherwise, this could undo a subscription newer than the retired one) + if (TryIssueWhileUnsubscribed(channel, () => retired.UnsubscribeFromServerAsync(subscriber, channel, flags, null, true), out var unsubscribe)) + { + ObserveUnsubscribe(unsubscribe, channel, retired, flags); + } + } + catch + { + // e.g. the multiplexer was disposed meanwhile; nothing more to do + } + } + + /// + /// Observes the outcome of a retired subscription's unsubscribe, to retry it if it was never written (so the server + /// never saw it); a failure the server replied with is left to , + /// which sees every reply, rather than only those of awaited commands. + /// + internal void ObserveUnsubscribe(Task unsubscribe, in RedisChannel channel, Subscription retired, CommandFlags flags) + { + if (unsubscribe.Status is TaskStatus.RanToCompletion) return; + + var observedChannel = channel; + _ = unsubscribe.ContinueWith( + t => + { + if (t.Exception?.InnerException is { } fault) OnUnsubscribeFailed(fault, observedChannel, retired, flags); + }, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously); + } + + internal void OnUnsubscribeFailed(Exception fault, in RedisChannel channel, Subscription retired, CommandFlags flags) + { + if (fault is RedisServerException) return; // a reply; see TrackSubscriptionsProcessor + if (new Availability.FaultContext(fault).NotApplied) OnUnsubscribeNotApplied(channel, retired, flags); + } + + /// + /// Issues a command via , unless has a subscription; holding the + /// same lock as , so that a subscription added afterwards has its subscribe command + /// issued after this one. + /// + /// if the command was issued, if the channel has a subscription. + internal bool TryIssueWhileUnsubscribed(in RedisChannel channel, Func> issue, [NotNullWhen(true)] out Task? issued) + { + lock (subscriptions) + { + if (subscriptions.ContainsKey(channel)) + { + issued = null; + return false; + } + issued = issue(); + return true; + } + } + /// /// Gets the subscriber counts for a channel. /// @@ -384,6 +474,11 @@ private bool UnsubscribeFromServerThenRemove(in RedisChannel channel, Subscripti { return sub.UnsubscribeFromServer(this, channel, flags, false); } + catch (Exception ex) + { + multiplexer.OnUnsubscribeFailed(ex, channel, sub, flags); + throw; + } finally { multiplexer.TryRemoveSubscription(channel, out _); @@ -394,7 +489,9 @@ private Task UnsubscribeFromServerThenRemoveAsync(in RedisChannel channel, { try { - return sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false); + var unsubscribe = sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false); + multiplexer.ObserveUnsubscribe(unsubscribe, channel, sub, flags); + return unsubscribe; } finally { diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 50ce1255f..c6847cace 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -539,6 +539,26 @@ public sealed class TrackSubscriptionsProcessor : ResultProcessor private ConnectionMultiplexer.Subscription? Subscription { get; } public TrackSubscriptionsProcessor(ConnectionMultiplexer.Subscription? sub) => Subscription = sub; + public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) + { + // The server declining to run an unsubscribe of a retired subscription (e.g. BUSY) leaves it subscribed, with + // nothing left on the client to unsubscribe; this sees every reply (unlike the caller, who sees none of a + // fire-and-forget command's), so it's what notices, see ConnectionMultiplexer.OnUnsubscribeNotApplied. (The + // subscription was retired before the command was issued, and so before its reply is read.) + if (Subscription is { IsRetired: true } retired + && message is Message.CommandChannelBase { Channel: var channel } + && message.Command is RedisCommand.UNSUBSCRIBE or RedisCommand.SUNSUBSCRIBE or RedisCommand.PUNSUBSCRIBE) + { + var copy = reader; + copy.MovePastBof(); + if (copy.IsError && Availability.FaultContext.IsKnownNotApplied(RedisErrorKindMetadata.Classify(copy), CommandStatus.Sent)) + { + connection.BridgeCouldBeNull?.Multiplexer.OnUnsubscribeNotApplied(channel, retired, message.Flags); + } + } + return base.SetResult(connection, message, ref reader); + } + protected override bool SetResultCore(PhysicalConnection connection, Message message, ref RespReader reader) { if (reader.IsAggregate) diff --git a/src/StackExchange.Redis/Subscription.cs b/src/StackExchange.Redis/Subscription.cs index a23f10ef5..a2d32dfce 100644 --- a/src/StackExchange.Redis/Subscription.cs +++ b/src/StackExchange.Redis/Subscription.cs @@ -38,6 +38,15 @@ internal abstract class Subscription internal void MarkRetired() => _isRetired = true; // only to be called while holding RegistrationLock + private int _declinedUnsubscribes; + + /// + /// Records that the server declined to run an unsubscribe of this (retired) subscription, see + /// . + /// + /// How many times that has happened so far. + internal int RecordDeclinedUnsubscribe() => Interlocked.Increment(ref _declinedUnsubscribes); + /// /// Adds a handler and/or queue, unless this subscription has been retired (in which case the caller should /// look it up again, which yields a fresh one). diff --git a/tests/StackExchange.Redis.Tests/FailedUnsubscribeUnitTests.cs b/tests/StackExchange.Redis.Tests/FailedUnsubscribeUnitTests.cs new file mode 100644 index 000000000..9e59a6269 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/FailedUnsubscribeUnitTests.cs @@ -0,0 +1,126 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Unsubscribing a channel's last handler retires its subscription on the client whether or not the server-side +/// unsubscribe succeeds; when the server declined to run it (e.g. BUSY), the unsubscribe must be re-issued, +/// otherwise the server stays subscribed (pushing messages that nothing receives) with nothing left to unsubscribe. +/// +[RunPerProtocol] +public class FailedUnsubscribeUnitTests(ITestOutputHelper log) +{ + private sealed class BusyServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _refusals; + + /// Makes the server reply BUSY to the next unsubscribes. + public void RefuseUnsubscribes(int count) => Volatile.Write(ref _refusals, count); + + protected override TypedRedisValue Unsubscribe(RedisClient client, in RedisRequest request) + => Interlocked.Decrement(ref _refusals) >= 0 + ? TypedRedisValue.Error("BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE.") + : base.Unsubscribe(client, request); + } + + private static async Task PublishUntil(ISubscriber subscriber, RedisChannel channel, Func condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + long received; + do + { + received = await subscriber.PublishAsync(channel, "probe"); + if (condition(received)) break; + await Task.Delay(50); + } + while (DateTime.UtcNow < deadline); + return received; + } + + [Fact] + public async Task DeclinedLastUnsubscribeIsRetried() + { + using var server = new BusyServer(log); + await using var conn = await server.ConnectAsync(); + var subscriber = conn.GetSubscriber(); + var channel = RedisChannel.Literal(nameof(DeclinedLastUnsubscribeIsRetried)); + Action handler = static (_, _) => { }; + + await subscriber.SubscribeAsync(channel, handler); + Assert.Equal(1, await subscriber.PublishAsync(channel, "subscribed")); + + server.RefuseUnsubscribes(3); + var ex = await Assert.ThrowsAsync(() => subscriber.UnsubscribeAsync(channel, handler)); + Assert.Equal(RedisErrorKind.Busy, ex.Kind); // the caller still learns that it failed + + // ...but the server ends up unsubscribed nonetheless, once it accepts the unsubscribe + Assert.Equal(0, await PublishUntil(subscriber, channel, static received => received == 0)); + } + + [Fact] + public async Task DeclinedFireAndForgetLastUnsubscribeIsRetried() + { + using var server = new BusyServer(log); + await using var conn = await server.ConnectAsync(); + var subscriber = conn.GetSubscriber(); + var channel = RedisChannel.Literal(nameof(DeclinedFireAndForgetLastUnsubscribeIsRetried)); + Action handler = static (_, _) => { }; + + await subscriber.SubscribeAsync(channel, handler); + Assert.Equal(1, await subscriber.PublishAsync(channel, "subscribed")); + + // nobody observes the reply of a fire-and-forget command, so it's the reply processing that has to notice + server.RefuseUnsubscribes(3); + await subscriber.UnsubscribeAsync(channel, handler, CommandFlags.FireAndForget); + + Assert.Equal(0, await PublishUntil(subscriber, channel, static received => received == 0)); + } + + [Fact] + public async Task DeclinedSyncLastUnsubscribeIsRetried() + { + using var server = new BusyServer(log); + await using var conn = await server.ConnectAsync(); + var subscriber = conn.GetSubscriber(); + var channel = RedisChannel.Literal(nameof(DeclinedSyncLastUnsubscribeIsRetried)); + Action handler = static (_, _) => { }; + + subscriber.Subscribe(channel, handler); + Assert.Equal(1, await subscriber.PublishAsync(channel, "subscribed")); + + server.RefuseUnsubscribes(3); + var ex = Assert.Throws(() => subscriber.Unsubscribe(channel, handler)); + Assert.Equal(RedisErrorKind.Busy, ex.Kind); + + Assert.Equal(0, await PublishUntil(subscriber, channel, static received => received == 0)); + } + + [Fact] + public async Task RetriedUnsubscribeDoesNotUndoNewerSubscription() + { + using var server = new BusyServer(log); + await using var conn = await server.ConnectAsync(); + var subscriber = conn.GetSubscriber(); + var channel = RedisChannel.Literal(nameof(RetriedUnsubscribeDoesNotUndoNewerSubscription)); + Action leaving = static (_, _) => { }; + var received = 0; + Action joining = (_, _) => Interlocked.Increment(ref received); + + await subscriber.SubscribeAsync(channel, leaving); + server.RefuseUnsubscribes(1); + await Assert.ThrowsAsync(() => subscriber.UnsubscribeAsync(channel, leaving)); + + // subscribed to again before the retry is due; from then on, the retry must not unsubscribe it + await subscriber.SubscribeAsync(channel, joining); + await Task.Delay(TimeSpan.FromSeconds(1)); // well past the first few retries + + Assert.Equal(1, await subscriber.PublishAsync(channel, "still subscribed")); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (Volatile.Read(ref received) == 0 && DateTime.UtcNow < deadline) await Task.Delay(20); + Assert.Equal(1, Volatile.Read(ref received)); + } +}