diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index 7ec643082..0aeb96f51 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -130,7 +130,7 @@ public FaultContext(Exception fault) /// public Maintenance.MaintenanceNotificationType MaintenanceType { get; } - private static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status) + internal static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status) { // the client never handed it to the socket, so the server cannot have seen it if (status is CommandStatus.WaitingToBeSent or CommandStatus.WaitingInBacklog) return true; diff --git a/src/StackExchange.Redis/RedisSubscriber.cs b/src/StackExchange.Redis/RedisSubscriber.cs index d0741e5f5..79394b29d 100644 --- a/src/StackExchange.Redis/RedisSubscriber.cs +++ b/src/StackExchange.Redis/RedisSubscriber.cs @@ -32,6 +32,22 @@ internal Subscription GetOrAddSubscription(in RedisChannel channel, CommandFlags return sub; } } + + /// + /// Adds a handler and/or queue to the channel's subscription, creating one if there is none; never to one that + /// is being retired by a concurrent unsubscribe, see . + /// + internal Subscription AddToSubscription(in RedisChannel channel, CommandFlags flags, Action? handler, ChannelMessageQueue? queue) + { + while (true) + { + var sub = GetOrAddSubscription(channel, flags); + if (sub.TryAdd(handler, queue)) return sub; + + // it was retired (and so removed) concurrently; looking it up again yields a fresh one + } + } + internal bool TryGetSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) => subscriptions.TryGetValue(channel, out sub); internal bool TryRemoveSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) { @@ -41,6 +57,96 @@ internal bool TryRemoveSubscription(in RedisChannel channel, [NotNullWhen(true)] } } + private static readonly TimeSpan UnsubscribeRetryInitialDelay = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan UnsubscribeRetryMaxDelay = TimeSpan.FromSeconds(5); + private const int UnsubscribeRetryMaxAttempts = 30; + + /// + /// Retiring a subscription removes it from the multiplexer whether or not the server-side unsubscribe succeeds. When + /// the server never ran that unsubscribe (e.g. it replied BUSY), it is still subscribed, and would keep pushing + /// messages that nothing receives, with nothing left on the client to unsubscribe; so this re-issues the unsubscribe, + /// after a backoff, unless the channel has been subscribed to again by then (at which point client and server agree + /// once more). It is called for each such failure - including those of the re-issued unsubscribes - from wherever the + /// failure is observed: for error replies (whether or not + /// anyone awaits the command, e.g. ), and + /// for failures with no reply at all (the command never having been written). + /// + internal void OnUnsubscribeNotApplied(in RedisChannel channel, Subscription retired, CommandFlags flags) + { + var attempt = retired.RecordDeclinedUnsubscribe(); + if (attempt > UnsubscribeRetryMaxAttempts) return; + + var delay = TimeSpan.FromTicks(Math.Min( + UnsubscribeRetryInitialDelay.Ticks << Math.Min(attempt - 1, 16), + UnsubscribeRetryMaxDelay.Ticks)); + _ = RetryUnsubscribeAsync(channel, retired, flags & Message.UserSelectableFlags, delay); + } + + private async Task RetryUnsubscribeAsync(RedisChannel channel, Subscription retired, CommandFlags flags, TimeSpan delay) + { + try + { + await Task.Delay(delay).ForAwait(); + if (_isDisposed) return; + + var subscriber = (RedisSubscriber)GetSubscriber(); + // only while the channel has no subscription again, and in order with any that's added afterwards; see + // TryIssueWhileUnsubscribed (otherwise, this could undo a subscription newer than the retired one) + if (TryIssueWhileUnsubscribed(channel, () => retired.UnsubscribeFromServerAsync(subscriber, channel, flags, null, true), out var unsubscribe)) + { + ObserveUnsubscribe(unsubscribe, channel, retired, flags); + } + } + catch + { + // e.g. the multiplexer was disposed meanwhile; nothing more to do + } + } + + /// + /// Observes the outcome of a retired subscription's unsubscribe, to retry it if it was never written (so the server + /// never saw it); a failure the server replied with is left to , + /// which sees every reply, rather than only those of awaited commands. + /// + internal void ObserveUnsubscribe(Task unsubscribe, in RedisChannel channel, Subscription retired, CommandFlags flags) + { + if (unsubscribe.Status is TaskStatus.RanToCompletion) return; + + var observedChannel = channel; + _ = unsubscribe.ContinueWith( + t => + { + if (t.Exception?.InnerException is { } fault) OnUnsubscribeFailed(fault, observedChannel, retired, flags); + }, + TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously); + } + + internal void OnUnsubscribeFailed(Exception fault, in RedisChannel channel, Subscription retired, CommandFlags flags) + { + if (fault is RedisServerException) return; // a reply; see TrackSubscriptionsProcessor + if (new Availability.FaultContext(fault).NotApplied) OnUnsubscribeNotApplied(channel, retired, flags); + } + + /// + /// Issues a command via , unless has a subscription; holding the + /// same lock as , so that a subscription added afterwards has its subscribe command + /// issued after this one. + /// + /// if the command was issued, if the channel has a subscription. + internal bool TryIssueWhileUnsubscribed(in RedisChannel channel, Func> issue, [NotNullWhen(true)] out Task? issued) + { + lock (subscriptions) + { + if (subscriptions.ContainsKey(channel)) + { + issued = null; + return false; + } + issued = issue(); + return true; + } + } + /// /// Gets the subscriber counts for a channel. /// @@ -248,8 +354,7 @@ private int Subscribe(RedisChannel channel, Action? ha ThrowIfNull(channel); if (handler == null && queue == null) { return 0; } - var sub = multiplexer.GetOrAddSubscription(channel, flags); - sub.Add(handler, queue); + var sub = multiplexer.AddToSubscription(channel, flags, handler, queue); return sub.EnsureSubscribedToServer(this, channel, flags, false); } @@ -291,8 +396,7 @@ private Task SubscribeAsync(RedisChannel channel, Action.Default(null); } - var sub = multiplexer.GetOrAddSubscription(channel, flags); - sub.Add(handler, queue); + var sub = multiplexer.AddToSubscription(channel, flags, handler, queue); return sub.EnsureSubscribedToServerAsync(this, channel, flags, false, server); } @@ -304,11 +408,17 @@ void ISubscriber.Unsubscribe(RedisChannel channel, Action? handler, ChannelMessageQueue? queue, CommandFlags flags) { ThrowIfNull(channel); - // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server - // ReSharper disable once SimplifyConditionalTernaryExpression - return UnregisterSubscription(channel, handler, queue, out var sub) - ? sub.UnsubscribeFromServer(this, channel, flags, false) - : true; + if (!multiplexer.TryGetSubscription(channel, out var sub)) return true; + + // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server; + // the latter while still holding the lock, see RetireIfEmpty + lock (sub.RegistrationLock) + { + // ReSharper disable once SimplifyConditionalTernaryExpression + return RetireIfEmpty(sub, handler, queue) + ? UnsubscribeFromServerThenRemove(channel, sub, flags) + : true; + } } Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action? handler, CommandFlags flags) @@ -317,36 +427,76 @@ Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action UnsubscribeAsync(in RedisChannel channel, Action? handler, ChannelMessageQueue? queue, CommandFlags flags) { ThrowIfNull(channel); - // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server - return UnregisterSubscription(channel, handler, queue, out var sub) - ? sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false) - : CompletedTask.Default(asyncState); + if (!multiplexer.TryGetSubscription(channel, out var sub)) return CompletedTask.Default(asyncState); + + // Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server; + // the latter is issued (though not awaited) while still holding the lock, see RetireIfEmpty + lock (sub.RegistrationLock) + { + return RetireIfEmpty(sub, handler, queue) + ? UnsubscribeFromServerThenRemoveAsync(channel, sub, flags) + : CompletedTask.Default(asyncState); + } } /// - /// Unregisters a handler or queue and returns if we should remove it from the server. + /// Unregisters a handler or queue, and retires the subscription if that was its last one; must be called while + /// holding the subscription's . If it returns , the + /// caller must then - still holding the lock - issue the unsubscribe command to the server, and only *then* remove + /// the subscription from the multiplexer (see ). That way, a concurrent + /// subscribe to the same channel still finds this subscription until the unsubscribe command has been issued, and + /// so has to wait for the lock (only to find it retired, and to look up a fresh one); so its subscribe command is + /// issued after the unsubscribe command, rather than being able to overtake it and leave the server unsubscribed. /// /// if we should remove the subscription from the server, otherwise. - private bool UnregisterSubscription(in RedisChannel channel, Action? handler, ChannelMessageQueue? queue, [NotNullWhen(true)] out Subscription? sub) + private static bool RetireIfEmpty(Subscription sub, Action? handler, ChannelMessageQueue? queue) { - ThrowIfNull(channel); - if (multiplexer.TryGetSubscription(channel, out sub)) + if (sub.IsRetired) return false; // already retired (and removed) concurrently + + if ((handler == null) & (queue == null)) { - if ((handler == null) & (queue == null)) - { - // This was a blanket wipe, so clear it completely - sub.MarkCompleted(); - multiplexer.TryRemoveSubscription(channel, out _); - return true; - } - else if (sub.Remove(handler, queue)) - { - // Or this was the last handler and/or queue, which also means unsubscribe - multiplexer.TryRemoveSubscription(channel, out _); - return true; - } + // This was a blanket wipe, so clear it completely + sub.MarkCompleted(); + } + else if (!sub.Remove(handler, queue)) + { + // Not the last handler and/or queue, so the subscription stays + return false; + } + + sub.MarkRetired(); + return true; + } + + private bool UnsubscribeFromServerThenRemove(in RedisChannel channel, Subscription sub, CommandFlags flags) + { + try + { + return sub.UnsubscribeFromServer(this, channel, flags, false); + } + catch (Exception ex) + { + multiplexer.OnUnsubscribeFailed(ex, channel, sub, flags); + throw; + } + finally + { + multiplexer.TryRemoveSubscription(channel, out _); + } + } + + private Task UnsubscribeFromServerThenRemoveAsync(in RedisChannel channel, Subscription sub, CommandFlags flags) + { + try + { + var unsubscribe = sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false); + multiplexer.ObserveUnsubscribe(unsubscribe, channel, sub, flags); + return unsubscribe; + } + finally + { + multiplexer.TryRemoveSubscription(channel, out _); } - return false; } // TODO: We need a new api to support SUNSUBSCRIBE all. Calling this now would unsubscribe both sharded and unsharded channels. @@ -356,10 +506,13 @@ public void UnsubscribeAll(CommandFlags flags = CommandFlags.None) var subs = multiplexer.GetSubscriptions(); foreach (var pair in subs) { - if (subs.TryRemove(pair.Key, out var sub)) + var sub = pair.Value; + lock (sub.RegistrationLock) { - sub.MarkCompleted(); - sub.UnsubscribeFromServer(this, pair.Key, flags, false); + if (RetireIfEmpty(sub, null, null)) + { + UnsubscribeFromServerThenRemove(pair.Key, sub, flags); + } } } } @@ -371,10 +524,13 @@ public Task UnsubscribeAllAsync(CommandFlags flags = CommandFlags.None) var subs = multiplexer.GetSubscriptions(); foreach (var pair in subs) { - if (subs.TryRemove(pair.Key, out var sub)) + var sub = pair.Value; + lock (sub.RegistrationLock) { - sub.MarkCompleted(); - last = sub.UnsubscribeFromServerAsync(this, pair.Key, flags, asyncState, false); + if (RetireIfEmpty(sub, null, null)) + { + last = UnsubscribeFromServerThenRemoveAsync(pair.Key, sub, flags); + } } } return last ?? CompletedTask.Default(asyncState); diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 50ce1255f..c6847cace 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -539,6 +539,26 @@ public sealed class TrackSubscriptionsProcessor : ResultProcessor private ConnectionMultiplexer.Subscription? Subscription { get; } public TrackSubscriptionsProcessor(ConnectionMultiplexer.Subscription? sub) => Subscription = sub; + public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) + { + // The server declining to run an unsubscribe of a retired subscription (e.g. BUSY) leaves it subscribed, with + // nothing left on the client to unsubscribe; this sees every reply (unlike the caller, who sees none of a + // fire-and-forget command's), so it's what notices, see ConnectionMultiplexer.OnUnsubscribeNotApplied. (The + // subscription was retired before the command was issued, and so before its reply is read.) + if (Subscription is { IsRetired: true } retired + && message is Message.CommandChannelBase { Channel: var channel } + && message.Command is RedisCommand.UNSUBSCRIBE or RedisCommand.SUNSUBSCRIBE or RedisCommand.PUNSUBSCRIBE) + { + var copy = reader; + copy.MovePastBof(); + if (copy.IsError && Availability.FaultContext.IsKnownNotApplied(RedisErrorKindMetadata.Classify(copy), CommandStatus.Sent)) + { + connection.BridgeCouldBeNull?.Multiplexer.OnUnsubscribeNotApplied(channel, retired, message.Flags); + } + } + return base.SetResult(connection, message, ref reader); + } + protected override bool SetResultCore(PhysicalConnection connection, Message message, ref RespReader reader) { if (reader.IsAggregate) diff --git a/src/StackExchange.Redis/Subscription.cs b/src/StackExchange.Redis/Subscription.cs index 767ab0a14..a2d32dfce 100644 --- a/src/StackExchange.Redis/Subscription.cs +++ b/src/StackExchange.Redis/Subscription.cs @@ -1,6 +1,7 @@ using System; using System.Buffers; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -19,6 +20,47 @@ internal abstract class Subscription private Action? _handlers; private readonly object _handlersLock = new(); private ChannelMessageQueue? _queues; + + // Guards this subscription's registration: handlers/queues being added to it, versus it being retired once its + // last one is removed (at which point it is also removed from the multiplexer, and unsubscribed from the server). + // Without it, a handler could be added to a subscription that is concurrently being retired; that handler would + // then never be invoked, since nothing routes messages to a retired subscription (and the server has been told + // to stop sending them). + private readonly object _registrationLock = new(); + private bool _isRetired; + + /// + /// Held while retiring this subscription, see . + /// + internal object RegistrationLock => _registrationLock; + + internal bool IsRetired => _isRetired; // only meaningful while holding RegistrationLock + + internal void MarkRetired() => _isRetired = true; // only to be called while holding RegistrationLock + + private int _declinedUnsubscribes; + + /// + /// Records that the server declined to run an unsubscribe of this (retired) subscription, see + /// . + /// + /// How many times that has happened so far. + internal int RecordDeclinedUnsubscribe() => Interlocked.Increment(ref _declinedUnsubscribes); + + /// + /// Adds a handler and/or queue, unless this subscription has been retired (in which case the caller should + /// look it up again, which yields a fresh one). + /// + internal bool TryAdd(Action? handler, ChannelMessageQueue? queue) + { + lock (_registrationLock) + { + if (_isRetired) return false; + Add(handler, queue); + return true; + } + } + public CommandFlags Flags { get; } public ResultProcessor.TrackSubscriptionsProcessor Processor { get; } @@ -506,11 +548,18 @@ internal override async Task UnsubscribeFromServerAsync( object? asyncState, bool internalCall) { - bool any = false; + // issue all of them before awaiting any, so that they are all issued synchronously (see RegistrationLock) + var pending = new List>(); foreach (var server in _servers) { var message = GetSubscriptionMessage(channel, SubscriptionAction.Unsubscribe, flags, internalCall); - any |= await subscriber.ExecuteAsync(message, Processor, server.Value).ForAwait(); + pending.Add(subscriber.ExecuteAsync(message, Processor, server.Value)); + } + + bool any = false; + foreach (var task in pending) + { + any |= await task.ForAwait(); } return any; diff --git a/tests/StackExchange.Redis.Tests/FailedUnsubscribeUnitTests.cs b/tests/StackExchange.Redis.Tests/FailedUnsubscribeUnitTests.cs new file mode 100644 index 000000000..9e59a6269 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/FailedUnsubscribeUnitTests.cs @@ -0,0 +1,126 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Unsubscribing a channel's last handler retires its subscription on the client whether or not the server-side +/// unsubscribe succeeds; when the server declined to run it (e.g. BUSY), the unsubscribe must be re-issued, +/// otherwise the server stays subscribed (pushing messages that nothing receives) with nothing left to unsubscribe. +/// +[RunPerProtocol] +public class FailedUnsubscribeUnitTests(ITestOutputHelper log) +{ + private sealed class BusyServer(ITestOutputHelper log) : InProcessTestServer(log) + { + private int _refusals; + + /// Makes the server reply BUSY to the next unsubscribes. + public void RefuseUnsubscribes(int count) => Volatile.Write(ref _refusals, count); + + protected override TypedRedisValue Unsubscribe(RedisClient client, in RedisRequest request) + => Interlocked.Decrement(ref _refusals) >= 0 + ? TypedRedisValue.Error("BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE.") + : base.Unsubscribe(client, request); + } + + private static async Task PublishUntil(ISubscriber subscriber, RedisChannel channel, Func condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + long received; + do + { + received = await subscriber.PublishAsync(channel, "probe"); + if (condition(received)) break; + await Task.Delay(50); + } + while (DateTime.UtcNow < deadline); + return received; + } + + [Fact] + public async Task DeclinedLastUnsubscribeIsRetried() + { + using var server = new BusyServer(log); + await using var conn = await server.ConnectAsync(); + var subscriber = conn.GetSubscriber(); + var channel = RedisChannel.Literal(nameof(DeclinedLastUnsubscribeIsRetried)); + Action handler = static (_, _) => { }; + + await subscriber.SubscribeAsync(channel, handler); + Assert.Equal(1, await subscriber.PublishAsync(channel, "subscribed")); + + server.RefuseUnsubscribes(3); + var ex = await Assert.ThrowsAsync(() => subscriber.UnsubscribeAsync(channel, handler)); + Assert.Equal(RedisErrorKind.Busy, ex.Kind); // the caller still learns that it failed + + // ...but the server ends up unsubscribed nonetheless, once it accepts the unsubscribe + Assert.Equal(0, await PublishUntil(subscriber, channel, static received => received == 0)); + } + + [Fact] + public async Task DeclinedFireAndForgetLastUnsubscribeIsRetried() + { + using var server = new BusyServer(log); + await using var conn = await server.ConnectAsync(); + var subscriber = conn.GetSubscriber(); + var channel = RedisChannel.Literal(nameof(DeclinedFireAndForgetLastUnsubscribeIsRetried)); + Action handler = static (_, _) => { }; + + await subscriber.SubscribeAsync(channel, handler); + Assert.Equal(1, await subscriber.PublishAsync(channel, "subscribed")); + + // nobody observes the reply of a fire-and-forget command, so it's the reply processing that has to notice + server.RefuseUnsubscribes(3); + await subscriber.UnsubscribeAsync(channel, handler, CommandFlags.FireAndForget); + + Assert.Equal(0, await PublishUntil(subscriber, channel, static received => received == 0)); + } + + [Fact] + public async Task DeclinedSyncLastUnsubscribeIsRetried() + { + using var server = new BusyServer(log); + await using var conn = await server.ConnectAsync(); + var subscriber = conn.GetSubscriber(); + var channel = RedisChannel.Literal(nameof(DeclinedSyncLastUnsubscribeIsRetried)); + Action handler = static (_, _) => { }; + + subscriber.Subscribe(channel, handler); + Assert.Equal(1, await subscriber.PublishAsync(channel, "subscribed")); + + server.RefuseUnsubscribes(3); + var ex = Assert.Throws(() => subscriber.Unsubscribe(channel, handler)); + Assert.Equal(RedisErrorKind.Busy, ex.Kind); + + Assert.Equal(0, await PublishUntil(subscriber, channel, static received => received == 0)); + } + + [Fact] + public async Task RetriedUnsubscribeDoesNotUndoNewerSubscription() + { + using var server = new BusyServer(log); + await using var conn = await server.ConnectAsync(); + var subscriber = conn.GetSubscriber(); + var channel = RedisChannel.Literal(nameof(RetriedUnsubscribeDoesNotUndoNewerSubscription)); + Action leaving = static (_, _) => { }; + var received = 0; + Action joining = (_, _) => Interlocked.Increment(ref received); + + await subscriber.SubscribeAsync(channel, leaving); + server.RefuseUnsubscribes(1); + await Assert.ThrowsAsync(() => subscriber.UnsubscribeAsync(channel, leaving)); + + // subscribed to again before the retry is due; from then on, the retry must not unsubscribe it + await subscriber.SubscribeAsync(channel, joining); + await Task.Delay(TimeSpan.FromSeconds(1)); // well past the first few retries + + Assert.Equal(1, await subscriber.PublishAsync(channel, "still subscribed")); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (Volatile.Read(ref received) == 0 && DateTime.UtcNow < deadline) await Task.Delay(20); + Assert.Equal(1, Volatile.Read(ref received)); + } +} diff --git a/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs b/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs new file mode 100644 index 000000000..752602584 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/SubscriptionRegistrationRaceUnitTests.cs @@ -0,0 +1,58 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class SubscriptionRegistrationRaceUnitTests(ITestOutputHelper output) +{ + /// + /// Removing a channel's last handler retires its subscription (locally, and on the server); a handler added to the + /// same channel concurrently must end up on a live subscription, rather than on the retired one - which nothing + /// routes messages to anymore, and which the server is unsubscribed from. + /// + [Fact] + public async Task SubscribeRacingLastUnsubscribeStillReceivesMessages() + { + using var server = new InProcessTestServer(output); + await using var conn = await ConnectionMultiplexer.ConnectAsync(server.GetClientConfig()); + var subscriber = conn.GetSubscriber(); + + // the race is narrow, so it takes a number of attempts to provoke it; each uses a fresh channel + const int Attempts = 300; + var missed = 0; + for (var i = 0; i < Attempts; i++) + { + var channel = RedisChannel.Literal($"{nameof(SubscribeRacingLastUnsubscribeStillReceivesMessages)}:{i}"); + Action leaving = static (_, _) => { }; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Action joining = (_, _) => received.TrySetResult(true); + + await subscriber.SubscribeAsync(channel, leaving); + + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var unsubscribe = Task.Run(async () => + { + await start.Task; + await subscriber.UnsubscribeAsync(channel, leaving); + }); + var subscribe = Task.Run(async () => + { + await start.Task; + await subscriber.SubscribeAsync(channel, joining); + }); + start.SetResult(true); + await Task.WhenAll(unsubscribe, subscribe); + + await subscriber.PublishAsync(channel, "hello"); + if (await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(2))) != received.Task) + { + missed++; + } + + await subscriber.UnsubscribeAsync(channel, joining); + } + + Assert.Equal(0, missed); + } +}