Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 96 additions & 37 deletions src/StackExchange.Redis/RedisSubscriber.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,22 @@ internal Subscription GetOrAddSubscription(in RedisChannel channel, CommandFlags
return sub;
}
}

/// <summary>
/// Adds a handler and/or queue to the channel's subscription, creating one if there is none; never to one that
/// is being retired by a concurrent unsubscribe, see <see cref="Subscription.RegistrationLock"/>.
/// </summary>
internal Subscription AddToSubscription(in RedisChannel channel, CommandFlags flags, Action<RedisChannel, RedisValue>? handler, ChannelMessageQueue? queue)
{
while (true)
{
var sub = GetOrAddSubscription(channel, flags);
if (sub.TryAdd(handler, queue)) return sub;

// it was retired (and so removed) concurrently; looking it up again yields a fresh one
}
}

internal bool TryGetSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub) => subscriptions.TryGetValue(channel, out sub);
internal bool TryRemoveSubscription(in RedisChannel channel, [NotNullWhen(true)] out Subscription? sub)
{
Expand Down Expand Up @@ -248,8 +264,7 @@ private int Subscribe(RedisChannel channel, Action<RedisChannel, RedisValue>? ha
ThrowIfNull(channel);
if (handler == null && queue == null) { return 0; }

var sub = multiplexer.GetOrAddSubscription(channel, flags);
sub.Add(handler, queue);
var sub = multiplexer.AddToSubscription(channel, flags, handler, queue);
return sub.EnsureSubscribedToServer(this, channel, flags, false);
}

Expand Down Expand Up @@ -291,8 +306,7 @@ private Task<int> SubscribeAsync(RedisChannel channel, Action<RedisChannel, Redi
ThrowIfNull(channel);
if (handler == null && queue == null) { return CompletedTask<int>.Default(null); }

var sub = multiplexer.GetOrAddSubscription(channel, flags);
sub.Add(handler, queue);
var sub = multiplexer.AddToSubscription(channel, flags, handler, queue);
return sub.EnsureSubscribedToServerAsync(this, channel, flags, false, server);
}

Expand All @@ -304,11 +318,17 @@ void ISubscriber.Unsubscribe(RedisChannel channel, Action<RedisChannel, RedisVal
public bool Unsubscribe(in RedisChannel channel, Action<RedisChannel, RedisValue>? handler, ChannelMessageQueue? queue, CommandFlags flags)
{
ThrowIfNull(channel);
// Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server
// ReSharper disable once SimplifyConditionalTernaryExpression
return UnregisterSubscription(channel, handler, queue, out var sub)
? sub.UnsubscribeFromServer(this, channel, flags, false)
: true;
if (!multiplexer.TryGetSubscription(channel, out var sub)) return true;

// Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server;
// the latter while still holding the lock, see RetireIfEmpty
lock (sub.RegistrationLock)
{
// ReSharper disable once SimplifyConditionalTernaryExpression
return RetireIfEmpty(sub, handler, queue)
? UnsubscribeFromServerThenRemove(channel, sub, flags)
: true;
}
}

Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action<RedisChannel, RedisValue>? handler, CommandFlags flags)
Expand All @@ -317,36 +337,69 @@ Task ISubscriber.UnsubscribeAsync(RedisChannel channel, Action<RedisChannel, Red
public Task<bool> UnsubscribeAsync(in RedisChannel channel, Action<RedisChannel, RedisValue>? handler, ChannelMessageQueue? queue, CommandFlags flags)
{
ThrowIfNull(channel);
// Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server
return UnregisterSubscription(channel, handler, queue, out var sub)
? sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false)
: CompletedTask<bool>.Default(asyncState);
if (!multiplexer.TryGetSubscription(channel, out var sub)) return CompletedTask<bool>.Default(asyncState);

// Unregister the subscription handler/queue, and if that returns true (last handler removed), also disconnect from the server;
// the latter is issued (though not awaited) while still holding the lock, see RetireIfEmpty
lock (sub.RegistrationLock)
{
return RetireIfEmpty(sub, handler, queue)
? UnsubscribeFromServerThenRemoveAsync(channel, sub, flags)
: CompletedTask<bool>.Default(asyncState);
}
}

/// <summary>
/// Unregisters a handler or queue and returns if we should remove it from the server.
/// Unregisters a handler or queue, and retires the subscription if that was its last one; must be called while
/// holding the subscription's <see cref="Subscription.RegistrationLock"/>. If it returns <see langword="true"/>, the
/// caller must then - still holding the lock - issue the unsubscribe command to the server, and only *then* remove
/// the subscription from the multiplexer (see <see cref="UnsubscribeFromServerThenRemove"/>). That way, a concurrent
/// subscribe to the same channel still finds this subscription until the unsubscribe command has been issued, and
/// so has to wait for the lock (only to find it retired, and to look up a fresh one); so its subscribe command is
/// issued after the unsubscribe command, rather than being able to overtake it and leave the server unsubscribed.
/// </summary>
/// <returns><see langword="true"/> if we should remove the subscription from the server, <see langword="false"/> otherwise.</returns>
private bool UnregisterSubscription(in RedisChannel channel, Action<RedisChannel, RedisValue>? handler, ChannelMessageQueue? queue, [NotNullWhen(true)] out Subscription? sub)
private static bool RetireIfEmpty(Subscription sub, Action<RedisChannel, RedisValue>? handler, ChannelMessageQueue? queue)
{
ThrowIfNull(channel);
if (multiplexer.TryGetSubscription(channel, out sub))
if (sub.IsRetired) return false; // already retired (and removed) concurrently

if ((handler == null) & (queue == null))
{
if ((handler == null) & (queue == null))
{
// This was a blanket wipe, so clear it completely
sub.MarkCompleted();
multiplexer.TryRemoveSubscription(channel, out _);
return true;
}
else if (sub.Remove(handler, queue))
{
// Or this was the last handler and/or queue, which also means unsubscribe
multiplexer.TryRemoveSubscription(channel, out _);
return true;
}
// This was a blanket wipe, so clear it completely
sub.MarkCompleted();
}
else if (!sub.Remove(handler, queue))
{
// Not the last handler and/or queue, so the subscription stays
return false;
}

sub.MarkRetired();
return true;
}

private bool UnsubscribeFromServerThenRemove(in RedisChannel channel, Subscription sub, CommandFlags flags)
{
try
{
return sub.UnsubscribeFromServer(this, channel, flags, false);
}
finally
{
multiplexer.TryRemoveSubscription(channel, out _);
}
}

private Task<bool> UnsubscribeFromServerThenRemoveAsync(in RedisChannel channel, Subscription sub, CommandFlags flags)
{
try
{
return sub.UnsubscribeFromServerAsync(this, channel, flags, asyncState, false);
}
finally
{
multiplexer.TryRemoveSubscription(channel, out _);
}
return false;
}

// TODO: We need a new api to support SUNSUBSCRIBE all. Calling this now would unsubscribe both sharded and unsharded channels.
Expand All @@ -356,10 +409,13 @@ public void UnsubscribeAll(CommandFlags flags = CommandFlags.None)
var subs = multiplexer.GetSubscriptions();
foreach (var pair in subs)
{
if (subs.TryRemove(pair.Key, out var sub))
var sub = pair.Value;
lock (sub.RegistrationLock)
{
sub.MarkCompleted();
sub.UnsubscribeFromServer(this, pair.Key, flags, false);
if (RetireIfEmpty(sub, null, null))
{
UnsubscribeFromServerThenRemove(pair.Key, sub, flags);
}
}
}
}
Expand All @@ -371,10 +427,13 @@ public Task UnsubscribeAllAsync(CommandFlags flags = CommandFlags.None)
var subs = multiplexer.GetSubscriptions();
foreach (var pair in subs)
{
if (subs.TryRemove(pair.Key, out var sub))
var sub = pair.Value;
lock (sub.RegistrationLock)
{
sub.MarkCompleted();
last = sub.UnsubscribeFromServerAsync(this, pair.Key, flags, asyncState, false);
if (RetireIfEmpty(sub, null, null))
{
last = UnsubscribeFromServerThenRemoveAsync(pair.Key, sub, flags);
}
}
}
return last ?? CompletedTask<bool>.Default(asyncState);
Expand Down
44 changes: 42 additions & 2 deletions src/StackExchange.Redis/Subscription.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Buffers;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
Expand All @@ -19,6 +20,38 @@ internal abstract class Subscription
private Action<RedisChannel, RedisValue>? _handlers;
private readonly object _handlersLock = new();
private ChannelMessageQueue? _queues;

// Guards this subscription's registration: handlers/queues being added to it, versus it being retired once its
// last one is removed (at which point it is also removed from the multiplexer, and unsubscribed from the server).
// Without it, a handler could be added to a subscription that is concurrently being retired; that handler would
// then never be invoked, since nothing routes messages to a retired subscription (and the server has been told
// to stop sending them).
private readonly object _registrationLock = new();
private bool _isRetired;

/// <summary>
/// Held while retiring this subscription, see <see cref="TryAdd"/>.
/// </summary>
internal object RegistrationLock => _registrationLock;

internal bool IsRetired => _isRetired; // only meaningful while holding RegistrationLock

internal void MarkRetired() => _isRetired = true; // only to be called while holding RegistrationLock

/// <summary>
/// Adds a handler and/or queue, unless this subscription has been retired (in which case the caller should
/// look it up again, which yields a fresh one).
/// </summary>
internal bool TryAdd(Action<RedisChannel, RedisValue>? handler, ChannelMessageQueue? queue)
{
lock (_registrationLock)
{
if (_isRetired) return false;
Add(handler, queue);
return true;
}
}

public CommandFlags Flags { get; }
public ResultProcessor.TrackSubscriptionsProcessor Processor { get; }

Expand Down Expand Up @@ -506,11 +539,18 @@ internal override async Task<bool> UnsubscribeFromServerAsync(
object? asyncState,
bool internalCall)
{
bool any = false;
// issue all of them before awaiting any, so that they are all issued synchronously (see RegistrationLock)
var pending = new List<Task<bool>>();
foreach (var server in _servers)
{
var message = GetSubscriptionMessage(channel, SubscriptionAction.Unsubscribe, flags, internalCall);
any |= await subscriber.ExecuteAsync(message, Processor, server.Value).ForAwait();
pending.Add(subscriber.ExecuteAsync(message, Processor, server.Value));
}

bool any = false;
foreach (var task in pending)
{
any |= await task.ForAwait();
}

return any;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
using System;
using System.Threading.Tasks;
using Xunit;

namespace StackExchange.Redis.Tests;

public class SubscriptionRegistrationRaceUnitTests(ITestOutputHelper output)
{
/// <summary>
/// Removing a channel's last handler retires its subscription (locally, and on the server); a handler added to the
/// same channel concurrently must end up on a live subscription, rather than on the retired one - which nothing
/// routes messages to anymore, and which the server is unsubscribed from.
/// </summary>
[Fact]
public async Task SubscribeRacingLastUnsubscribeStillReceivesMessages()
{
using var server = new InProcessTestServer(output);
await using var conn = await ConnectionMultiplexer.ConnectAsync(server.GetClientConfig());
var subscriber = conn.GetSubscriber();

// the race is narrow, so it takes a number of attempts to provoke it; each uses a fresh channel
const int Attempts = 300;
var missed = 0;
for (var i = 0; i < Attempts; i++)
{
var channel = RedisChannel.Literal($"{nameof(SubscribeRacingLastUnsubscribeStillReceivesMessages)}:{i}");
Action<RedisChannel, RedisValue> leaving = static (_, _) => { };
var received = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
Action<RedisChannel, RedisValue> joining = (_, _) => received.TrySetResult(true);

await subscriber.SubscribeAsync(channel, leaving);

var start = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
var unsubscribe = Task.Run(async () =>
{
await start.Task;
await subscriber.UnsubscribeAsync(channel, leaving);
});
var subscribe = Task.Run(async () =>
{
await start.Task;
await subscriber.SubscribeAsync(channel, joining);
});
start.SetResult(true);
await Task.WhenAll(unsubscribe, subscribe);

await subscriber.PublishAsync(channel, "hello");
if (await Task.WhenAny(received.Task, Task.Delay(TimeSpan.FromSeconds(2))) != received.Task)
{
missed++;
}

await subscriber.UnsubscribeAsync(channel, joining);
}

Assert.Equal(0, missed);
}
}
Loading