Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,32 @@ private void OnCapacitySnapshotPublished(object? sender, EventArgs eventArgs)
_ = ReconcileObservedAsync();
}

/// <summary>
/// Schedules an observed reconcile after a running session reports an unexpected channel fault.
/// Does not open a new AMQP connection.
/// </summary>
private void OnSessionChannelFaulted()
{
if (!IsRunning)
{
return;
}

_ = ReconcileObservedAsync();
}

/// <summary>
/// Returns whether a tracked session still owns an open broker consumer.
/// <see cref="ArticleWorkConsumerState.Running"/> alone is not sufficient after channel failure.
/// </summary>
/// <param name="session">Tracked session candidate.</param>
/// <returns>
/// <see langword="true"/> when the session is running and its consume channel reports itself open.
/// </returns>
private static bool IsHealthyRunningSession(ArticleWorkConsumerSession session) =>
session.State == ArticleWorkConsumerState.Running
&& session.Channel is { IsOpen: true };

/// <summary>Reconciles after a capacity snapshot.</summary>
/// <returns>
/// A task that completes when <see cref="ReconcileAsync"/> finishes or its exception has been logged.
Expand Down Expand Up @@ -558,8 +584,9 @@ private async Task ReconcileObservedAsync()
/// Sessions whose topology did declare are still started.
/// </returns>
/// <remarks>
/// A tracked running session whose key is still desired is kept. Any other tracked session whose key is no longer desired is removed and retired.
/// A desired key with no running tracked session is started. Prefetch is the configured positive count, otherwise 1.
/// A tracked running session whose key is still desired and whose consume channel is open is kept.
/// Any other tracked session is removed and retired, including a Running session whose channel is closed.
/// A desired key with no healthy tracked session is started. Prefetch is the configured positive count, otherwise 1.
/// Sessions started in one pass share a new pipeline. A start failure disposes sessions started in that pass and propagates; they are not tracked.
/// If shutdown is set before publication, those new sessions are disposed of instead of tracked.
/// No desired slots is success. A desired slot whose topology is not ready is not success and does not log completion.
Expand All @@ -581,17 +608,14 @@ private async Task<ReconcilePass> ReconcileSessionsAsync(CancellationToken cance
{
foreach (var pair in _sessions.ToArray())
{
if (desired.ContainsKey(pair.Key) && pair.Value.State == ArticleWorkConsumerState.Running)
if (desired.ContainsKey(pair.Key) && IsHealthyRunningSession(pair.Value))
{
desired.Remove(pair.Key);
continue;
}

if (!desired.ContainsKey(pair.Key))
{
_sessions.Remove(pair.Key);
stale.Add(pair.Value);
}
_sessions.Remove(pair.Key);
stale.Add(pair.Value);
}
}

Expand Down Expand Up @@ -637,7 +661,10 @@ private async Task<ReconcilePass> ReconcileSessionsAsync(CancellationToken cance
_logger,
_runtime.Shutdown,
identity.ConnectionNumber,
identity.ConnectionLimit);
identity.ConnectionLimit)
{
UnexpectedChannelFault = OnSessionChannelFaulted,
};
await session.StartAsync(cancellationToken).ConfigureAwait(false);
started.Add(session);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,12 @@ internal sealed class ArticleWorkConsumerSession : IAsyncDisposable
/// <summary>One after <see cref="DisposeAsync"/> has entered disposal.</summary>
private int _disposed;

/// <summary>
/// Optional callback invoked when an unexpected channel fault retires this session while it was running.
/// The consumer service uses it to schedule an observed reconcile without reconnecting.
/// </summary>
internal Action? UnexpectedChannelFault { get; set; }

/// <summary>
/// Initializes a new session with the default drain-and-finish shutdown policy for connection slot 1 of 1.
/// </summary>
Expand Down Expand Up @@ -316,22 +322,36 @@ internal async Task StartAsync(CancellationToken cancellationToken)
.BasicConsumeAsync(_queue, _prefetch, OnDeliveryAsync, cancellationToken)
.ConfigureAwait(false);

// Install identity under the gate before subscribing so a fault callback can match this session.
// A ChannelFaulted raised in the gap before subscribe is recovered by the IsOpen check below.
IRabbitMqManualAckChannel installed;
lock (_gate)
{
_channel = channel;
installed = channel;
_channel = installed;
_consumerTag = tag;
Generation = handle.Generation;
_state = ArticleWorkConsumerState.Running;
channel = null;
}

installed.ChannelFaulted += OnChannelFaulted;
ArticleWorkLogMessages.ConsumerRunning(_logger, _backbone, _queue, Generation, tag);
if (!installed.IsOpen)
{
OnChannelFaulted(
installed,
new RabbitMqChannelFaultedEventArgs(
RabbitMqChannelFaultKind.ChannelShutdown,
"Channel closed during consumer start."));
}
}
catch (Exception ex)
{
ArticleWorkLogMessages.ConsumerStartFailed(_logger, _backbone, _queue, ex.Message);
if (channel is not null)
{
channel.ChannelFaulted -= OnChannelFaulted;
await channel.DisposeAsync().ConfigureAwait(false);
}

Expand Down Expand Up @@ -500,6 +520,7 @@ private async Task RetireCoreAsync(

if (channel is not null)
{
channel.ChannelFaulted -= OnChannelFaulted;
await channel.DisposeAsync().ConfigureAwait(false);
}

Expand All @@ -513,6 +534,35 @@ private async Task RetireCoreAsync(
ArticleWorkLogMessages.ConsumerStopped(_logger, _backbone, Generation);
}

/// <summary>
/// Retires this session when its installed channel reports an unexpected fault, then notifies the service to reconcile.
/// </summary>
/// <param name="sender">Faulted channel. Ignored when it is not the installed channel.</param>
/// <param name="eventArgs">Fault classification and reason.</param>
/// <remarks>
/// Late callbacks from a retired channel are ignored via identity check.
/// Expected application cancel and disposal do not raise <see cref="IRabbitMqManualAckChannel.ChannelFaulted"/>.
/// </remarks>
private void OnChannelFaulted(object? sender, RabbitMqChannelFaultedEventArgs eventArgs)
{
lock (_gate)
{
if (!ReferenceEquals(_channel, sender) || _state != ArticleWorkConsumerState.Running)
{
return;
}
}

ArticleWorkLogMessages.ConsumerChannelFaulted(
_logger,
_backbone,
Generation,
eventArgs.Kind,
eventArgs.Reason);
UnexpectedChannelFault?.Invoke();
_ = RetireAsync();
}

/// <summary>
/// Issues Basic.Cancel and contains any failure, so retirement still drains and disposes the channel.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using VectorNNTPD.Common.Messaging.RabbitMq;

namespace VectorNNTPD.BackFillerService.ArticleWork
{
/// <summary>
Expand Down Expand Up @@ -353,5 +355,41 @@ internal static partial void ArticleProcessed(
Level = LogLevel.Error,
Message = "Article Work reconcile loop failed and will retry: {Reason}")]
internal static partial void ConsumerReconcileFailed(ILogger logger, string Reason);

/// <summary>
/// Writes a Warning event when a running consumer session observes an unexpected channel fault or broker consumer cancellation.
/// </summary>
/// <param name="logger">Logger that receives the event.</param>
/// <param name="backbone">Provider backbone label.</param>
/// <param name="generation">Connection generation of the faulted channel.</param>
/// <param name="kind">Fault classification.</param>
/// <param name="reason">Broker or client reason text.</param>
[LoggerMessage(
EventId = 5323,
Level = LogLevel.Warning,
Message = "Article Work consumer channel faulted backbone={Backbone} generation={Generation} kind={Kind}: {Reason}")]
internal static partial void ConsumerChannelFaulted(
ILogger logger,
string backbone,
long generation,
RabbitMqChannelFaultKind kind,
string reason);

/// <summary>
/// Writes a Warning event when the response publisher observes an unexpected publish-channel fault and will rebuild on the current connection.
/// </summary>
/// <param name="logger">Logger that receives the event.</param>
/// <param name="generation">Connection generation of the faulted channel.</param>
/// <param name="kind">Fault classification.</param>
/// <param name="reason">Broker or client reason text.</param>
[LoggerMessage(
EventId = 5324,
Level = LogLevel.Warning,
Message = "Article Work publish channel faulted generation={Generation} kind={Kind}: {Reason}")]
internal static partial void PublisherChannelFaulted(
ILogger logger,
long generation,
RabbitMqChannelFaultKind kind,
string reason);
}
}
Loading
Loading