diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
index 22f5d7e9..7b570bf2 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
@@ -528,6 +528,32 @@ private void OnCapacitySnapshotPublished(object? sender, EventArgs eventArgs)
_ = ReconcileObservedAsync();
}
+ ///
+ /// Schedules an observed reconcile after a running session reports an unexpected channel fault.
+ /// Does not open a new AMQP connection.
+ ///
+ private void OnSessionChannelFaulted()
+ {
+ if (!IsRunning)
+ {
+ return;
+ }
+
+ _ = ReconcileObservedAsync();
+ }
+
+ ///
+ /// Returns whether a tracked session still owns an open broker consumer.
+ /// alone is not sufficient after channel failure.
+ ///
+ /// Tracked session candidate.
+ ///
+ /// when the session is running and its consume channel reports itself open.
+ ///
+ private static bool IsHealthyRunningSession(ArticleWorkConsumerSession session) =>
+ session.State == ArticleWorkConsumerState.Running
+ && session.Channel is { IsOpen: true };
+
/// Reconciles after a capacity snapshot.
///
/// A task that completes when finishes or its exception has been logged.
@@ -558,8 +584,9 @@ private async Task ReconcileObservedAsync()
/// Sessions whose topology did declare are still started.
///
///
- /// A tracked running session whose key is still desired is kept. Any other tracked session whose key is no longer desired is removed and retired.
- /// A desired key with no running tracked session is started. Prefetch is the configured positive count, otherwise 1.
+ /// A tracked running session whose key is still desired and whose consume channel is open is kept.
+ /// Any other tracked session is removed and retired, including a Running session whose channel is closed.
+ /// A desired key with no healthy tracked session is started. Prefetch is the configured positive count, otherwise 1.
/// Sessions started in one pass share a new pipeline. A start failure disposes sessions started in that pass and propagates; they are not tracked.
/// If shutdown is set before publication, those new sessions are disposed of instead of tracked.
/// No desired slots is success. A desired slot whose topology is not ready is not success and does not log completion.
@@ -581,17 +608,14 @@ private async Task ReconcileSessionsAsync(CancellationToken cance
{
foreach (var pair in _sessions.ToArray())
{
- if (desired.ContainsKey(pair.Key) && pair.Value.State == ArticleWorkConsumerState.Running)
+ if (desired.ContainsKey(pair.Key) && IsHealthyRunningSession(pair.Value))
{
desired.Remove(pair.Key);
continue;
}
- if (!desired.ContainsKey(pair.Key))
- {
- _sessions.Remove(pair.Key);
- stale.Add(pair.Value);
- }
+ _sessions.Remove(pair.Key);
+ stale.Add(pair.Value);
}
}
@@ -637,7 +661,10 @@ private async Task ReconcileSessionsAsync(CancellationToken cance
_logger,
_runtime.Shutdown,
identity.ConnectionNumber,
- identity.ConnectionLimit);
+ identity.ConnectionLimit)
+ {
+ UnexpectedChannelFault = OnSessionChannelFaulted,
+ };
await session.StartAsync(cancellationToken).ConfigureAwait(false);
started.Add(session);
}
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs
index 3c2002ea..54c46502 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs
@@ -98,6 +98,12 @@ internal sealed class ArticleWorkConsumerSession : IAsyncDisposable
/// One after has entered disposal.
private int _disposed;
+ ///
+ /// Optional callback invoked when an unexpected channel fault retires this session while it was running.
+ /// The consumer service uses it to schedule an observed reconcile without reconnecting.
+ ///
+ internal Action? UnexpectedChannelFault { get; set; }
+
///
/// Initializes a new session with the default drain-and-finish shutdown policy for connection slot 1 of 1.
///
@@ -316,22 +322,36 @@ internal async Task StartAsync(CancellationToken cancellationToken)
.BasicConsumeAsync(_queue, _prefetch, OnDeliveryAsync, cancellationToken)
.ConfigureAwait(false);
+ // Install identity under the gate before subscribing so a fault callback can match this session.
+ // A ChannelFaulted raised in the gap before subscribe is recovered by the IsOpen check below.
+ IRabbitMqManualAckChannel installed;
lock (_gate)
{
- _channel = channel;
+ installed = channel;
+ _channel = installed;
_consumerTag = tag;
Generation = handle.Generation;
_state = ArticleWorkConsumerState.Running;
channel = null;
}
+ installed.ChannelFaulted += OnChannelFaulted;
ArticleWorkLogMessages.ConsumerRunning(_logger, _backbone, _queue, Generation, tag);
+ if (!installed.IsOpen)
+ {
+ OnChannelFaulted(
+ installed,
+ new RabbitMqChannelFaultedEventArgs(
+ RabbitMqChannelFaultKind.ChannelShutdown,
+ "Channel closed during consumer start."));
+ }
}
catch (Exception ex)
{
ArticleWorkLogMessages.ConsumerStartFailed(_logger, _backbone, _queue, ex.Message);
if (channel is not null)
{
+ channel.ChannelFaulted -= OnChannelFaulted;
await channel.DisposeAsync().ConfigureAwait(false);
}
@@ -500,6 +520,7 @@ private async Task RetireCoreAsync(
if (channel is not null)
{
+ channel.ChannelFaulted -= OnChannelFaulted;
await channel.DisposeAsync().ConfigureAwait(false);
}
@@ -513,6 +534,35 @@ private async Task RetireCoreAsync(
ArticleWorkLogMessages.ConsumerStopped(_logger, _backbone, Generation);
}
+ ///
+ /// Retires this session when its installed channel reports an unexpected fault, then notifies the service to reconcile.
+ ///
+ /// Faulted channel. Ignored when it is not the installed channel.
+ /// Fault classification and reason.
+ ///
+ /// Late callbacks from a retired channel are ignored via identity check.
+ /// Expected application cancel and disposal do not raise .
+ ///
+ private void OnChannelFaulted(object? sender, RabbitMqChannelFaultedEventArgs eventArgs)
+ {
+ lock (_gate)
+ {
+ if (!ReferenceEquals(_channel, sender) || _state != ArticleWorkConsumerState.Running)
+ {
+ return;
+ }
+ }
+
+ ArticleWorkLogMessages.ConsumerChannelFaulted(
+ _logger,
+ _backbone,
+ Generation,
+ eventArgs.Kind,
+ eventArgs.Reason);
+ UnexpectedChannelFault?.Invoke();
+ _ = RetireAsync();
+ }
+
///
/// Issues Basic.Cancel and contains any failure, so retirement still drains and disposes the channel.
///
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs
index 60c29851..c41a316b 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs
@@ -1,3 +1,5 @@
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
namespace VectorNNTPD.BackFillerService.ArticleWork
{
///
@@ -353,5 +355,41 @@ internal static partial void ArticleProcessed(
Level = LogLevel.Error,
Message = "Article Work reconcile loop failed and will retry: {Reason}")]
internal static partial void ConsumerReconcileFailed(ILogger logger, string Reason);
+
+ ///
+ /// Writes a Warning event when a running consumer session observes an unexpected channel fault or broker consumer cancellation.
+ ///
+ /// Logger that receives the event.
+ /// Provider backbone label.
+ /// Connection generation of the faulted channel.
+ /// Fault classification.
+ /// Broker or client reason text.
+ [LoggerMessage(
+ EventId = 5323,
+ Level = LogLevel.Warning,
+ Message = "Article Work consumer channel faulted backbone={Backbone} generation={Generation} kind={Kind}: {Reason}")]
+ internal static partial void ConsumerChannelFaulted(
+ ILogger logger,
+ string backbone,
+ long generation,
+ RabbitMqChannelFaultKind kind,
+ string reason);
+
+ ///
+ /// Writes a Warning event when the response publisher observes an unexpected publish-channel fault and will rebuild on the current connection.
+ ///
+ /// Logger that receives the event.
+ /// Connection generation of the faulted channel.
+ /// Fault classification.
+ /// Broker or client reason text.
+ [LoggerMessage(
+ EventId = 5324,
+ Level = LogLevel.Warning,
+ Message = "Article Work publish channel faulted generation={Generation} kind={Kind}: {Reason}")]
+ internal static partial void PublisherChannelFaulted(
+ ILogger logger,
+ long generation,
+ RabbitMqChannelFaultKind kind,
+ string reason);
}
}
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs
index 0f927d21..3eaa7627 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs
@@ -1,5 +1,6 @@
using VectorNNTPD.BackFillerService.Configuration;
using VectorNNTPD.BackFillerService.Logging;
+using VectorNNTPD.Common.Core;
using VectorNNTPD.Common.Messaging.RabbitMq;
namespace VectorNNTPD.BackFillerService.ArticleWork
@@ -9,8 +10,9 @@ namespace VectorNNTPD.BackFillerService.ArticleWork
///
///
/// Startup fails if a publishing channel cannot be opened on the current generation.
- /// After start, connection replacement rebuilds the publisher channel. This type
- /// does not implement a second connection-recovery loop and never ACK/NACKs deliveries.
+ /// After start, connection replacement and independent publish-channel faults rebuild the publisher channel
+ /// on the current connection generation. This type does not implement a second connection-recovery loop
+ /// and never ACK/NACKs deliveries.
/// Completing means the broker confirmed and the publication generation was still current afterward.
/// Confirmation is not permission to ACK the original delivery.
/// One publication holds . The confirmation wait is cancelled by the caller token, disposal, or the configured confirmation timeout.
@@ -213,6 +215,7 @@ public async ValueTask DisposeAsync()
ArticleWorkLogMessages.PublisherRetiring(_logger, Generation);
_connections.ConnectionReplaced -= OnConnectionReplaced;
+ DetachPublishChannelFaultHandler();
await _runCts.CancelAsync().ConfigureAwait(false);
Task replace;
@@ -306,6 +309,7 @@ public async Task PublishAsync(ArticleWorkResponseIntent intent, CancellationTok
try
{
ThrowIfNotRunning();
+ await EnsureOpenPublishChannelAsync(linked.Token).ConfigureAwait(false);
var channel = CurrentOpenChannelOrThrow();
generation = channel.Generation;
if (!IsPublisherGenerationCurrent(generation))
@@ -400,6 +404,12 @@ internal async Task InstallPublishChannelAsync(IRabbitMqPublishChannel candidate
else
{
replaced = _channel;
+ if (replaced is not null)
+ {
+ replaced.ChannelFaulted -= OnPublishChannelFaulted;
+ }
+
+ candidate.ChannelFaulted += OnPublishChannelFaulted;
_channel = candidate;
}
}
@@ -429,6 +439,45 @@ private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEven
return;
}
+ ScheduleChannelRebuild();
+ }
+
+ ///
+ /// Schedules a publish-channel rebuild when the installed channel faults independently of connection replacement.
+ ///
+ /// Faulted channel. Ignored when it is not the installed channel.
+ /// Fault classification and reason.
+ private void OnPublishChannelFaulted(object? sender, RabbitMqChannelFaultedEventArgs eventArgs)
+ {
+ long generation;
+ lock (_gate)
+ {
+ if (_stopping || _channel is null || !ReferenceEquals(_channel, sender))
+ {
+ return;
+ }
+
+ generation = _channel.Generation;
+ }
+
+ ArticleWorkLogMessages.PublisherChannelFaulted(
+ _logger,
+ generation,
+ eventArgs.Kind,
+ eventArgs.Reason);
+ ScheduleChannelRebuild();
+ }
+
+ ///
+ /// Chains a non-cancelling rebuild after any in-flight rebuild when disposal has not started.
+ ///
+ ///
+ /// Overwriting with a sibling left prior
+ /// waiters on unobserved; disposal could then dispose the semaphore while those
+ /// waiters were still queued. Chaining keeps every rebuild on the task Dispose awaits.
+ ///
+ private void ScheduleChannelRebuild()
+ {
lock (_gate)
{
if (_stopping)
@@ -436,16 +485,39 @@ private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEven
return;
}
- _replaceTask = RebuildAfterReplacementAsync();
+ _replaceTask = ContinueRebuildAsync(_replaceTask);
}
}
- /// Rebuilds the publication channel after a connection replacement unless the startup never succeeded or disposal has started.
+ ///
+ /// Awaits then runs one rebuild so later connection or channel-fault requests are not dropped.
+ ///
+ ///
+ /// Earlier rebuild or completed task. Non-fatal failures are ignored here because
+ /// already logged them; failures propagate.
+ ///
+ /// A task that completes when the follow-on rebuild finishes.
+ private async Task ContinueRebuildAsync(Task previous)
+ {
+ try
+ {
+ await previous.ConfigureAwait(false);
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ // Prior rebuild logged at RebuildPublishChannelAsync; continue so a later request is not dropped.
+ }
+
+ await RebuildPublishChannelAsync().ConfigureAwait(false);
+ }
+
+ /// Rebuilds the publication channel on the current connection unless startup never succeeded or disposal has started.
/// A task that completes when the rebuild attempt finishes. Failures are logged and not rethrown.
///
/// Waits on without a cancellation token, then opens the channel with .
+ /// Used for both connection replacement and independent channel faults.
///
- private async Task RebuildAfterReplacementAsync()
+ private async Task RebuildPublishChannelAsync()
{
await _replaceGate.WaitAsync().ConfigureAwait(false);
try
@@ -474,6 +546,48 @@ private async Task RebuildAfterReplacementAsync()
}
}
+ ///
+ /// Waits for any in-flight rebuild and, when the installed channel is closed, schedules another rebuild on the current connection.
+ ///
+ /// Cancels the wait for an in-flight rebuild.
+ ///
+ /// Does not take while holding ; it only awaits the tracked rebuild task.
+ ///
+ private async Task EnsureOpenPublishChannelAsync(CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ Task replace;
+ var open = false;
+ lock (_gate)
+ {
+ if (_channel is { IsOpen: true })
+ {
+ open = true;
+ replace = Task.CompletedTask;
+ }
+ else if (_stopping)
+ {
+ replace = Task.CompletedTask;
+ }
+ else if (!_replaceTask.IsCompleted)
+ {
+ replace = _replaceTask;
+ }
+ else
+ {
+ _replaceTask = ContinueRebuildAsync(_replaceTask);
+ replace = _replaceTask;
+ }
+ }
+
+ if (open)
+ {
+ return;
+ }
+
+ await replace.WaitAsync(cancellationToken).ConfigureAwait(false);
+ }
+
/// Opens a publication channel on the current connection and installs it when the generation is still current.
///
/// when the caller is . A missing connection then throws.
@@ -525,6 +639,11 @@ private async Task DisposeChannelAsync()
lock (_gate)
{
channel = _channel;
+ if (channel is not null)
+ {
+ channel.ChannelFaulted -= OnPublishChannelFaulted;
+ }
+
_channel = null;
}
@@ -534,6 +653,18 @@ private async Task DisposeChannelAsync()
}
}
+ /// Unsubscribes the fault handler from the installed channel without disposing it.
+ private void DetachPublishChannelFaultHandler()
+ {
+ lock (_gate)
+ {
+ if (_channel is not null)
+ {
+ _channel.ChannelFaulted -= OnPublishChannelFaulted;
+ }
+ }
+ }
+
///
/// Waits until can be acquired, then releases it so disposal can dispose of the semaphore.
///
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs
index 5a87ffa4..646689b8 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs
@@ -20,6 +20,12 @@ internal interface IRabbitMqManualAckChannel : IAsyncDisposable
///
bool IsOpen { get; }
+ ///
+ /// Raised once when this channel becomes unexpectedly unusable while the process connection may still be open.
+ /// Expected application cancel and disposal do not raise this event.
+ ///
+ event EventHandler? ChannelFaulted;
+
///
/// Declares an exchange. Idempotent for compatible existing entities; incompatible
/// existing entities fail closed (never deleted or mutated).
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqPublishChannel.cs
index 72d42700..4df1bdd7 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqPublishChannel.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqPublishChannel.cs
@@ -16,6 +16,12 @@ internal interface IRabbitMqPublishChannel : IAsyncDisposable
/// Gets whether the channel currently reports itself open.
bool IsOpen { get; }
+ ///
+ /// Raised once when this publish channel becomes unexpectedly unusable while the process connection may still be open.
+ /// Disposal does not raise this event.
+ ///
+ event EventHandler? ChannelFaulted;
+
///
/// Publishes one message described by and waits for a publisher confirmation.
///
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultKind.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultKind.cs
new file mode 100644
index 00000000..6045421c
--- /dev/null
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultKind.cs
@@ -0,0 +1,12 @@
+namespace VectorNNTPD.Common.Messaging.RabbitMq
+{
+ /// Why a caller-owned AMQP channel became unusable while the process connection may still be open.
+ internal enum RabbitMqChannelFaultKind
+ {
+ /// The AMQP channel shut down (broker or peer close independent of connection loss).
+ ChannelShutdown = 0,
+
+ /// The broker cancelled the consumer tag without an application basic.cancel.
+ ConsumerCancelled = 1,
+ }
+}
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultedEventArgs.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultedEventArgs.cs
new file mode 100644
index 00000000..f987e55e
--- /dev/null
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultedEventArgs.cs
@@ -0,0 +1,7 @@
+namespace VectorNNTPD.Common.Messaging.RabbitMq
+{
+ /// Details for a single unexpected channel or consumer fault.
+ /// Fault classification.
+ /// Broker or client reason text; never credentials or payload bytes.
+ internal sealed record RabbitMqChannelFaultedEventArgs(RabbitMqChannelFaultKind Kind, string Reason);
+}
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs
index 0526f4c1..f55622e3 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs
@@ -42,6 +42,15 @@ internal sealed class RabbitMqClientManualAckChannel : IRabbitMqManualAckChannel
/// 1 after the first has marked the channel disposed.
private int _disposed;
+ /// 1 after an unexpected fault has been raised once for this channel.
+ private int _faultRaised;
+
+ ///
+ /// 1 after this wrapper initiates so broker
+ /// is treated as expected.
+ ///
+ private int _applicationCancel;
+
///
/// Initializes a new wrapper around an opened channel.
///
@@ -52,11 +61,15 @@ internal RabbitMqClientManualAckChannel(IChannel channel, long generation)
ArgumentNullException.ThrowIfNull(channel);
_channel = channel;
Generation = generation;
+ _channel.ChannelShutdownAsync += OnChannelShutdownAsync;
}
///
public long Generation { get; }
+ ///
+ public event EventHandler? ChannelFaulted;
+
/// Threads currently waiting to enter consumer registration or disposal.
internal int LifecycleWaiters => Volatile.Read(ref _lifecycleWaiters);
@@ -163,6 +176,8 @@ public async Task BasicConsumeAsync(
var consumer = new AsyncEventingBasicConsumer(_channel);
_onDelivery = onDelivery;
consumer.ReceivedAsync += OnReceivedAsync;
+ consumer.UnregisteredAsync += OnConsumerUnregisteredAsync;
+ consumer.ShutdownAsync += OnConsumerShutdownAsync;
try
{
var consumerTag = await _channel
@@ -174,7 +189,7 @@ public async Task BasicConsumeAsync(
catch (Exception)
{
_onDelivery = null;
- consumer.ReceivedAsync -= OnReceivedAsync;
+ DetachConsumerHandlers(consumer);
throw;
}
}
@@ -189,6 +204,7 @@ public Task BasicCancelAsync(string consumerTag, CancellationToken cancellationT
{
ArgumentException.ThrowIfNullOrWhiteSpace(consumerTag);
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
+ Interlocked.Exchange(ref _applicationCancel, 1);
return _channel.BasicCancelAsync(consumerTag, cancellationToken: cancellationToken);
}
@@ -223,6 +239,7 @@ async Task MarkDisposedAsync()
}
DetachConsumer();
+ _channel.ChannelShutdownAsync -= OnChannelShutdownAsync;
close = true;
}
@@ -286,13 +303,96 @@ private void DetachConsumer()
{
if (_consumer is not null)
{
- _consumer.ReceivedAsync -= OnReceivedAsync;
+ DetachConsumerHandlers(_consumer);
_consumer = null;
}
_onDelivery = null;
}
+ /// Removes delivery and fault handlers from .
+ /// Consumer previously attached by .
+ private void DetachConsumerHandlers(AsyncEventingBasicConsumer consumer)
+ {
+ consumer.ReceivedAsync -= OnReceivedAsync;
+ consumer.UnregisteredAsync -= OnConsumerUnregisteredAsync;
+ consumer.ShutdownAsync -= OnConsumerShutdownAsync;
+ }
+
+ /// Raises once for an unexpected channel shutdown.
+ /// Unused.
+ /// Client shutdown details.
+ private Task OnChannelShutdownAsync(object sender, ShutdownEventArgs eventArgs)
+ {
+ TryRaiseChannelFaulted(
+ RabbitMqChannelFaultKind.ChannelShutdown,
+ FormatShutdownReason(eventArgs));
+ return Task.CompletedTask;
+ }
+
+ ///
+ /// Raises when the broker cancels the consumer and this wrapper did not issue cancel.
+ ///
+ /// Unused.
+ /// Cancelled consumer tags from the client.
+ private Task OnConsumerUnregisteredAsync(object sender, ConsumerEventArgs eventArgs)
+ {
+ if (Volatile.Read(ref _applicationCancel) == 1)
+ {
+ return Task.CompletedTask;
+ }
+
+ var tags = eventArgs.ConsumerTags is { Length: > 0 }
+ ? string.Join(',', eventArgs.ConsumerTags)
+ : "(none)";
+ TryRaiseChannelFaulted(
+ RabbitMqChannelFaultKind.ConsumerCancelled,
+ $"Consumer unregistered by broker (tags={tags}).");
+ return Task.CompletedTask;
+ }
+
+ /// Raises once for a consumer-scoped shutdown.
+ /// Unused.
+ /// Client shutdown details.
+ private Task OnConsumerShutdownAsync(object sender, ShutdownEventArgs eventArgs)
+ {
+ TryRaiseChannelFaulted(
+ RabbitMqChannelFaultKind.ChannelShutdown,
+ FormatShutdownReason(eventArgs));
+ return Task.CompletedTask;
+ }
+
+ ///
+ /// Raises at most once when disposal has not marked the channel closed.
+ ///
+ /// Fault classification.
+ /// Broker or client reason text.
+ private void TryRaiseChannelFaulted(RabbitMqChannelFaultKind kind, string reason)
+ {
+ if (Volatile.Read(ref _disposed) == 1)
+ {
+ return;
+ }
+
+ if (Interlocked.Exchange(ref _faultRaised, 1) == 1)
+ {
+ return;
+ }
+
+ ChannelFaulted?.Invoke(this, new RabbitMqChannelFaultedEventArgs(kind, reason));
+ }
+
+ /// Formats a client shutdown reason without credentials or payload bytes.
+ /// Client shutdown details.
+ /// Stable reason text for logs and recovery.
+ private static string FormatShutdownReason(ShutdownEventArgs eventArgs)
+ {
+ var replyText = string.IsNullOrWhiteSpace(eventArgs.ReplyText)
+ ? "(none)"
+ : eventArgs.ReplyText;
+ return $"Channel shutdown initiator={eventArgs.Initiator} code={eventArgs.ReplyCode} text={replyText}";
+ }
+
///
/// Closes an open client channel, then disposes it.
/// Already-closed, interrupted, I/O, disposed, and timeout failures are written to the Serilog self-log.
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs
index 7aa49a1c..62a2996f 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs
@@ -1,4 +1,5 @@
using RabbitMQ.Client;
+using RabbitMQ.Client.Events;
using RabbitMQ.Client.Exceptions;
namespace VectorNNTPD.Common.Messaging.RabbitMq
@@ -22,19 +23,26 @@ internal sealed class RabbitMqClientPublishChannel : IRabbitMqPublishChannel
/// 1 after the first .
private int _disposed;
+ /// 1 after an unexpected fault has been raised once for this channel.
+ private int _faultRaised;
+
/// Initializes a new wrapper around an opened confirm-enabled channel.
internal RabbitMqClientPublishChannel(IChannel channel, long generation)
{
ArgumentNullException.ThrowIfNull(channel);
_channel = channel;
Generation = generation;
+ _channel.ChannelShutdownAsync += OnChannelShutdownAsync;
}
///
public long Generation { get; }
///
- public bool IsOpen => _channel.IsOpen;
+ public event EventHandler? ChannelFaulted;
+
+ ///
+ public bool IsOpen => Volatile.Read(ref _disposed) == 0 && _channel.IsOpen;
///
public Task PublishConfirmedAsync(
@@ -119,6 +127,8 @@ public async ValueTask DisposeAsync()
return;
}
+ _channel.ChannelShutdownAsync -= OnChannelShutdownAsync;
+
try
{
if (_channel.IsOpen)
@@ -133,6 +143,32 @@ public async ValueTask DisposeAsync()
await _channel.DisposeAsync().ConfigureAwait(false);
}
+ /// Raises once for an unexpected channel shutdown.
+ /// Unused.
+ /// Client shutdown details.
+ private Task OnChannelShutdownAsync(object sender, ShutdownEventArgs eventArgs)
+ {
+ if (Volatile.Read(ref _disposed) == 1)
+ {
+ return Task.CompletedTask;
+ }
+
+ if (Interlocked.Exchange(ref _faultRaised, 1) == 1)
+ {
+ return Task.CompletedTask;
+ }
+
+ var replyText = string.IsNullOrWhiteSpace(eventArgs.ReplyText)
+ ? "(none)"
+ : eventArgs.ReplyText;
+ ChannelFaulted?.Invoke(
+ this,
+ new RabbitMqChannelFaultedEventArgs(
+ RabbitMqChannelFaultKind.ChannelShutdown,
+ $"Channel shutdown initiator={eventArgs.Initiator} code={eventArgs.ReplyCode} text={replyText}"));
+ return Task.CompletedTask;
+ }
+
///
/// Builds AMQP properties for a confirmed publication.
///
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs
new file mode 100644
index 00000000..d084b7b2
--- /dev/null
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs
@@ -0,0 +1,550 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using VectorNNTPD.BackFillerService.ArticleWork;
+using VectorNNTPD.BackFillerService.Nntp;
+using VectorNNTPD.BackFillerService.RabbitMq;
+using VectorNNTPD.BackFillerService.Tests.Fixtures;
+using VectorNNTPD.BackFillerService.Tests.RabbitMq;
+using VectorNNTPD.BackFillerService.Tests.TestDoubles;
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
+namespace VectorNNTPD.BackFillerService.Tests.ArticleWork
+{
+ ///
+ /// Channel-scoped recovery for Article Work consumers and the response publisher without reconnecting a healthy AMQP connection.
+ ///
+ public sealed class ArticleWorkChannelFailureRecoveryTests
+ {
+ ///
+ /// Unexpected consumer-channel shutdown retires the failed session and starts a replacement on the same connection generation.
+ ///
+ [Fact]
+ public async Task Consumer_channel_shutdown_retires_session_and_starts_replacement_without_reconnect()
+ {
+ await using var context = await StartConsumerAsync();
+ var first = Assert.Single(context.Consumer.Sessions);
+ var firstChannel = Assert.IsType(first.Channel);
+ Assert.Equal(1, context.Factory.ConnectCount);
+
+ firstChannel.SimulateChannelShutdown();
+
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Consumer.Sessions.Count == 1
+ && !ReferenceEquals(context.Consumer.Sessions[0], first)
+ && context.Consumer.Sessions[0].State == ArticleWorkConsumerState.Running
+ && context.Consumer.Sessions[0].Channel is { IsOpen: true },
+ TimeSpan.FromSeconds(2));
+
+ Assert.Equal(ArticleWorkConsumerState.Stopped, first.State);
+ Assert.Equal(1, firstChannel.DisposeCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ Assert.Equal(1, context.Connections.ConnectionGeneration);
+ Assert.True(context.Factory.LastConnection!.Channels.Count > 1);
+ Assert.Equal(1, Assert.IsType(context.Consumer.Sessions[0].Channel).ConsumeCount);
+ }
+
+ ///
+ /// Broker-side consumer cancellation recovers while the AMQP connection remains open.
+ ///
+ [Fact]
+ public async Task Broker_consumer_cancellation_recovers_while_connection_stays_open()
+ {
+ await using var context = await StartConsumerAsync();
+ var first = Assert.Single(context.Consumer.Sessions);
+ var firstChannel = Assert.IsType(first.Channel);
+
+ firstChannel.SimulateConsumerCancelled();
+
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Consumer.Sessions.Count == 1
+ && !ReferenceEquals(context.Consumer.Sessions[0], first)
+ && context.Consumer.Sessions[0].Channel is { IsOpen: true },
+ TimeSpan.FromSeconds(2));
+
+ Assert.Equal(1, context.Factory.ConnectCount);
+ Assert.True(context.Factory.LastConnection!.IsOpen);
+ Assert.Equal(ArticleWorkConsumerState.Stopped, first.State);
+ }
+
+ ///
+ /// Reconciliation does not keep a session merely because its state is Running when the channel is closed.
+ ///
+ [Fact]
+ public async Task Reconcile_retires_running_session_with_closed_channel()
+ {
+ await using var context = await StartConsumerAsync();
+ var first = Assert.Single(context.Consumer.Sessions);
+ var firstChannel = Assert.IsType(first.Channel);
+ firstChannel.IsOpen = false;
+
+ await context.Consumer.ReconcileAsync(CancellationToken.None);
+
+ var replacement = Assert.Single(context.Consumer.Sessions);
+ Assert.NotSame(first, replacement);
+ Assert.Equal(ArticleWorkConsumerState.Running, replacement.State);
+ Assert.True(replacement.Channel!.IsOpen);
+ Assert.Equal(ArticleWorkConsumerState.Stopped, first.State);
+ Assert.Equal(1, firstChannel.DisposeCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// A late fault callback from a retired channel does not retire its replacement.
+ ///
+ [Fact]
+ public async Task Late_fault_from_old_session_does_not_retire_replacement()
+ {
+ await using var context = await StartConsumerAsync();
+ var first = Assert.Single(context.Consumer.Sessions);
+ var firstChannel = Assert.IsType(first.Channel);
+ firstChannel.SimulateChannelShutdown();
+
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Consumer.Sessions.Count == 1 && !ReferenceEquals(context.Consumer.Sessions[0], first),
+ TimeSpan.FromSeconds(2));
+
+ var replacement = Assert.Single(context.Consumer.Sessions);
+ var replacementChannel = Assert.IsType(replacement.Channel);
+
+ firstChannel.SimulateChannelShutdown("stale late callback");
+
+ await Task.Yield();
+ Assert.Same(replacement, Assert.Single(context.Consumer.Sessions));
+ Assert.Equal(ArticleWorkConsumerState.Running, replacement.State);
+ Assert.Same(replacementChannel, replacement.Channel);
+ Assert.Equal(0, replacementChannel.DisposeCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// Repeated fault notifications for one session retire it once and leave a single replacement.
+ ///
+ [Fact]
+ public async Task Repeated_channel_faults_do_not_duplicate_active_consumers()
+ {
+ await using var context = await StartConsumerAsync();
+ var first = Assert.Single(context.Consumer.Sessions);
+ var firstChannel = Assert.IsType(first.Channel);
+
+ firstChannel.SimulateChannelShutdown("first");
+ firstChannel.SimulateConsumerCancelled("second");
+ firstChannel.SimulateChannelShutdown("third");
+
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Consumer.Sessions.Count == 1
+ && !ReferenceEquals(context.Consumer.Sessions[0], first)
+ && context.Consumer.Sessions[0].State == ArticleWorkConsumerState.Running,
+ TimeSpan.FromSeconds(2));
+
+ Assert.Single(context.Consumer.Sessions);
+ Assert.Equal(1, firstChannel.DisposeCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ Assert.True(context.Factory.LastConnection!.Channels.Count > 1);
+ }
+
+ ///
+ /// Expected retirement cancel does not raise a fault-driven reconnect of the AMQP connection.
+ ///
+ [Fact]
+ public async Task Expected_retirement_does_not_open_a_new_connection()
+ {
+ await using var context = await StartConsumerAsync();
+ var session = Assert.Single(context.Consumer.Sessions);
+ var channel = Assert.IsType(session.Channel);
+
+ await session.RetireAsync();
+
+ Assert.Equal(ArticleWorkConsumerState.Stopped, session.State);
+ Assert.Equal(1, channel.CancelCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// Application shutdown retires sessions without starting replacements or reconnecting.
+ ///
+ [Fact]
+ public async Task Application_shutdown_does_not_start_channel_fault_replacements()
+ {
+ await using var context = await StartConsumerAsync();
+ var session = Assert.Single(context.Consumer.Sessions);
+ var channel = Assert.IsType(session.Channel);
+
+ await context.Consumer.DisposeAsync();
+
+ Assert.Empty(context.Consumer.Sessions);
+ Assert.Equal(ArticleWorkConsumerState.Stopped, session.State);
+ Assert.Equal(1, channel.DisposeCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+
+ channel.SimulateChannelShutdown("after shutdown");
+ await Task.Yield();
+ Assert.Empty(context.Consumer.Sessions);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// In-flight delivery admitted before an unexpected channel fault completes without ACK/NACK on the dead channel
+ /// (broker redelivery) and without a second settlement attempt after retirement.
+ ///
+ [Fact]
+ public async Task In_flight_delivery_during_channel_fault_is_left_unsettled_for_redelivery()
+ {
+ var handler = new ControllableArticleWorkHandler
+ {
+ Outcome = ArticleWorkOutcome.ArticleNotFound,
+ Error = "missing",
+ Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously),
+ Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously),
+ };
+ await using var context = await StartConsumerAsync(handler);
+ var session = Assert.Single(context.Consumer.Sessions);
+ var channel = Assert.IsType(session.Channel);
+ var processing = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical());
+ await handler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(2));
+
+ channel.SimulateChannelShutdown();
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => session.State == ArticleWorkConsumerState.Retiring
+ || session.State == ArticleWorkConsumerState.Stopped,
+ TimeSpan.FromSeconds(2));
+
+ handler.Gate!.TrySetResult();
+ await processing.WaitAsync(TimeSpan.FromSeconds(2));
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => session.State == ArticleWorkConsumerState.Stopped
+ && context.Consumer.Sessions.Count == 1
+ && !ReferenceEquals(context.Consumer.Sessions[0], session),
+ TimeSpan.FromSeconds(2));
+
+ Assert.Empty(channel.Settlements);
+ Assert.Equal(1, channel.DisposeCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// Publish-channel shutdown installs a replacement on the same connection generation.
+ ///
+ [Fact]
+ public async Task Publisher_channel_shutdown_replaces_channel_without_reconnect()
+ {
+ await using var context = await StartPublisherAsync();
+ var first = context.PublishChannel;
+ Assert.Equal(1, context.Factory.ConnectCount);
+
+ first.SimulateChannelShutdown();
+
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Publisher.Channel is not null
+ && !ReferenceEquals(context.Publisher.Channel, first)
+ && context.Publisher.Channel.IsOpen,
+ TimeSpan.FromSeconds(2));
+
+ Assert.Equal(1, first.DisposeCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ Assert.Equal(1, context.Publisher.Generation);
+ Assert.Equal(2, context.Factory.LastConnection!.PublishChannels.Count);
+ }
+
+ ///
+ /// After publish-channel replacement, the next publish succeeds through the new channel.
+ ///
+ [Fact]
+ public async Task Publish_after_channel_replacement_uses_the_new_channel()
+ {
+ await using var context = await StartPublisherAsync();
+ var first = context.PublishChannel;
+ first.SimulateChannelShutdown();
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Publisher.Channel is not null && !ReferenceEquals(context.Publisher.Channel, first),
+ TimeSpan.FromSeconds(2));
+
+ var replacement = context.PublishChannel;
+ await context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None);
+
+ Assert.Empty(first.Publications);
+ Assert.Single(replacement.Publications);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// A late fault from an obsolete publish channel does not dispose the installed replacement.
+ ///
+ [Fact]
+ public async Task Late_publish_channel_fault_does_not_invalidate_replacement()
+ {
+ await using var context = await StartPublisherAsync();
+ var first = context.PublishChannel;
+ first.SimulateChannelShutdown();
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Publisher.Channel is not null && !ReferenceEquals(context.Publisher.Channel, first),
+ TimeSpan.FromSeconds(2));
+
+ var replacement = context.PublishChannel;
+ first.SimulateChannelShutdown("stale");
+
+ await Task.Yield();
+ Assert.Same(replacement, context.Publisher.Channel);
+ Assert.True(replacement.IsOpen);
+ Assert.Equal(0, replacement.DisposeCount);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// Concurrent publishes wait for channel rebuild rather than racing duplicate replacements unsafely.
+ ///
+ [Fact]
+ public async Task Concurrent_publishes_after_channel_fault_share_one_replacement()
+ {
+ await using var context = await StartPublisherAsync();
+ var first = context.PublishChannel;
+ first.SimulateChannelShutdown();
+
+ var publish1 = context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None);
+ var publish2 = context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None);
+ await Task.WhenAll(publish1, publish2).WaitAsync(TimeSpan.FromSeconds(2));
+
+ Assert.Equal(1, first.DisposeCount);
+ Assert.Equal(2, context.Factory.LastConnection!.PublishChannels.Count);
+ Assert.Equal(2, context.PublishChannel.Publications.Count);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// Failed channel replacement does not report publish success.
+ ///
+ [Fact]
+ public async Task Failed_publish_channel_replacement_does_not_report_success()
+ {
+ await using var context = await StartPublisherAsync();
+ var first = context.PublishChannel;
+ context.Factory.LastConnection!.CreatePublishChannelException =
+ new InvalidOperationException("no publish channel");
+ first.SimulateChannelShutdown();
+
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Publisher.Channel is null || !context.Publisher.Channel.IsOpen,
+ TimeSpan.FromSeconds(2));
+
+ await Assert.ThrowsAsync(
+ () => context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None));
+ Assert.Empty(first.Publications);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ }
+
+ ///
+ /// Caller cancellation during publish wait does not leave a permanently closed installed channel when rebuild already completed.
+ ///
+ [Fact]
+ public async Task Caller_cancellation_does_not_trigger_unwanted_connection_recovery()
+ {
+ await using var context = await StartPublisherAsync();
+ using var cts = new CancellationTokenSource();
+ cts.Cancel();
+
+ await Assert.ThrowsAnyAsync(
+ () => context.Publisher.PublishAsync(SuccessIntent(), cts.Token));
+
+ Assert.Equal(1, context.Factory.ConnectCount);
+ Assert.True(context.PublishChannel.IsOpen);
+ }
+
+ ///
+ /// Publisher disposal prevents channel-fault rebuild work after stop.
+ ///
+ [Fact]
+ public async Task Publisher_shutdown_ignores_late_channel_faults()
+ {
+ await using var context = await StartPublisherAsync();
+ var channel = context.PublishChannel;
+ await context.Publisher.DisposeAsync();
+
+ channel.SimulateChannelShutdown();
+ await Task.Yield();
+
+ Assert.Equal(ArticleWorkResponsePublisherState.Stopped, context.Publisher.State);
+ Assert.Null(context.Publisher.Channel);
+ Assert.Equal(1, context.Factory.ConnectCount);
+ Assert.Single(context.Factory.LastConnection!.PublishChannels);
+ }
+
+ ///
+ /// Dispose while a channel-fault rebuild is blocked, then connection replacement is chained, completes without
+ /// deadlock or an unobserved replace-gate waiter after the publisher disposes its semaphores.
+ ///
+ [Fact]
+ public async Task Publisher_shutdown_during_chained_channel_and_connection_rebuild_does_not_deadlock()
+ {
+ await using var context = await StartPublisherAsync();
+ var first = context.PublishChannel;
+ var firstConnection = context.Factory.LastConnection!;
+ firstConnection.CreatePublishChannelStarted =
+ new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ firstConnection.BlockCreatePublishChannel =
+ new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ first.SimulateChannelShutdown();
+ await firstConnection.CreatePublishChannelStarted.Task.WaitAsync(TimeSpan.FromSeconds(2));
+
+ var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ context.Factory.Connected = secondConnected;
+ firstConnection.SimulateLost();
+ await secondConnected.Task.WaitAsync(TimeSpan.FromSeconds(2));
+
+ var disposing = context.Publisher.DisposeAsync().AsTask();
+ firstConnection.BlockCreatePublishChannel.TrySetResult();
+ await disposing.WaitAsync(TimeSpan.FromSeconds(2));
+
+ Assert.Equal(ArticleWorkResponsePublisherState.Stopped, context.Publisher.State);
+ Assert.Null(context.Publisher.Channel);
+ }
+
+ ///
+ /// Connection replacement that races with channel-only recovery still installs a channel on the new generation.
+ ///
+ [Fact]
+ public async Task Connection_replacement_chained_after_channel_fault_installs_new_generation()
+ {
+ await using var context = await StartPublisherAsync();
+ var first = context.PublishChannel;
+ var firstConnection = context.Factory.LastConnection!;
+ firstConnection.CreatePublishChannelStarted =
+ new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ firstConnection.BlockCreatePublishChannel =
+ new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ first.SimulateChannelShutdown();
+ await firstConnection.CreatePublishChannelStarted.Task.WaitAsync(TimeSpan.FromSeconds(2));
+
+ var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ context.Factory.Connected = secondConnected;
+ firstConnection.SimulateLost();
+ await secondConnected.Task.WaitAsync(TimeSpan.FromSeconds(2));
+
+ firstConnection.BlockCreatePublishChannel.TrySetResult();
+
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => context.Publisher.Generation == 2
+ && context.Publisher.Channel is { IsOpen: true },
+ TimeSpan.FromSeconds(2));
+
+ Assert.NotSame(first, context.Publisher.Channel);
+ Assert.Equal(2, context.Factory.ConnectCount);
+ await context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None);
+ Assert.Single(context.PublishChannel.Publications);
+ }
+
+ private static ArticleWorkResponseIntent SuccessIntent() =>
+ new(
+ ArticleWorkOutcome.Success,
+ Guid.Parse(ArticleWorkTestDeliveries.CanonicalRequestId),
+ ArticleWorkTestDeliveries.CanonicalMessageId,
+ "Giganews",
+ ArticleWorkTestDeliveries.CanonicalCorrelationId,
+ ArticleWorkTestDeliveries.CanonicalReplyTo,
+ Error: null,
+ ArticleWorkTestDeliveries.CanonicalFqdn,
+ ArticleWorkTestDeliveries.CanonicalListenerPort,
+ ArticleWorkTestDeliveries.CanonicalArticleIdHex);
+
+ private static async Task StartConsumerAsync(IArticleWorkHandler? handler = null)
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ var consumer = new ArticleWorkConsumerService(
+ connections,
+ RabbitMqServiceTests.CreateFastRuntime(),
+ handler ?? new DeferredArticleWorkHandler(),
+ new RecordingArticleWorkResponsePublisher(),
+ NullLogger.Instance,
+ catalog: new StaticBackFillerProviderCatalog(
+ [
+ new BackFillerProviderDefinition(
+ "Giganews",
+ "127.0.0.1",
+ 119,
+ false,
+ "nntp-user",
+ "p",
+ 0,
+ 1),
+ ]),
+ capacity: CreateCapacity("Giganews", 1));
+ await consumer.StartAsync(CancellationToken.None);
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => consumer.Sessions.Count == 1 && consumer.Sessions[0].State == ArticleWorkConsumerState.Running,
+ TimeSpan.FromSeconds(2));
+ return new ConsumerHarness(factory, connections, consumer);
+ }
+
+ private static async Task StartPublisherAsync()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory
+ {
+ DefaultPublishConfirmBehavior = FakePublishConfirmBehavior.Confirm,
+ };
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ factory.LastConnection!.DefaultPublishConfirmBehavior = FakePublishConfirmBehavior.Confirm;
+ var publisher = new ArticleWorkResponsePublisher(
+ connections,
+ RabbitMqServiceTests.CreateFastRuntime(),
+ NullLogger.Instance);
+ await publisher.StartAsync(CancellationToken.None);
+ Assert.Equal(ArticleWorkResponsePublisherState.Running, publisher.State);
+ return new PublisherHarness(factory, connections, publisher);
+ }
+
+ private static BackboneUsableCapacityState CreateCapacity(string backbone, int active)
+ {
+ var capacity = new BackboneUsableCapacityState();
+ if (active > 0)
+ {
+ capacity.PublishSnapshot(new Dictionary(StringComparer.OrdinalIgnoreCase)
+ {
+ [backbone] = active,
+ });
+ }
+
+ return capacity;
+ }
+
+ private sealed class ConsumerHarness(
+ FakeBackFillerRabbitMqConnectionFactory factory,
+ RabbitMqService connections,
+ ArticleWorkConsumerService consumer) : IAsyncDisposable
+ {
+ public FakeBackFillerRabbitMqConnectionFactory Factory { get; } = factory;
+
+ public RabbitMqService Connections { get; } = connections;
+
+ public ArticleWorkConsumerService Consumer { get; } = consumer;
+
+ public async ValueTask DisposeAsync()
+ {
+ await Consumer.DisposeAsync();
+ await Connections.DisposeAsync();
+ }
+ }
+
+ private sealed class PublisherHarness(
+ FakeBackFillerRabbitMqConnectionFactory factory,
+ RabbitMqService connections,
+ ArticleWorkResponsePublisher publisher) : IAsyncDisposable
+ {
+ public FakeBackFillerRabbitMqConnectionFactory Factory { get; } = factory;
+
+ public RabbitMqService Connections { get; } = connections;
+
+ public ArticleWorkResponsePublisher Publisher { get; } = publisher;
+
+ public FakeBackFillerRabbitMqPublishChannel PublishChannel =>
+ Assert.IsType(Publisher.Channel);
+
+ public async ValueTask DisposeAsync()
+ {
+ await Publisher.DisposeAsync();
+ await Connections.DisposeAsync();
+ }
+ }
+ }
+}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs
index 122c7964..bd61b600 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs
@@ -101,6 +101,13 @@ private sealed class TestManualAckChannel : IRabbitMqManualAckChannel
///
public bool IsOpen => true;
+ ///
+ public event EventHandler? ChannelFaulted
+ {
+ add { }
+ remove { }
+ }
+
///
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqChannelShutdownTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqChannelShutdownTests.cs
index 4c557aba..c9c38ef9 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqChannelShutdownTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqChannelShutdownTests.cs
@@ -77,6 +77,11 @@ internal static ChannelDouble Create(Exception closeException)
protected override object? Invoke(MethodInfo? targetMethod, object?[]? args)
{
ArgumentNullException.ThrowIfNull(targetMethod);
+ if (targetMethod.Name is "add_ChannelShutdownAsync" or "remove_ChannelShutdownAsync")
+ {
+ return null;
+ }
+
if (targetMethod.Name == "get_IsOpen")
{
return true;
@@ -93,9 +98,11 @@ internal static ChannelDouble Create(Exception closeException)
return ValueTask.CompletedTask;
}
- if (targetMethod.ReturnType == typeof(ValueTask))
+ if (targetMethod.ReturnType == typeof(void) || targetMethod.ReturnType == typeof(ValueTask))
{
- return ValueTask.CompletedTask;
+ return targetMethod.ReturnType == typeof(ValueTask)
+ ? ValueTask.CompletedTask
+ : null;
}
if (typeof(Task).IsAssignableFrom(targetMethod.ReturnType))
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqClientManualAckChannelTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqClientManualAckChannelTests.cs
index 248454a1..d9f92620 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqClientManualAckChannelTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqClientManualAckChannelTests.cs
@@ -436,6 +436,11 @@ internal string[] Snapshot()
protected override object? Invoke(MethodInfo? targetMethod, object?[]? args)
{
ArgumentNullException.ThrowIfNull(targetMethod);
+ if (targetMethod.Name is "add_ChannelShutdownAsync" or "remove_ChannelShutdownAsync")
+ {
+ return null;
+ }
+
if (targetMethod.Name == "get_IsOpen")
{
return IsOpen;
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
index 6e11c7fe..32f3d91f 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
@@ -363,6 +363,9 @@ internal sealed class FakeBackFillerRabbitMqChannel(long generation) : IRabbitMq
/// Writable open flag; when false, consume and settlement operations throw.
public bool IsOpen { get; set; } = true;
+ /// Raised by / ; not wired to a real broker.
+ public event EventHandler? ChannelFaulted;
+
/// Number of exclusive dispose passes completed on this channel.
public int DisposeCount { get; private set; }
@@ -561,7 +564,7 @@ public Task BasicConsumeAsync(
return Task.FromResult(ConsumerTag);
}
- /// Increments without unregistering the consumer callback.
+ /// Increments without unregistering the consumer callback or raising .
/// Validated but not matched against .
/// Honored before incrementing the counter.
public Task BasicCancelAsync(string consumerTag, CancellationToken cancellationToken)
@@ -572,6 +575,32 @@ public Task BasicCancelAsync(string consumerTag, CancellationToken cancellationT
return Task.CompletedTask;
}
+ ///
+ /// Marks the channel closed and raises as an unexpected channel shutdown.
+ /// Leaves the process connection open.
+ ///
+ /// Fault reason text.
+ internal void SimulateChannelShutdown(string reason = "channel closed by broker")
+ {
+ IsOpen = false;
+ ChannelFaulted?.Invoke(
+ this,
+ new RabbitMqChannelFaultedEventArgs(RabbitMqChannelFaultKind.ChannelShutdown, reason));
+ }
+
+ ///
+ /// Marks the channel closed and raises as a broker-originated consumer cancellation.
+ /// Leaves the process connection open.
+ ///
+ /// Fault reason text.
+ internal void SimulateConsumerCancelled(string reason = "consumer cancelled by broker")
+ {
+ IsOpen = false;
+ ChannelFaulted?.Invoke(
+ this,
+ new RabbitMqChannelFaultedEventArgs(RabbitMqChannelFaultKind.ConsumerCancelled, reason));
+ }
+
///
/// Runs ack gates, optionally throws , then appends an ack settlement.
/// Tracks broker-op overlap via while inside the exclusive section.
@@ -760,6 +789,9 @@ internal sealed class FakeBackFillerRabbitMqPublishChannel(long generation) : IR
/// Writable open flag; when false, publish throws; some confirm failures also clear this flag.
public bool IsOpen { get; set; } = true;
+ /// Raised by ; not wired to a real broker.
+ public event EventHandler? ChannelFaulted;
+
/// Number of dispose calls completed on this publish channel.
public int DisposeCount { get; private set; }
@@ -848,6 +880,11 @@ public async Task PublishConfirmedAsync(RabbitMqConfirmedPublication publication
throw new OperationCanceledException("publisher confirm timeout", cancellationToken);
case FakePublishConfirmBehavior.CloseChannel:
IsOpen = false;
+ ChannelFaulted?.Invoke(
+ this,
+ new RabbitMqChannelFaultedEventArgs(
+ RabbitMqChannelFaultKind.ChannelShutdown,
+ "channel closed during confirmation"));
throw new InvalidOperationException("RabbitMQ publish channel closed during confirmation.");
default:
if (ConfirmGate is not null)
@@ -864,6 +901,19 @@ public async Task PublishConfirmedAsync(RabbitMqConfirmedPublication publication
/// When set, thrown after incrementing dispose count and clearing .
public Exception? DisposeException { get; set; }
+ ///
+ /// Marks the channel closed and raises as an unexpected channel shutdown.
+ /// Leaves the process connection open.
+ ///
+ /// Fault reason text.
+ internal void SimulateChannelShutdown(string reason = "publish channel closed by broker")
+ {
+ IsOpen = false;
+ ChannelFaulted?.Invoke(
+ this,
+ new RabbitMqChannelFaultedEventArgs(RabbitMqChannelFaultKind.ChannelShutdown, reason));
+ }
+
/// Increments dispose count, marks the channel closed, and optionally throws .
public ValueTask DisposeAsync()
{
diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs
index d69c6497..cd6f4577 100644
--- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs
+++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs
@@ -982,6 +982,13 @@ private sealed class LoggingPublishChannel(long generation, LoggingConnection co
///
public bool IsOpen { get; private set; } = true;
+ ///
+ public event EventHandler? ChannelFaulted
+ {
+ add { }
+ remove { }
+ }
+
///
/// Number of calls.
///