diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs index 22f5d7e9..7b570bf2 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs @@ -528,6 +528,32 @@ private void OnCapacitySnapshotPublished(object? sender, EventArgs eventArgs) _ = ReconcileObservedAsync(); } + /// + /// Schedules an observed reconcile after a running session reports an unexpected channel fault. + /// Does not open a new AMQP connection. + /// + private void OnSessionChannelFaulted() + { + if (!IsRunning) + { + return; + } + + _ = ReconcileObservedAsync(); + } + + /// + /// Returns whether a tracked session still owns an open broker consumer. + /// alone is not sufficient after channel failure. + /// + /// Tracked session candidate. + /// + /// when the session is running and its consume channel reports itself open. + /// + private static bool IsHealthyRunningSession(ArticleWorkConsumerSession session) => + session.State == ArticleWorkConsumerState.Running + && session.Channel is { IsOpen: true }; + /// Reconciles after a capacity snapshot. /// /// A task that completes when finishes or its exception has been logged. @@ -558,8 +584,9 @@ private async Task ReconcileObservedAsync() /// Sessions whose topology did declare are still started. /// /// - /// A tracked running session whose key is still desired is kept. Any other tracked session whose key is no longer desired is removed and retired. - /// A desired key with no running tracked session is started. Prefetch is the configured positive count, otherwise 1. + /// A tracked running session whose key is still desired and whose consume channel is open is kept. + /// Any other tracked session is removed and retired, including a Running session whose channel is closed. + /// A desired key with no healthy tracked session is started. Prefetch is the configured positive count, otherwise 1. /// Sessions started in one pass share a new pipeline. A start failure disposes sessions started in that pass and propagates; they are not tracked. /// If shutdown is set before publication, those new sessions are disposed of instead of tracked. /// No desired slots is success. A desired slot whose topology is not ready is not success and does not log completion. @@ -581,17 +608,14 @@ private async Task ReconcileSessionsAsync(CancellationToken cance { foreach (var pair in _sessions.ToArray()) { - if (desired.ContainsKey(pair.Key) && pair.Value.State == ArticleWorkConsumerState.Running) + if (desired.ContainsKey(pair.Key) && IsHealthyRunningSession(pair.Value)) { desired.Remove(pair.Key); continue; } - if (!desired.ContainsKey(pair.Key)) - { - _sessions.Remove(pair.Key); - stale.Add(pair.Value); - } + _sessions.Remove(pair.Key); + stale.Add(pair.Value); } } @@ -637,7 +661,10 @@ private async Task ReconcileSessionsAsync(CancellationToken cance _logger, _runtime.Shutdown, identity.ConnectionNumber, - identity.ConnectionLimit); + identity.ConnectionLimit) + { + UnexpectedChannelFault = OnSessionChannelFaulted, + }; await session.StartAsync(cancellationToken).ConfigureAwait(false); started.Add(session); } diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs index 3c2002ea..54c46502 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs @@ -98,6 +98,12 @@ internal sealed class ArticleWorkConsumerSession : IAsyncDisposable /// One after has entered disposal. private int _disposed; + /// + /// Optional callback invoked when an unexpected channel fault retires this session while it was running. + /// The consumer service uses it to schedule an observed reconcile without reconnecting. + /// + internal Action? UnexpectedChannelFault { get; set; } + /// /// Initializes a new session with the default drain-and-finish shutdown policy for connection slot 1 of 1. /// @@ -316,22 +322,36 @@ internal async Task StartAsync(CancellationToken cancellationToken) .BasicConsumeAsync(_queue, _prefetch, OnDeliveryAsync, cancellationToken) .ConfigureAwait(false); + // Install identity under the gate before subscribing so a fault callback can match this session. + // A ChannelFaulted raised in the gap before subscribe is recovered by the IsOpen check below. + IRabbitMqManualAckChannel installed; lock (_gate) { - _channel = channel; + installed = channel; + _channel = installed; _consumerTag = tag; Generation = handle.Generation; _state = ArticleWorkConsumerState.Running; channel = null; } + installed.ChannelFaulted += OnChannelFaulted; ArticleWorkLogMessages.ConsumerRunning(_logger, _backbone, _queue, Generation, tag); + if (!installed.IsOpen) + { + OnChannelFaulted( + installed, + new RabbitMqChannelFaultedEventArgs( + RabbitMqChannelFaultKind.ChannelShutdown, + "Channel closed during consumer start.")); + } } catch (Exception ex) { ArticleWorkLogMessages.ConsumerStartFailed(_logger, _backbone, _queue, ex.Message); if (channel is not null) { + channel.ChannelFaulted -= OnChannelFaulted; await channel.DisposeAsync().ConfigureAwait(false); } @@ -500,6 +520,7 @@ private async Task RetireCoreAsync( if (channel is not null) { + channel.ChannelFaulted -= OnChannelFaulted; await channel.DisposeAsync().ConfigureAwait(false); } @@ -513,6 +534,35 @@ private async Task RetireCoreAsync( ArticleWorkLogMessages.ConsumerStopped(_logger, _backbone, Generation); } + /// + /// Retires this session when its installed channel reports an unexpected fault, then notifies the service to reconcile. + /// + /// Faulted channel. Ignored when it is not the installed channel. + /// Fault classification and reason. + /// + /// Late callbacks from a retired channel are ignored via identity check. + /// Expected application cancel and disposal do not raise . + /// + private void OnChannelFaulted(object? sender, RabbitMqChannelFaultedEventArgs eventArgs) + { + lock (_gate) + { + if (!ReferenceEquals(_channel, sender) || _state != ArticleWorkConsumerState.Running) + { + return; + } + } + + ArticleWorkLogMessages.ConsumerChannelFaulted( + _logger, + _backbone, + Generation, + eventArgs.Kind, + eventArgs.Reason); + UnexpectedChannelFault?.Invoke(); + _ = RetireAsync(); + } + /// /// Issues Basic.Cancel and contains any failure, so retirement still drains and disposes the channel. /// diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs index 60c29851..c41a316b 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs @@ -1,3 +1,5 @@ +using VectorNNTPD.Common.Messaging.RabbitMq; + namespace VectorNNTPD.BackFillerService.ArticleWork { /// @@ -353,5 +355,41 @@ internal static partial void ArticleProcessed( Level = LogLevel.Error, Message = "Article Work reconcile loop failed and will retry: {Reason}")] internal static partial void ConsumerReconcileFailed(ILogger logger, string Reason); + + /// + /// Writes a Warning event when a running consumer session observes an unexpected channel fault or broker consumer cancellation. + /// + /// Logger that receives the event. + /// Provider backbone label. + /// Connection generation of the faulted channel. + /// Fault classification. + /// Broker or client reason text. + [LoggerMessage( + EventId = 5323, + Level = LogLevel.Warning, + Message = "Article Work consumer channel faulted backbone={Backbone} generation={Generation} kind={Kind}: {Reason}")] + internal static partial void ConsumerChannelFaulted( + ILogger logger, + string backbone, + long generation, + RabbitMqChannelFaultKind kind, + string reason); + + /// + /// Writes a Warning event when the response publisher observes an unexpected publish-channel fault and will rebuild on the current connection. + /// + /// Logger that receives the event. + /// Connection generation of the faulted channel. + /// Fault classification. + /// Broker or client reason text. + [LoggerMessage( + EventId = 5324, + Level = LogLevel.Warning, + Message = "Article Work publish channel faulted generation={Generation} kind={Kind}: {Reason}")] + internal static partial void PublisherChannelFaulted( + ILogger logger, + long generation, + RabbitMqChannelFaultKind kind, + string reason); } } diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs index 0f927d21..3eaa7627 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs @@ -1,5 +1,6 @@ using VectorNNTPD.BackFillerService.Configuration; using VectorNNTPD.BackFillerService.Logging; +using VectorNNTPD.Common.Core; using VectorNNTPD.Common.Messaging.RabbitMq; namespace VectorNNTPD.BackFillerService.ArticleWork @@ -9,8 +10,9 @@ namespace VectorNNTPD.BackFillerService.ArticleWork /// /// /// Startup fails if a publishing channel cannot be opened on the current generation. - /// After start, connection replacement rebuilds the publisher channel. This type - /// does not implement a second connection-recovery loop and never ACK/NACKs deliveries. + /// After start, connection replacement and independent publish-channel faults rebuild the publisher channel + /// on the current connection generation. This type does not implement a second connection-recovery loop + /// and never ACK/NACKs deliveries. /// Completing means the broker confirmed and the publication generation was still current afterward. /// Confirmation is not permission to ACK the original delivery. /// One publication holds . The confirmation wait is cancelled by the caller token, disposal, or the configured confirmation timeout. @@ -213,6 +215,7 @@ public async ValueTask DisposeAsync() ArticleWorkLogMessages.PublisherRetiring(_logger, Generation); _connections.ConnectionReplaced -= OnConnectionReplaced; + DetachPublishChannelFaultHandler(); await _runCts.CancelAsync().ConfigureAwait(false); Task replace; @@ -306,6 +309,7 @@ public async Task PublishAsync(ArticleWorkResponseIntent intent, CancellationTok try { ThrowIfNotRunning(); + await EnsureOpenPublishChannelAsync(linked.Token).ConfigureAwait(false); var channel = CurrentOpenChannelOrThrow(); generation = channel.Generation; if (!IsPublisherGenerationCurrent(generation)) @@ -400,6 +404,12 @@ internal async Task InstallPublishChannelAsync(IRabbitMqPublishChannel candidate else { replaced = _channel; + if (replaced is not null) + { + replaced.ChannelFaulted -= OnPublishChannelFaulted; + } + + candidate.ChannelFaulted += OnPublishChannelFaulted; _channel = candidate; } } @@ -429,6 +439,45 @@ private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEven return; } + ScheduleChannelRebuild(); + } + + /// + /// Schedules a publish-channel rebuild when the installed channel faults independently of connection replacement. + /// + /// Faulted channel. Ignored when it is not the installed channel. + /// Fault classification and reason. + private void OnPublishChannelFaulted(object? sender, RabbitMqChannelFaultedEventArgs eventArgs) + { + long generation; + lock (_gate) + { + if (_stopping || _channel is null || !ReferenceEquals(_channel, sender)) + { + return; + } + + generation = _channel.Generation; + } + + ArticleWorkLogMessages.PublisherChannelFaulted( + _logger, + generation, + eventArgs.Kind, + eventArgs.Reason); + ScheduleChannelRebuild(); + } + + /// + /// Chains a non-cancelling rebuild after any in-flight rebuild when disposal has not started. + /// + /// + /// Overwriting with a sibling left prior + /// waiters on unobserved; disposal could then dispose the semaphore while those + /// waiters were still queued. Chaining keeps every rebuild on the task Dispose awaits. + /// + private void ScheduleChannelRebuild() + { lock (_gate) { if (_stopping) @@ -436,16 +485,39 @@ private void OnConnectionReplaced(object? sender, RabbitMqConnectionReplacedEven return; } - _replaceTask = RebuildAfterReplacementAsync(); + _replaceTask = ContinueRebuildAsync(_replaceTask); } } - /// Rebuilds the publication channel after a connection replacement unless the startup never succeeded or disposal has started. + /// + /// Awaits then runs one rebuild so later connection or channel-fault requests are not dropped. + /// + /// + /// Earlier rebuild or completed task. Non-fatal failures are ignored here because + /// already logged them; failures propagate. + /// + /// A task that completes when the follow-on rebuild finishes. + private async Task ContinueRebuildAsync(Task previous) + { + try + { + await previous.ConfigureAwait(false); + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + // Prior rebuild logged at RebuildPublishChannelAsync; continue so a later request is not dropped. + } + + await RebuildPublishChannelAsync().ConfigureAwait(false); + } + + /// Rebuilds the publication channel on the current connection unless startup never succeeded or disposal has started. /// A task that completes when the rebuild attempt finishes. Failures are logged and not rethrown. /// /// Waits on without a cancellation token, then opens the channel with . + /// Used for both connection replacement and independent channel faults. /// - private async Task RebuildAfterReplacementAsync() + private async Task RebuildPublishChannelAsync() { await _replaceGate.WaitAsync().ConfigureAwait(false); try @@ -474,6 +546,48 @@ private async Task RebuildAfterReplacementAsync() } } + /// + /// Waits for any in-flight rebuild and, when the installed channel is closed, schedules another rebuild on the current connection. + /// + /// Cancels the wait for an in-flight rebuild. + /// + /// Does not take while holding ; it only awaits the tracked rebuild task. + /// + private async Task EnsureOpenPublishChannelAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Task replace; + var open = false; + lock (_gate) + { + if (_channel is { IsOpen: true }) + { + open = true; + replace = Task.CompletedTask; + } + else if (_stopping) + { + replace = Task.CompletedTask; + } + else if (!_replaceTask.IsCompleted) + { + replace = _replaceTask; + } + else + { + _replaceTask = ContinueRebuildAsync(_replaceTask); + replace = _replaceTask; + } + } + + if (open) + { + return; + } + + await replace.WaitAsync(cancellationToken).ConfigureAwait(false); + } + /// Opens a publication channel on the current connection and installs it when the generation is still current. /// /// when the caller is . A missing connection then throws. @@ -525,6 +639,11 @@ private async Task DisposeChannelAsync() lock (_gate) { channel = _channel; + if (channel is not null) + { + channel.ChannelFaulted -= OnPublishChannelFaulted; + } + _channel = null; } @@ -534,6 +653,18 @@ private async Task DisposeChannelAsync() } } + /// Unsubscribes the fault handler from the installed channel without disposing it. + private void DetachPublishChannelFaultHandler() + { + lock (_gate) + { + if (_channel is not null) + { + _channel.ChannelFaulted -= OnPublishChannelFaulted; + } + } + } + /// /// Waits until can be acquired, then releases it so disposal can dispose of the semaphore. /// diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs index 5a87ffa4..646689b8 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs @@ -20,6 +20,12 @@ internal interface IRabbitMqManualAckChannel : IAsyncDisposable /// bool IsOpen { get; } + /// + /// Raised once when this channel becomes unexpectedly unusable while the process connection may still be open. + /// Expected application cancel and disposal do not raise this event. + /// + event EventHandler? ChannelFaulted; + /// /// Declares an exchange. Idempotent for compatible existing entities; incompatible /// existing entities fail closed (never deleted or mutated). diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqPublishChannel.cs index 72d42700..4df1bdd7 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqPublishChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqPublishChannel.cs @@ -16,6 +16,12 @@ internal interface IRabbitMqPublishChannel : IAsyncDisposable /// Gets whether the channel currently reports itself open. bool IsOpen { get; } + /// + /// Raised once when this publish channel becomes unexpectedly unusable while the process connection may still be open. + /// Disposal does not raise this event. + /// + event EventHandler? ChannelFaulted; + /// /// Publishes one message described by and waits for a publisher confirmation. /// diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultKind.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultKind.cs new file mode 100644 index 00000000..6045421c --- /dev/null +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultKind.cs @@ -0,0 +1,12 @@ +namespace VectorNNTPD.Common.Messaging.RabbitMq +{ + /// Why a caller-owned AMQP channel became unusable while the process connection may still be open. + internal enum RabbitMqChannelFaultKind + { + /// The AMQP channel shut down (broker or peer close independent of connection loss). + ChannelShutdown = 0, + + /// The broker cancelled the consumer tag without an application basic.cancel. + ConsumerCancelled = 1, + } +} diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultedEventArgs.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultedEventArgs.cs new file mode 100644 index 00000000..f987e55e --- /dev/null +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqChannelFaultedEventArgs.cs @@ -0,0 +1,7 @@ +namespace VectorNNTPD.Common.Messaging.RabbitMq +{ + /// Details for a single unexpected channel or consumer fault. + /// Fault classification. + /// Broker or client reason text; never credentials or payload bytes. + internal sealed record RabbitMqChannelFaultedEventArgs(RabbitMqChannelFaultKind Kind, string Reason); +} diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs index 0526f4c1..f55622e3 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs @@ -42,6 +42,15 @@ internal sealed class RabbitMqClientManualAckChannel : IRabbitMqManualAckChannel /// 1 after the first has marked the channel disposed. private int _disposed; + /// 1 after an unexpected fault has been raised once for this channel. + private int _faultRaised; + + /// + /// 1 after this wrapper initiates so broker + /// is treated as expected. + /// + private int _applicationCancel; + /// /// Initializes a new wrapper around an opened channel. /// @@ -52,11 +61,15 @@ internal RabbitMqClientManualAckChannel(IChannel channel, long generation) ArgumentNullException.ThrowIfNull(channel); _channel = channel; Generation = generation; + _channel.ChannelShutdownAsync += OnChannelShutdownAsync; } /// public long Generation { get; } + /// + public event EventHandler? ChannelFaulted; + /// Threads currently waiting to enter consumer registration or disposal. internal int LifecycleWaiters => Volatile.Read(ref _lifecycleWaiters); @@ -163,6 +176,8 @@ public async Task BasicConsumeAsync( var consumer = new AsyncEventingBasicConsumer(_channel); _onDelivery = onDelivery; consumer.ReceivedAsync += OnReceivedAsync; + consumer.UnregisteredAsync += OnConsumerUnregisteredAsync; + consumer.ShutdownAsync += OnConsumerShutdownAsync; try { var consumerTag = await _channel @@ -174,7 +189,7 @@ public async Task BasicConsumeAsync( catch (Exception) { _onDelivery = null; - consumer.ReceivedAsync -= OnReceivedAsync; + DetachConsumerHandlers(consumer); throw; } } @@ -189,6 +204,7 @@ public Task BasicCancelAsync(string consumerTag, CancellationToken cancellationT { ArgumentException.ThrowIfNullOrWhiteSpace(consumerTag); ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + Interlocked.Exchange(ref _applicationCancel, 1); return _channel.BasicCancelAsync(consumerTag, cancellationToken: cancellationToken); } @@ -223,6 +239,7 @@ async Task MarkDisposedAsync() } DetachConsumer(); + _channel.ChannelShutdownAsync -= OnChannelShutdownAsync; close = true; } @@ -286,13 +303,96 @@ private void DetachConsumer() { if (_consumer is not null) { - _consumer.ReceivedAsync -= OnReceivedAsync; + DetachConsumerHandlers(_consumer); _consumer = null; } _onDelivery = null; } + /// Removes delivery and fault handlers from . + /// Consumer previously attached by . + private void DetachConsumerHandlers(AsyncEventingBasicConsumer consumer) + { + consumer.ReceivedAsync -= OnReceivedAsync; + consumer.UnregisteredAsync -= OnConsumerUnregisteredAsync; + consumer.ShutdownAsync -= OnConsumerShutdownAsync; + } + + /// Raises once for an unexpected channel shutdown. + /// Unused. + /// Client shutdown details. + private Task OnChannelShutdownAsync(object sender, ShutdownEventArgs eventArgs) + { + TryRaiseChannelFaulted( + RabbitMqChannelFaultKind.ChannelShutdown, + FormatShutdownReason(eventArgs)); + return Task.CompletedTask; + } + + /// + /// Raises when the broker cancels the consumer and this wrapper did not issue cancel. + /// + /// Unused. + /// Cancelled consumer tags from the client. + private Task OnConsumerUnregisteredAsync(object sender, ConsumerEventArgs eventArgs) + { + if (Volatile.Read(ref _applicationCancel) == 1) + { + return Task.CompletedTask; + } + + var tags = eventArgs.ConsumerTags is { Length: > 0 } + ? string.Join(',', eventArgs.ConsumerTags) + : "(none)"; + TryRaiseChannelFaulted( + RabbitMqChannelFaultKind.ConsumerCancelled, + $"Consumer unregistered by broker (tags={tags})."); + return Task.CompletedTask; + } + + /// Raises once for a consumer-scoped shutdown. + /// Unused. + /// Client shutdown details. + private Task OnConsumerShutdownAsync(object sender, ShutdownEventArgs eventArgs) + { + TryRaiseChannelFaulted( + RabbitMqChannelFaultKind.ChannelShutdown, + FormatShutdownReason(eventArgs)); + return Task.CompletedTask; + } + + /// + /// Raises at most once when disposal has not marked the channel closed. + /// + /// Fault classification. + /// Broker or client reason text. + private void TryRaiseChannelFaulted(RabbitMqChannelFaultKind kind, string reason) + { + if (Volatile.Read(ref _disposed) == 1) + { + return; + } + + if (Interlocked.Exchange(ref _faultRaised, 1) == 1) + { + return; + } + + ChannelFaulted?.Invoke(this, new RabbitMqChannelFaultedEventArgs(kind, reason)); + } + + /// Formats a client shutdown reason without credentials or payload bytes. + /// Client shutdown details. + /// Stable reason text for logs and recovery. + private static string FormatShutdownReason(ShutdownEventArgs eventArgs) + { + var replyText = string.IsNullOrWhiteSpace(eventArgs.ReplyText) + ? "(none)" + : eventArgs.ReplyText; + return $"Channel shutdown initiator={eventArgs.Initiator} code={eventArgs.ReplyCode} text={replyText}"; + } + /// /// Closes an open client channel, then disposes it. /// Already-closed, interrupted, I/O, disposed, and timeout failures are written to the Serilog self-log. diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs index 7aa49a1c..62a2996f 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs @@ -1,4 +1,5 @@ using RabbitMQ.Client; +using RabbitMQ.Client.Events; using RabbitMQ.Client.Exceptions; namespace VectorNNTPD.Common.Messaging.RabbitMq @@ -22,19 +23,26 @@ internal sealed class RabbitMqClientPublishChannel : IRabbitMqPublishChannel /// 1 after the first . private int _disposed; + /// 1 after an unexpected fault has been raised once for this channel. + private int _faultRaised; + /// Initializes a new wrapper around an opened confirm-enabled channel. internal RabbitMqClientPublishChannel(IChannel channel, long generation) { ArgumentNullException.ThrowIfNull(channel); _channel = channel; Generation = generation; + _channel.ChannelShutdownAsync += OnChannelShutdownAsync; } /// public long Generation { get; } /// - public bool IsOpen => _channel.IsOpen; + public event EventHandler? ChannelFaulted; + + /// + public bool IsOpen => Volatile.Read(ref _disposed) == 0 && _channel.IsOpen; /// public Task PublishConfirmedAsync( @@ -119,6 +127,8 @@ public async ValueTask DisposeAsync() return; } + _channel.ChannelShutdownAsync -= OnChannelShutdownAsync; + try { if (_channel.IsOpen) @@ -133,6 +143,32 @@ public async ValueTask DisposeAsync() await _channel.DisposeAsync().ConfigureAwait(false); } + /// Raises once for an unexpected channel shutdown. + /// Unused. + /// Client shutdown details. + private Task OnChannelShutdownAsync(object sender, ShutdownEventArgs eventArgs) + { + if (Volatile.Read(ref _disposed) == 1) + { + return Task.CompletedTask; + } + + if (Interlocked.Exchange(ref _faultRaised, 1) == 1) + { + return Task.CompletedTask; + } + + var replyText = string.IsNullOrWhiteSpace(eventArgs.ReplyText) + ? "(none)" + : eventArgs.ReplyText; + ChannelFaulted?.Invoke( + this, + new RabbitMqChannelFaultedEventArgs( + RabbitMqChannelFaultKind.ChannelShutdown, + $"Channel shutdown initiator={eventArgs.Initiator} code={eventArgs.ReplyCode} text={replyText}")); + return Task.CompletedTask; + } + /// /// Builds AMQP properties for a confirmed publication. /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs new file mode 100644 index 00000000..d084b7b2 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs @@ -0,0 +1,550 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.RabbitMq; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.RabbitMq; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork +{ + /// + /// Channel-scoped recovery for Article Work consumers and the response publisher without reconnecting a healthy AMQP connection. + /// + public sealed class ArticleWorkChannelFailureRecoveryTests + { + /// + /// Unexpected consumer-channel shutdown retires the failed session and starts a replacement on the same connection generation. + /// + [Fact] + public async Task Consumer_channel_shutdown_retires_session_and_starts_replacement_without_reconnect() + { + await using var context = await StartConsumerAsync(); + var first = Assert.Single(context.Consumer.Sessions); + var firstChannel = Assert.IsType(first.Channel); + Assert.Equal(1, context.Factory.ConnectCount); + + firstChannel.SimulateChannelShutdown(); + + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Consumer.Sessions.Count == 1 + && !ReferenceEquals(context.Consumer.Sessions[0], first) + && context.Consumer.Sessions[0].State == ArticleWorkConsumerState.Running + && context.Consumer.Sessions[0].Channel is { IsOpen: true }, + TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.Equal(1, firstChannel.DisposeCount); + Assert.Equal(1, context.Factory.ConnectCount); + Assert.Equal(1, context.Connections.ConnectionGeneration); + Assert.True(context.Factory.LastConnection!.Channels.Count > 1); + Assert.Equal(1, Assert.IsType(context.Consumer.Sessions[0].Channel).ConsumeCount); + } + + /// + /// Broker-side consumer cancellation recovers while the AMQP connection remains open. + /// + [Fact] + public async Task Broker_consumer_cancellation_recovers_while_connection_stays_open() + { + await using var context = await StartConsumerAsync(); + var first = Assert.Single(context.Consumer.Sessions); + var firstChannel = Assert.IsType(first.Channel); + + firstChannel.SimulateConsumerCancelled(); + + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Consumer.Sessions.Count == 1 + && !ReferenceEquals(context.Consumer.Sessions[0], first) + && context.Consumer.Sessions[0].Channel is { IsOpen: true }, + TimeSpan.FromSeconds(2)); + + Assert.Equal(1, context.Factory.ConnectCount); + Assert.True(context.Factory.LastConnection!.IsOpen); + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + } + + /// + /// Reconciliation does not keep a session merely because its state is Running when the channel is closed. + /// + [Fact] + public async Task Reconcile_retires_running_session_with_closed_channel() + { + await using var context = await StartConsumerAsync(); + var first = Assert.Single(context.Consumer.Sessions); + var firstChannel = Assert.IsType(first.Channel); + firstChannel.IsOpen = false; + + await context.Consumer.ReconcileAsync(CancellationToken.None); + + var replacement = Assert.Single(context.Consumer.Sessions); + Assert.NotSame(first, replacement); + Assert.Equal(ArticleWorkConsumerState.Running, replacement.State); + Assert.True(replacement.Channel!.IsOpen); + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.Equal(1, firstChannel.DisposeCount); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// A late fault callback from a retired channel does not retire its replacement. + /// + [Fact] + public async Task Late_fault_from_old_session_does_not_retire_replacement() + { + await using var context = await StartConsumerAsync(); + var first = Assert.Single(context.Consumer.Sessions); + var firstChannel = Assert.IsType(first.Channel); + firstChannel.SimulateChannelShutdown(); + + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Consumer.Sessions.Count == 1 && !ReferenceEquals(context.Consumer.Sessions[0], first), + TimeSpan.FromSeconds(2)); + + var replacement = Assert.Single(context.Consumer.Sessions); + var replacementChannel = Assert.IsType(replacement.Channel); + + firstChannel.SimulateChannelShutdown("stale late callback"); + + await Task.Yield(); + Assert.Same(replacement, Assert.Single(context.Consumer.Sessions)); + Assert.Equal(ArticleWorkConsumerState.Running, replacement.State); + Assert.Same(replacementChannel, replacement.Channel); + Assert.Equal(0, replacementChannel.DisposeCount); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// Repeated fault notifications for one session retire it once and leave a single replacement. + /// + [Fact] + public async Task Repeated_channel_faults_do_not_duplicate_active_consumers() + { + await using var context = await StartConsumerAsync(); + var first = Assert.Single(context.Consumer.Sessions); + var firstChannel = Assert.IsType(first.Channel); + + firstChannel.SimulateChannelShutdown("first"); + firstChannel.SimulateConsumerCancelled("second"); + firstChannel.SimulateChannelShutdown("third"); + + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Consumer.Sessions.Count == 1 + && !ReferenceEquals(context.Consumer.Sessions[0], first) + && context.Consumer.Sessions[0].State == ArticleWorkConsumerState.Running, + TimeSpan.FromSeconds(2)); + + Assert.Single(context.Consumer.Sessions); + Assert.Equal(1, firstChannel.DisposeCount); + Assert.Equal(1, context.Factory.ConnectCount); + Assert.True(context.Factory.LastConnection!.Channels.Count > 1); + } + + /// + /// Expected retirement cancel does not raise a fault-driven reconnect of the AMQP connection. + /// + [Fact] + public async Task Expected_retirement_does_not_open_a_new_connection() + { + await using var context = await StartConsumerAsync(); + var session = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(session.Channel); + + await session.RetireAsync(); + + Assert.Equal(ArticleWorkConsumerState.Stopped, session.State); + Assert.Equal(1, channel.CancelCount); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// Application shutdown retires sessions without starting replacements or reconnecting. + /// + [Fact] + public async Task Application_shutdown_does_not_start_channel_fault_replacements() + { + await using var context = await StartConsumerAsync(); + var session = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(session.Channel); + + await context.Consumer.DisposeAsync(); + + Assert.Empty(context.Consumer.Sessions); + Assert.Equal(ArticleWorkConsumerState.Stopped, session.State); + Assert.Equal(1, channel.DisposeCount); + Assert.Equal(1, context.Factory.ConnectCount); + + channel.SimulateChannelShutdown("after shutdown"); + await Task.Yield(); + Assert.Empty(context.Consumer.Sessions); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// In-flight delivery admitted before an unexpected channel fault completes without ACK/NACK on the dead channel + /// (broker redelivery) and without a second settlement attempt after retirement. + /// + [Fact] + public async Task In_flight_delivery_during_channel_fault_is_left_unsettled_for_redelivery() + { + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "missing", + Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + await using var context = await StartConsumerAsync(handler); + var session = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(session.Channel); + var processing = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + await handler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + channel.SimulateChannelShutdown(); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => session.State == ArticleWorkConsumerState.Retiring + || session.State == ArticleWorkConsumerState.Stopped, + TimeSpan.FromSeconds(2)); + + handler.Gate!.TrySetResult(); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => session.State == ArticleWorkConsumerState.Stopped + && context.Consumer.Sessions.Count == 1 + && !ReferenceEquals(context.Consumer.Sessions[0], session), + TimeSpan.FromSeconds(2)); + + Assert.Empty(channel.Settlements); + Assert.Equal(1, channel.DisposeCount); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// Publish-channel shutdown installs a replacement on the same connection generation. + /// + [Fact] + public async Task Publisher_channel_shutdown_replaces_channel_without_reconnect() + { + await using var context = await StartPublisherAsync(); + var first = context.PublishChannel; + Assert.Equal(1, context.Factory.ConnectCount); + + first.SimulateChannelShutdown(); + + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Publisher.Channel is not null + && !ReferenceEquals(context.Publisher.Channel, first) + && context.Publisher.Channel.IsOpen, + TimeSpan.FromSeconds(2)); + + Assert.Equal(1, first.DisposeCount); + Assert.Equal(1, context.Factory.ConnectCount); + Assert.Equal(1, context.Publisher.Generation); + Assert.Equal(2, context.Factory.LastConnection!.PublishChannels.Count); + } + + /// + /// After publish-channel replacement, the next publish succeeds through the new channel. + /// + [Fact] + public async Task Publish_after_channel_replacement_uses_the_new_channel() + { + await using var context = await StartPublisherAsync(); + var first = context.PublishChannel; + first.SimulateChannelShutdown(); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Publisher.Channel is not null && !ReferenceEquals(context.Publisher.Channel, first), + TimeSpan.FromSeconds(2)); + + var replacement = context.PublishChannel; + await context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None); + + Assert.Empty(first.Publications); + Assert.Single(replacement.Publications); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// A late fault from an obsolete publish channel does not dispose the installed replacement. + /// + [Fact] + public async Task Late_publish_channel_fault_does_not_invalidate_replacement() + { + await using var context = await StartPublisherAsync(); + var first = context.PublishChannel; + first.SimulateChannelShutdown(); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Publisher.Channel is not null && !ReferenceEquals(context.Publisher.Channel, first), + TimeSpan.FromSeconds(2)); + + var replacement = context.PublishChannel; + first.SimulateChannelShutdown("stale"); + + await Task.Yield(); + Assert.Same(replacement, context.Publisher.Channel); + Assert.True(replacement.IsOpen); + Assert.Equal(0, replacement.DisposeCount); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// Concurrent publishes wait for channel rebuild rather than racing duplicate replacements unsafely. + /// + [Fact] + public async Task Concurrent_publishes_after_channel_fault_share_one_replacement() + { + await using var context = await StartPublisherAsync(); + var first = context.PublishChannel; + first.SimulateChannelShutdown(); + + var publish1 = context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None); + var publish2 = context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None); + await Task.WhenAll(publish1, publish2).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(1, first.DisposeCount); + Assert.Equal(2, context.Factory.LastConnection!.PublishChannels.Count); + Assert.Equal(2, context.PublishChannel.Publications.Count); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// Failed channel replacement does not report publish success. + /// + [Fact] + public async Task Failed_publish_channel_replacement_does_not_report_success() + { + await using var context = await StartPublisherAsync(); + var first = context.PublishChannel; + context.Factory.LastConnection!.CreatePublishChannelException = + new InvalidOperationException("no publish channel"); + first.SimulateChannelShutdown(); + + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Publisher.Channel is null || !context.Publisher.Channel.IsOpen, + TimeSpan.FromSeconds(2)); + + await Assert.ThrowsAsync( + () => context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None)); + Assert.Empty(first.Publications); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// Caller cancellation during publish wait does not leave a permanently closed installed channel when rebuild already completed. + /// + [Fact] + public async Task Caller_cancellation_does_not_trigger_unwanted_connection_recovery() + { + await using var context = await StartPublisherAsync(); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync( + () => context.Publisher.PublishAsync(SuccessIntent(), cts.Token)); + + Assert.Equal(1, context.Factory.ConnectCount); + Assert.True(context.PublishChannel.IsOpen); + } + + /// + /// Publisher disposal prevents channel-fault rebuild work after stop. + /// + [Fact] + public async Task Publisher_shutdown_ignores_late_channel_faults() + { + await using var context = await StartPublisherAsync(); + var channel = context.PublishChannel; + await context.Publisher.DisposeAsync(); + + channel.SimulateChannelShutdown(); + await Task.Yield(); + + Assert.Equal(ArticleWorkResponsePublisherState.Stopped, context.Publisher.State); + Assert.Null(context.Publisher.Channel); + Assert.Equal(1, context.Factory.ConnectCount); + Assert.Single(context.Factory.LastConnection!.PublishChannels); + } + + /// + /// Dispose while a channel-fault rebuild is blocked, then connection replacement is chained, completes without + /// deadlock or an unobserved replace-gate waiter after the publisher disposes its semaphores. + /// + [Fact] + public async Task Publisher_shutdown_during_chained_channel_and_connection_rebuild_does_not_deadlock() + { + await using var context = await StartPublisherAsync(); + var first = context.PublishChannel; + var firstConnection = context.Factory.LastConnection!; + firstConnection.CreatePublishChannelStarted = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + firstConnection.BlockCreatePublishChannel = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + first.SimulateChannelShutdown(); + await firstConnection.CreatePublishChannelStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Factory.Connected = secondConnected; + firstConnection.SimulateLost(); + await secondConnected.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + var disposing = context.Publisher.DisposeAsync().AsTask(); + firstConnection.BlockCreatePublishChannel.TrySetResult(); + await disposing.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkResponsePublisherState.Stopped, context.Publisher.State); + Assert.Null(context.Publisher.Channel); + } + + /// + /// Connection replacement that races with channel-only recovery still installs a channel on the new generation. + /// + [Fact] + public async Task Connection_replacement_chained_after_channel_fault_installs_new_generation() + { + await using var context = await StartPublisherAsync(); + var first = context.PublishChannel; + var firstConnection = context.Factory.LastConnection!; + firstConnection.CreatePublishChannelStarted = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + firstConnection.BlockCreatePublishChannel = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + first.SimulateChannelShutdown(); + await firstConnection.CreatePublishChannelStarted.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.Factory.Connected = secondConnected; + firstConnection.SimulateLost(); + await secondConnected.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + firstConnection.BlockCreatePublishChannel.TrySetResult(); + + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Publisher.Generation == 2 + && context.Publisher.Channel is { IsOpen: true }, + TimeSpan.FromSeconds(2)); + + Assert.NotSame(first, context.Publisher.Channel); + Assert.Equal(2, context.Factory.ConnectCount); + await context.Publisher.PublishAsync(SuccessIntent(), CancellationToken.None); + Assert.Single(context.PublishChannel.Publications); + } + + private static ArticleWorkResponseIntent SuccessIntent() => + new( + ArticleWorkOutcome.Success, + Guid.Parse(ArticleWorkTestDeliveries.CanonicalRequestId), + ArticleWorkTestDeliveries.CanonicalMessageId, + "Giganews", + ArticleWorkTestDeliveries.CanonicalCorrelationId, + ArticleWorkTestDeliveries.CanonicalReplyTo, + Error: null, + ArticleWorkTestDeliveries.CanonicalFqdn, + ArticleWorkTestDeliveries.CanonicalListenerPort, + ArticleWorkTestDeliveries.CanonicalArticleIdHex); + + private static async Task StartConsumerAsync(IArticleWorkHandler? handler = null) + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + var consumer = new ArticleWorkConsumerService( + connections, + RabbitMqServiceTests.CreateFastRuntime(), + handler ?? new DeferredArticleWorkHandler(), + new RecordingArticleWorkResponsePublisher(), + NullLogger.Instance, + catalog: new StaticBackFillerProviderCatalog( + [ + new BackFillerProviderDefinition( + "Giganews", + "127.0.0.1", + 119, + false, + "nntp-user", + "p", + 0, + 1), + ]), + capacity: CreateCapacity("Giganews", 1)); + await consumer.StartAsync(CancellationToken.None); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => consumer.Sessions.Count == 1 && consumer.Sessions[0].State == ArticleWorkConsumerState.Running, + TimeSpan.FromSeconds(2)); + return new ConsumerHarness(factory, connections, consumer); + } + + private static async Task StartPublisherAsync() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory + { + DefaultPublishConfirmBehavior = FakePublishConfirmBehavior.Confirm, + }; + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + factory.LastConnection!.DefaultPublishConfirmBehavior = FakePublishConfirmBehavior.Confirm; + var publisher = new ArticleWorkResponsePublisher( + connections, + RabbitMqServiceTests.CreateFastRuntime(), + NullLogger.Instance); + await publisher.StartAsync(CancellationToken.None); + Assert.Equal(ArticleWorkResponsePublisherState.Running, publisher.State); + return new PublisherHarness(factory, connections, publisher); + } + + private static BackboneUsableCapacityState CreateCapacity(string backbone, int active) + { + var capacity = new BackboneUsableCapacityState(); + if (active > 0) + { + capacity.PublishSnapshot(new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [backbone] = active, + }); + } + + return capacity; + } + + private sealed class ConsumerHarness( + FakeBackFillerRabbitMqConnectionFactory factory, + RabbitMqService connections, + ArticleWorkConsumerService consumer) : IAsyncDisposable + { + public FakeBackFillerRabbitMqConnectionFactory Factory { get; } = factory; + + public RabbitMqService Connections { get; } = connections; + + public ArticleWorkConsumerService Consumer { get; } = consumer; + + public async ValueTask DisposeAsync() + { + await Consumer.DisposeAsync(); + await Connections.DisposeAsync(); + } + } + + private sealed class PublisherHarness( + FakeBackFillerRabbitMqConnectionFactory factory, + RabbitMqService connections, + ArticleWorkResponsePublisher publisher) : IAsyncDisposable + { + public FakeBackFillerRabbitMqConnectionFactory Factory { get; } = factory; + + public RabbitMqService Connections { get; } = connections; + + public ArticleWorkResponsePublisher Publisher { get; } = publisher; + + public FakeBackFillerRabbitMqPublishChannel PublishChannel => + Assert.IsType(Publisher.Channel); + + public async ValueTask DisposeAsync() + { + await Publisher.DisposeAsync(); + await Connections.DisposeAsync(); + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs index 122c7964..bd61b600 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs @@ -101,6 +101,13 @@ private sealed class TestManualAckChannel : IRabbitMqManualAckChannel /// public bool IsOpen => true; + /// + public event EventHandler? ChannelFaulted + { + add { } + remove { } + } + /// public ValueTask DisposeAsync() => ValueTask.CompletedTask; diff --git a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqChannelShutdownTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqChannelShutdownTests.cs index 4c557aba..c9c38ef9 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqChannelShutdownTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqChannelShutdownTests.cs @@ -77,6 +77,11 @@ internal static ChannelDouble Create(Exception closeException) protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) { ArgumentNullException.ThrowIfNull(targetMethod); + if (targetMethod.Name is "add_ChannelShutdownAsync" or "remove_ChannelShutdownAsync") + { + return null; + } + if (targetMethod.Name == "get_IsOpen") { return true; @@ -93,9 +98,11 @@ internal static ChannelDouble Create(Exception closeException) return ValueTask.CompletedTask; } - if (targetMethod.ReturnType == typeof(ValueTask)) + if (targetMethod.ReturnType == typeof(void) || targetMethod.ReturnType == typeof(ValueTask)) { - return ValueTask.CompletedTask; + return targetMethod.ReturnType == typeof(ValueTask) + ? ValueTask.CompletedTask + : null; } if (typeof(Task).IsAssignableFrom(targetMethod.ReturnType)) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqClientManualAckChannelTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqClientManualAckChannelTests.cs index 248454a1..d9f92620 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqClientManualAckChannelTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqClientManualAckChannelTests.cs @@ -436,6 +436,11 @@ internal string[] Snapshot() protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) { ArgumentNullException.ThrowIfNull(targetMethod); + if (targetMethod.Name is "add_ChannelShutdownAsync" or "remove_ChannelShutdownAsync") + { + return null; + } + if (targetMethod.Name == "get_IsOpen") { return IsOpen; diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs index 6e11c7fe..32f3d91f 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs @@ -363,6 +363,9 @@ internal sealed class FakeBackFillerRabbitMqChannel(long generation) : IRabbitMq /// Writable open flag; when false, consume and settlement operations throw. public bool IsOpen { get; set; } = true; + /// Raised by / ; not wired to a real broker. + public event EventHandler? ChannelFaulted; + /// Number of exclusive dispose passes completed on this channel. public int DisposeCount { get; private set; } @@ -561,7 +564,7 @@ public Task BasicConsumeAsync( return Task.FromResult(ConsumerTag); } - /// Increments without unregistering the consumer callback. + /// Increments without unregistering the consumer callback or raising . /// Validated but not matched against . /// Honored before incrementing the counter. public Task BasicCancelAsync(string consumerTag, CancellationToken cancellationToken) @@ -572,6 +575,32 @@ public Task BasicCancelAsync(string consumerTag, CancellationToken cancellationT return Task.CompletedTask; } + /// + /// Marks the channel closed and raises as an unexpected channel shutdown. + /// Leaves the process connection open. + /// + /// Fault reason text. + internal void SimulateChannelShutdown(string reason = "channel closed by broker") + { + IsOpen = false; + ChannelFaulted?.Invoke( + this, + new RabbitMqChannelFaultedEventArgs(RabbitMqChannelFaultKind.ChannelShutdown, reason)); + } + + /// + /// Marks the channel closed and raises as a broker-originated consumer cancellation. + /// Leaves the process connection open. + /// + /// Fault reason text. + internal void SimulateConsumerCancelled(string reason = "consumer cancelled by broker") + { + IsOpen = false; + ChannelFaulted?.Invoke( + this, + new RabbitMqChannelFaultedEventArgs(RabbitMqChannelFaultKind.ConsumerCancelled, reason)); + } + /// /// Runs ack gates, optionally throws , then appends an ack settlement. /// Tracks broker-op overlap via while inside the exclusive section. @@ -760,6 +789,9 @@ internal sealed class FakeBackFillerRabbitMqPublishChannel(long generation) : IR /// Writable open flag; when false, publish throws; some confirm failures also clear this flag. public bool IsOpen { get; set; } = true; + /// Raised by ; not wired to a real broker. + public event EventHandler? ChannelFaulted; + /// Number of dispose calls completed on this publish channel. public int DisposeCount { get; private set; } @@ -848,6 +880,11 @@ public async Task PublishConfirmedAsync(RabbitMqConfirmedPublication publication throw new OperationCanceledException("publisher confirm timeout", cancellationToken); case FakePublishConfirmBehavior.CloseChannel: IsOpen = false; + ChannelFaulted?.Invoke( + this, + new RabbitMqChannelFaultedEventArgs( + RabbitMqChannelFaultKind.ChannelShutdown, + "channel closed during confirmation")); throw new InvalidOperationException("RabbitMQ publish channel closed during confirmation."); default: if (ConfirmGate is not null) @@ -864,6 +901,19 @@ public async Task PublishConfirmedAsync(RabbitMqConfirmedPublication publication /// When set, thrown after incrementing dispose count and clearing . public Exception? DisposeException { get; set; } + /// + /// Marks the channel closed and raises as an unexpected channel shutdown. + /// Leaves the process connection open. + /// + /// Fault reason text. + internal void SimulateChannelShutdown(string reason = "publish channel closed by broker") + { + IsOpen = false; + ChannelFaulted?.Invoke( + this, + new RabbitMqChannelFaultedEventArgs(RabbitMqChannelFaultKind.ChannelShutdown, reason)); + } + /// Increments dispose count, marks the channel closed, and optionally throws . public ValueTask DisposeAsync() { diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs index d69c6497..cd6f4577 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs @@ -982,6 +982,13 @@ private sealed class LoggingPublishChannel(long generation, LoggingConnection co /// public bool IsOpen { get; private set; } = true; + /// + public event EventHandler? ChannelFaulted + { + add { } + remove { } + } + /// /// Number of calls. ///