From c35ec7a41b19ae1b795931ccaa5721c87d27bff3 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 21 Oct 2026 20:45:16 +0200 Subject: [PATCH] Isolate Article Work topology declaration failures --- .../ArticleWork/ArticleWorkConsumerService.cs | 124 ++++-- ...leWorkTopologyDeclarationIsolationTests.cs | 386 ++++++++++++++++++ .../TestDoubles/FakeBackFillerRabbitMq.cs | 14 + 3 files changed, 498 insertions(+), 26 deletions(-) create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs index 7b570bf2..1326cdcf 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs @@ -1,3 +1,4 @@ +using RabbitMQ.Client.Exceptions; using VectorNNTPD.BackFillerService.Configuration; using VectorNNTPD.BackFillerService.Logging; using VectorNNTPD.BackFillerService.Nntp; @@ -730,8 +731,10 @@ private enum ReconcilePass /// Backbones whose declaration succeeded. Empty when there is nothing to declare, the connection is not ready, or every declaration failed. /// /// - /// When a current connection exists, one manual-ack channel is opened for the pass and disposed of afterward. A failure for one backbone is logged and skipped. - /// A failure to open the channel is logged for every backbone not already declared. is not logged. + /// Declarations reuse one manual-ack channel while it remains open. A failed declaration disposes that channel + /// before the next backbone is attempted, because RabbitMQ closes the channel on broker-side declare failures + /// such as PRECONDITION_FAILED. A later backbone therefore never reuses an unusable channel. + /// The connection generation is rechecked before each channel open. is not logged. /// private async Task> EnsureProviderTopologyAsync( IEnumerable backbones, @@ -749,7 +752,7 @@ private async Task> EnsureProviderTopologyAsync( return ready; } - if (!_connections.TryGetCurrent(out var handle)) + if (!_connections.TryGetCurrent(out _)) { foreach (var backbone in unique) { @@ -766,15 +769,24 @@ private async Task> EnsureProviderTopologyAsync( IRabbitMqManualAckChannel? channel = null; try { - channel = await handle.Connection - .CreateManualAckChannelAsync(handle.Generation, cancellationToken) - .ConfigureAwait(false); foreach (var backbone in unique.OrderBy(static name => name, StringComparer.OrdinalIgnoreCase)) { cancellationToken.ThrowIfCancellationRequested(); var queue = BackFillerRabbitMqTopology.ComposeProviderEntity(backbone); try { + channel = await EnsureOpenTopologyChannelAsync(channel, cancellationToken) + .ConfigureAwait(false); + if (channel is null) + { + ArticleWorkLogMessages.ProviderTopologyDeclareFailed( + _logger, + backbone, + queue, + "RabbitMQ connection is not ready for topology declaration."); + continue; + } + await BackFillerArticleWorkTopology .DeclareProviderEndpointAsync(channel, backbone, cancellationToken) .ConfigureAwait(false); @@ -787,35 +799,95 @@ await BackFillerArticleWorkTopology _logger, backbone, queue, - ex.Message); + FormatTopologyDeclareFailure(ex)); + channel = await DisposeTopologyChannelAsync(channel).ConfigureAwait(false); } } } - catch (Exception ex) when (ex is not OperationCanceledException) + finally { - foreach (var backbone in unique) - { - if (ready.Contains(backbone)) - { - continue; - } + _ = await DisposeTopologyChannelAsync(channel).ConfigureAwait(false); + } - ArticleWorkLogMessages.ProviderTopologyDeclareFailed( - _logger, - backbone, - BackFillerRabbitMqTopology.ComposeProviderEntity(backbone), - ex.Message); - } + return ready; + } + + /// + /// Returns when it is still open; otherwise disposes it and opens a new channel on the current generation. + /// + /// Previous topology channel, or . + /// Cancels channel creation. + /// + /// An open manual-ack channel, or when no current connection is available. + /// + private async Task EnsureOpenTopologyChannelAsync( + IRabbitMqManualAckChannel? channel, + CancellationToken cancellationToken) + { + if (channel is { IsOpen: true }) + { + return channel; } - finally + + _ = await DisposeTopologyChannelAsync(channel).ConfigureAwait(false); + if (!_connections.TryGetCurrent(out var handle) || !handle.IsCurrent) { - if (channel is not null) - { - await channel.DisposeAsync().ConfigureAwait(false); - } + return null; } - return ready; + return await handle.Connection + .CreateManualAckChannelAsync(handle.Generation, cancellationToken) + .ConfigureAwait(false); + } + + /// Disposes when non-null and returns . + /// Topology channel to release. + /// so the caller can clear its local handle. + private static async Task DisposeTopologyChannelAsync( + IRabbitMqManualAckChannel? channel) + { + if (channel is not null) + { + await channel.DisposeAsync().ConfigureAwait(false); + } + + return null; + } + + /// + /// Formats a topology declaration failure, preferring structured AMQP shutdown details when present. + /// + /// Failure from channel creation or declare/bind. + /// Stable log text that includes reply code and text for broker channel closes. + private static string FormatTopologyDeclareFailure(Exception exception) + { + if (exception is OperationInterruptedException interrupted + && interrupted.ShutdownReason is { } interruptedShutdown) + { + return FormatAmqpShutdownFailure(interruptedShutdown, interrupted.Message); + } + + if (exception is AlreadyClosedException alreadyClosed + && alreadyClosed.ShutdownReason is { } closedShutdown) + { + return FormatAmqpShutdownFailure(closedShutdown, alreadyClosed.Message); + } + + return exception.Message; + } + + /// Formats broker channel-close metadata without credentials or payload bytes. + /// Client shutdown details from the failed RPC. + /// Exception message appended for context. + /// Log text including reply code, reply text, and initiator. + private static string FormatAmqpShutdownFailure( + RabbitMQ.Client.Events.ShutdownEventArgs shutdown, + string fallbackMessage) + { + var replyText = string.IsNullOrWhiteSpace(shutdown.ReplyText) + ? "(none)" + : shutdown.ReplyText; + return $"AMQP channel closed replyCode={shutdown.ReplyCode} text={replyText} initiator={shutdown.Initiator}: {fallbackMessage}"; } /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs new file mode 100644 index 00000000..7acc02a2 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs @@ -0,0 +1,386 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.RabbitMq; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.RabbitMq; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork +{ + /// + /// Topology declaration isolation: one backbone's channel-closing declare failure must not block later backbones. + /// + public sealed class ArticleWorkTopologyDeclarationIsolationTests + { + /// + /// AMQP reply code for PRECONDITION_FAILED (inequivalent queue arguments). + /// + private const ushort PreconditionFailedReplyCode = 406; + + /// + /// Event id for . + /// + private const int ProviderTopologyDeclareFailedEventId = 5320; + + /// + /// Alphabetically earlier backbone closes the topology channel; a later backbone still declares and gets a consumer. + /// + [Fact] + public async Task First_declare_failure_does_not_block_later_backbone() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + ArmQueueDeclareFailure( + factory, + queue => queue.Equals("backfiller.abavia", StringComparison.Ordinal) + ? CreatePreconditionFailed("backfiller.abavia") + : null); + + var logger = new CollectingLogger(); + var consumer = CreateConsumer( + connections, + CreateCatalog([("Abavia", 1), ("Giganews", 1)]), + CreateCapacity([("Abavia", 1), ("Giganews", 1)]), + logger); + + await consumer.StartAsync(CancellationToken.None); + + var session = Assert.Single(consumer.Sessions); + Assert.Equal("Giganews", session.Backbone); + Assert.Equal(ArticleWorkConsumerState.Running, session.State); + Assert.Contains( + factory.LastConnection!.Channels, + static channel => channel.QueueDeclarations.Any(static q => q.Name == "backfiller.giganews")); + Assert.DoesNotContain( + factory.LastConnection.Channels.SelectMany(static c => c.QueueDeclarations), + static q => q.Name == "backfiller.abavia"); + Assert.Contains( + logger.Entries, + static entry => entry.EventId.Id == ProviderTopologyDeclareFailedEventId + && entry.Message.Contains("backfiller.abavia", StringComparison.Ordinal) + && entry.Message.Contains($"replyCode={PreconditionFailedReplyCode}", StringComparison.Ordinal)); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// After a declare failure closes a channel, that channel is not reused for a later backbone. + /// + [Fact] + public async Task Closed_topology_channel_is_not_reused_for_later_backbone() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + ArmQueueDeclareFailure( + factory, + queue => queue.Equals("backfiller.abavia", StringComparison.Ordinal) + ? CreatePreconditionFailed("backfiller.abavia") + : null); + + var consumer = CreateConsumer( + connections, + CreateCatalog([("Abavia", 1), ("Giganews", 1)]), + CreateCapacity([("Abavia", 1), ("Giganews", 1)])); + + await consumer.StartAsync(CancellationToken.None); + + var topologyChannels = factory.LastConnection!.Channels + .Where(static channel => channel.ConsumeCount == 0) + .ToArray(); + Assert.True(topologyChannels.Length >= 2); + var failed = Assert.Single( + topologyChannels, + static channel => channel.QueueDeclarations.Count == 0 && channel.DisposeCount == 1); + Assert.False(failed.IsOpen); + Assert.Contains( + failed.ExchangeDeclarations, + static exchange => exchange.Name == "backfiller.abavia"); + var succeeded = Assert.Single( + topologyChannels, + static channel => channel.QueueDeclarations.Any(static q => q.Name == "backfiller.giganews")); + Assert.NotSame(failed, succeeded); + Assert.Equal(1, succeeded.DisposeCount); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// Multiple independently failing backbones do not prevent a later healthy backbone from declaring and consuming. + /// + [Fact] + public async Task Multiple_declare_failures_do_not_block_a_later_healthy_backbone() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + ArmQueueDeclareFailure( + factory, + queue => queue is "backfiller.abavia" or "backfiller.eweka" + ? CreatePreconditionFailed(queue) + : null); + + var consumer = CreateConsumer( + connections, + CreateCatalog([("Abavia", 1), ("Eweka", 1), ("Giganews", 1)]), + CreateCapacity([("Abavia", 1), ("Eweka", 1), ("Giganews", 1)])); + + await consumer.StartAsync(CancellationToken.None); + + var session = Assert.Single(consumer.Sessions); + Assert.Equal("Giganews", session.Backbone); + Assert.Contains( + factory.LastConnection!.Channels, + static channel => channel.QueueDeclarations.Any(static q => q.Name == "backfiller.giganews")); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// A backbone that failed topology declaration is retried on a later reconcile and can start once the failure clears. + /// + [Fact] + public async Task Failed_backbone_can_be_declared_on_a_later_reconcile_pass() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + var failAbavia = true; + ArmQueueDeclareFailure( + factory, + queue => failAbavia && queue.Equals("backfiller.abavia", StringComparison.Ordinal) + ? CreatePreconditionFailed("backfiller.abavia") + : null); + + var consumer = CreateConsumer( + connections, + CreateCatalog([("Abavia", 1), ("Giganews", 1)]), + CreateCapacity([("Abavia", 1), ("Giganews", 1)])); + + await consumer.StartAsync(CancellationToken.None); + Assert.Equal("Giganews", Assert.Single(consumer.Sessions).Backbone); + + failAbavia = false; + await consumer.ReconcileAsync(CancellationToken.None); + + Assert.Equal(2, consumer.Sessions.Count); + Assert.Contains(consumer.Sessions, static session => session.Backbone == "Abavia"); + Assert.Contains(consumer.Sessions, static session => session.Backbone == "Giganews"); + Assert.All(consumer.Sessions, static session => Assert.Equal(ArticleWorkConsumerState.Running, session.State)); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// An already-running healthy consumer is retained while another backbone's topology failure is isolated. + /// + [Fact] + public async Task Running_consumer_is_retained_when_another_backbone_declare_fails() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + var capacity = CreateCapacity([("Giganews", 1)]); + var consumer = CreateConsumer( + connections, + CreateCatalog([("Abavia", 1), ("Giganews", 1)]), + capacity); + + await consumer.StartAsync(CancellationToken.None); + var healthy = Assert.Single(consumer.Sessions); + Assert.Equal("Giganews", healthy.Backbone); + var healthyChannel = Assert.IsType(healthy.Channel); + + ArmQueueDeclareFailure( + factory, + queue => queue.Equals("backfiller.abavia", StringComparison.Ordinal) + ? CreatePreconditionFailed("backfiller.abavia") + : null); + capacity.PublishSnapshot(new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Giganews"] = 1, + ["Abavia"] = 1, + }); + await consumer.ReconcileAsync(CancellationToken.None); + + Assert.Same(healthy, Assert.Single(consumer.Sessions)); + Assert.Equal(ArticleWorkConsumerState.Running, healthy.State); + Assert.Same(healthyChannel, healthy.Channel); + Assert.Equal(0, healthyChannel.DisposeCount); + Assert.Contains( + factory.LastConnection!.Channels, + static channel => channel.ExchangeDeclarations.Any(static e => e.Name == "backfiller.abavia") + && channel.QueueDeclarations.Count == 0); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// Topology channels opened for the pass are disposed after success and after failure. + /// + [Fact] + public async Task Topology_channels_are_disposed_after_success_and_failure() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + ArmQueueDeclareFailure( + factory, + queue => queue.Equals("backfiller.abavia", StringComparison.Ordinal) + ? CreatePreconditionFailed("backfiller.abavia") + : null); + + var consumer = CreateConsumer( + connections, + CreateCatalog([("Abavia", 1), ("Giganews", 1)]), + CreateCapacity([("Abavia", 1), ("Giganews", 1)])); + + await consumer.StartAsync(CancellationToken.None); + + var topologyChannels = factory.LastConnection!.Channels + .Where(static channel => channel.ConsumeCount == 0) + .ToArray(); + Assert.All(topologyChannels, static channel => Assert.Equal(1, channel.DisposeCount)); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// Cancellation during topology declaration stops further backbone attempts and propagates. + /// + [Fact] + public async Task Cancellation_stops_further_topology_declarations() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(); + factory.LastConnection!.OnManualAckChannelCreated = _ => cts.Cancel(); + + var consumer = CreateConsumer( + connections, + CreateCatalog([("Abavia", 1), ("Giganews", 1)]), + CreateCapacity([("Abavia", 1), ("Giganews", 1)])); + + await Assert.ThrowsAnyAsync( + () => consumer.StartAsync(cts.Token)); + + Assert.Empty(consumer.Sessions); + Assert.True(factory.LastConnection.Channels.Count <= 1); + Assert.DoesNotContain( + factory.LastConnection.Channels.SelectMany(static c => c.QueueDeclarations), + static q => q.Name == "backfiller.giganews"); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// Successful declarations for several backbones share one open topology channel for the pass. + /// + [Fact] + public async Task Successful_declarations_reuse_one_open_topology_channel() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + var consumer = CreateConsumer( + connections, + CreateCatalog([("Eweka", 1), ("Giganews", 1)]), + CreateCapacity([("Eweka", 1), ("Giganews", 1)])); + + await consumer.StartAsync(CancellationToken.None); + + Assert.Equal(2, consumer.Sessions.Count); + var topology = Assert.Single( + factory.LastConnection!.Channels, + static channel => channel.ConsumeCount == 0); + Assert.Equal(2, topology.QueueDeclarations.Count); + Assert.Equal(1, topology.DisposeCount); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + private static void ArmQueueDeclareFailure( + FakeBackFillerRabbitMqConnectionFactory factory, + Func failure) + { + factory.LastConnection!.OnManualAckChannelCreated = channel => + { + channel.QueueDeclareFailure = failure; + }; + } + + private static OperationInterruptedException CreatePreconditionFailed(string queue) + { + var shutdown = new ShutdownEventArgs( + ShutdownInitiator.Peer, + PreconditionFailedReplyCode, + $"PRECONDITION_FAILED - inequivalent arg 'x-queue-type' for queue '{queue}'", + cause: null!, + CancellationToken.None); + return new OperationInterruptedException(shutdown); + } + + private static ArticleWorkConsumerService CreateConsumer( + IRabbitMqService connections, + IBackFillerProviderCatalog catalog, + BackboneUsableCapacityState capacity, + ILogger? logger = null) => + new( + connections, + RabbitMqServiceTests.CreateFastRuntime(), + new DeferredArticleWorkHandler(), + new RecordingArticleWorkResponsePublisher(), + logger ?? NullLogger.Instance, + catalog, + capacity); + + private static StaticBackFillerProviderCatalog CreateCatalog( + IReadOnlyList<(string Backbone, int MaxSessions)> providers) => + new( + [ + ..providers.Select(static p => new BackFillerProviderDefinition( + p.Backbone, + "127.0.0.1", + 119, + false, + "nntp-user", + "p", + 0, + p.MaxSessions)), + ]); + + private static BackboneUsableCapacityState CreateCapacity( + IReadOnlyList<(string Backbone, int Active)> capacities) + { + var state = new BackboneUsableCapacityState(); + var snapshot = capacities + .Where(static pair => pair.Active > 0) + .ToDictionary( + static pair => pair.Backbone, + static pair => pair.Active, + StringComparer.OrdinalIgnoreCase); + if (snapshot.Count > 0) + { + state.PublishSnapshot(snapshot); + } + + return state; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs index 32f3d91f..1eca084e 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs @@ -411,6 +411,12 @@ internal sealed class FakeBackFillerRabbitMqChannel(long generation) : IRabbitMq /// When set, thrown from queue declare instead of recording the declaration. public Exception? QueueDeclareException { get; set; } + /// + /// When it returns an exception, that queue name fails and clears to model a broker channel close. + /// Checked before . + /// + public Func? QueueDeclareFailure { get; set; } + /// When set, thrown from queue bind instead of recording the binding. public Exception? QueueBindException { get; set; } @@ -493,8 +499,16 @@ public Task QueueDeclareAsync( { ArgumentException.ThrowIfNullOrWhiteSpace(queue); cancellationToken.ThrowIfCancellationRequested(); + if (QueueDeclareFailure?.Invoke(queue) is { } selectiveFailure) + { + // Broker PRECONDITION_FAILED and similar declare failures close the AMQP channel. + IsOpen = false; + throw selectiveFailure; + } + if (QueueDeclareException is not null) { + IsOpen = false; throw QueueDeclareException; }