diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
index 7b570bf2..1326cdcf 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
@@ -1,3 +1,4 @@
+using RabbitMQ.Client.Exceptions;
using VectorNNTPD.BackFillerService.Configuration;
using VectorNNTPD.BackFillerService.Logging;
using VectorNNTPD.BackFillerService.Nntp;
@@ -730,8 +731,10 @@ private enum ReconcilePass
/// Backbones whose declaration succeeded. Empty when there is nothing to declare, the connection is not ready, or every declaration failed.
///
///
- /// When a current connection exists, one manual-ack channel is opened for the pass and disposed of afterward. A failure for one backbone is logged and skipped.
- /// A failure to open the channel is logged for every backbone not already declared. is not logged.
+ /// Declarations reuse one manual-ack channel while it remains open. A failed declaration disposes that channel
+ /// before the next backbone is attempted, because RabbitMQ closes the channel on broker-side declare failures
+ /// such as PRECONDITION_FAILED. A later backbone therefore never reuses an unusable channel.
+ /// The connection generation is rechecked before each channel open. is not logged.
///
private async Task> EnsureProviderTopologyAsync(
IEnumerable backbones,
@@ -749,7 +752,7 @@ private async Task> EnsureProviderTopologyAsync(
return ready;
}
- if (!_connections.TryGetCurrent(out var handle))
+ if (!_connections.TryGetCurrent(out _))
{
foreach (var backbone in unique)
{
@@ -766,15 +769,24 @@ private async Task> EnsureProviderTopologyAsync(
IRabbitMqManualAckChannel? channel = null;
try
{
- channel = await handle.Connection
- .CreateManualAckChannelAsync(handle.Generation, cancellationToken)
- .ConfigureAwait(false);
foreach (var backbone in unique.OrderBy(static name => name, StringComparer.OrdinalIgnoreCase))
{
cancellationToken.ThrowIfCancellationRequested();
var queue = BackFillerRabbitMqTopology.ComposeProviderEntity(backbone);
try
{
+ channel = await EnsureOpenTopologyChannelAsync(channel, cancellationToken)
+ .ConfigureAwait(false);
+ if (channel is null)
+ {
+ ArticleWorkLogMessages.ProviderTopologyDeclareFailed(
+ _logger,
+ backbone,
+ queue,
+ "RabbitMQ connection is not ready for topology declaration.");
+ continue;
+ }
+
await BackFillerArticleWorkTopology
.DeclareProviderEndpointAsync(channel, backbone, cancellationToken)
.ConfigureAwait(false);
@@ -787,35 +799,95 @@ await BackFillerArticleWorkTopology
_logger,
backbone,
queue,
- ex.Message);
+ FormatTopologyDeclareFailure(ex));
+ channel = await DisposeTopologyChannelAsync(channel).ConfigureAwait(false);
}
}
}
- catch (Exception ex) when (ex is not OperationCanceledException)
+ finally
{
- foreach (var backbone in unique)
- {
- if (ready.Contains(backbone))
- {
- continue;
- }
+ _ = await DisposeTopologyChannelAsync(channel).ConfigureAwait(false);
+ }
- ArticleWorkLogMessages.ProviderTopologyDeclareFailed(
- _logger,
- backbone,
- BackFillerRabbitMqTopology.ComposeProviderEntity(backbone),
- ex.Message);
- }
+ return ready;
+ }
+
+ ///
+ /// Returns when it is still open; otherwise disposes it and opens a new channel on the current generation.
+ ///
+ /// Previous topology channel, or .
+ /// Cancels channel creation.
+ ///
+ /// An open manual-ack channel, or when no current connection is available.
+ ///
+ private async Task EnsureOpenTopologyChannelAsync(
+ IRabbitMqManualAckChannel? channel,
+ CancellationToken cancellationToken)
+ {
+ if (channel is { IsOpen: true })
+ {
+ return channel;
}
- finally
+
+ _ = await DisposeTopologyChannelAsync(channel).ConfigureAwait(false);
+ if (!_connections.TryGetCurrent(out var handle) || !handle.IsCurrent)
{
- if (channel is not null)
- {
- await channel.DisposeAsync().ConfigureAwait(false);
- }
+ return null;
}
- return ready;
+ return await handle.Connection
+ .CreateManualAckChannelAsync(handle.Generation, cancellationToken)
+ .ConfigureAwait(false);
+ }
+
+ /// Disposes when non-null and returns .
+ /// Topology channel to release.
+ /// so the caller can clear its local handle.
+ private static async Task DisposeTopologyChannelAsync(
+ IRabbitMqManualAckChannel? channel)
+ {
+ if (channel is not null)
+ {
+ await channel.DisposeAsync().ConfigureAwait(false);
+ }
+
+ return null;
+ }
+
+ ///
+ /// Formats a topology declaration failure, preferring structured AMQP shutdown details when present.
+ ///
+ /// Failure from channel creation or declare/bind.
+ /// Stable log text that includes reply code and text for broker channel closes.
+ private static string FormatTopologyDeclareFailure(Exception exception)
+ {
+ if (exception is OperationInterruptedException interrupted
+ && interrupted.ShutdownReason is { } interruptedShutdown)
+ {
+ return FormatAmqpShutdownFailure(interruptedShutdown, interrupted.Message);
+ }
+
+ if (exception is AlreadyClosedException alreadyClosed
+ && alreadyClosed.ShutdownReason is { } closedShutdown)
+ {
+ return FormatAmqpShutdownFailure(closedShutdown, alreadyClosed.Message);
+ }
+
+ return exception.Message;
+ }
+
+ /// Formats broker channel-close metadata without credentials or payload bytes.
+ /// Client shutdown details from the failed RPC.
+ /// Exception message appended for context.
+ /// Log text including reply code, reply text, and initiator.
+ private static string FormatAmqpShutdownFailure(
+ RabbitMQ.Client.Events.ShutdownEventArgs shutdown,
+ string fallbackMessage)
+ {
+ var replyText = string.IsNullOrWhiteSpace(shutdown.ReplyText)
+ ? "(none)"
+ : shutdown.ReplyText;
+ return $"AMQP channel closed replyCode={shutdown.ReplyCode} text={replyText} initiator={shutdown.Initiator}: {fallbackMessage}";
}
///
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs
new file mode 100644
index 00000000..7acc02a2
--- /dev/null
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs
@@ -0,0 +1,386 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using RabbitMQ.Client;
+using RabbitMQ.Client.Events;
+using RabbitMQ.Client.Exceptions;
+using VectorNNTPD.BackFillerService.ArticleWork;
+using VectorNNTPD.BackFillerService.Nntp;
+using VectorNNTPD.BackFillerService.RabbitMq;
+using VectorNNTPD.BackFillerService.Tests.Fixtures;
+using VectorNNTPD.BackFillerService.Tests.RabbitMq;
+using VectorNNTPD.BackFillerService.Tests.TestDoubles;
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
+namespace VectorNNTPD.BackFillerService.Tests.ArticleWork
+{
+ ///
+ /// Topology declaration isolation: one backbone's channel-closing declare failure must not block later backbones.
+ ///
+ public sealed class ArticleWorkTopologyDeclarationIsolationTests
+ {
+ ///
+ /// AMQP reply code for PRECONDITION_FAILED (inequivalent queue arguments).
+ ///
+ private const ushort PreconditionFailedReplyCode = 406;
+
+ ///
+ /// Event id for .
+ ///
+ private const int ProviderTopologyDeclareFailedEventId = 5320;
+
+ ///
+ /// Alphabetically earlier backbone closes the topology channel; a later backbone still declares and gets a consumer.
+ ///
+ [Fact]
+ public async Task First_declare_failure_does_not_block_later_backbone()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ ArmQueueDeclareFailure(
+ factory,
+ queue => queue.Equals("backfiller.abavia", StringComparison.Ordinal)
+ ? CreatePreconditionFailed("backfiller.abavia")
+ : null);
+
+ var logger = new CollectingLogger();
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Abavia", 1), ("Giganews", 1)]),
+ CreateCapacity([("Abavia", 1), ("Giganews", 1)]),
+ logger);
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ var session = Assert.Single(consumer.Sessions);
+ Assert.Equal("Giganews", session.Backbone);
+ Assert.Equal(ArticleWorkConsumerState.Running, session.State);
+ Assert.Contains(
+ factory.LastConnection!.Channels,
+ static channel => channel.QueueDeclarations.Any(static q => q.Name == "backfiller.giganews"));
+ Assert.DoesNotContain(
+ factory.LastConnection.Channels.SelectMany(static c => c.QueueDeclarations),
+ static q => q.Name == "backfiller.abavia");
+ Assert.Contains(
+ logger.Entries,
+ static entry => entry.EventId.Id == ProviderTopologyDeclareFailedEventId
+ && entry.Message.Contains("backfiller.abavia", StringComparison.Ordinal)
+ && entry.Message.Contains($"replyCode={PreconditionFailedReplyCode}", StringComparison.Ordinal));
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// After a declare failure closes a channel, that channel is not reused for a later backbone.
+ ///
+ [Fact]
+ public async Task Closed_topology_channel_is_not_reused_for_later_backbone()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ ArmQueueDeclareFailure(
+ factory,
+ queue => queue.Equals("backfiller.abavia", StringComparison.Ordinal)
+ ? CreatePreconditionFailed("backfiller.abavia")
+ : null);
+
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Abavia", 1), ("Giganews", 1)]),
+ CreateCapacity([("Abavia", 1), ("Giganews", 1)]));
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ var topologyChannels = factory.LastConnection!.Channels
+ .Where(static channel => channel.ConsumeCount == 0)
+ .ToArray();
+ Assert.True(topologyChannels.Length >= 2);
+ var failed = Assert.Single(
+ topologyChannels,
+ static channel => channel.QueueDeclarations.Count == 0 && channel.DisposeCount == 1);
+ Assert.False(failed.IsOpen);
+ Assert.Contains(
+ failed.ExchangeDeclarations,
+ static exchange => exchange.Name == "backfiller.abavia");
+ var succeeded = Assert.Single(
+ topologyChannels,
+ static channel => channel.QueueDeclarations.Any(static q => q.Name == "backfiller.giganews"));
+ Assert.NotSame(failed, succeeded);
+ Assert.Equal(1, succeeded.DisposeCount);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Multiple independently failing backbones do not prevent a later healthy backbone from declaring and consuming.
+ ///
+ [Fact]
+ public async Task Multiple_declare_failures_do_not_block_a_later_healthy_backbone()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ ArmQueueDeclareFailure(
+ factory,
+ queue => queue is "backfiller.abavia" or "backfiller.eweka"
+ ? CreatePreconditionFailed(queue)
+ : null);
+
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Abavia", 1), ("Eweka", 1), ("Giganews", 1)]),
+ CreateCapacity([("Abavia", 1), ("Eweka", 1), ("Giganews", 1)]));
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ var session = Assert.Single(consumer.Sessions);
+ Assert.Equal("Giganews", session.Backbone);
+ Assert.Contains(
+ factory.LastConnection!.Channels,
+ static channel => channel.QueueDeclarations.Any(static q => q.Name == "backfiller.giganews"));
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// A backbone that failed topology declaration is retried on a later reconcile and can start once the failure clears.
+ ///
+ [Fact]
+ public async Task Failed_backbone_can_be_declared_on_a_later_reconcile_pass()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ var failAbavia = true;
+ ArmQueueDeclareFailure(
+ factory,
+ queue => failAbavia && queue.Equals("backfiller.abavia", StringComparison.Ordinal)
+ ? CreatePreconditionFailed("backfiller.abavia")
+ : null);
+
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Abavia", 1), ("Giganews", 1)]),
+ CreateCapacity([("Abavia", 1), ("Giganews", 1)]));
+
+ await consumer.StartAsync(CancellationToken.None);
+ Assert.Equal("Giganews", Assert.Single(consumer.Sessions).Backbone);
+
+ failAbavia = false;
+ await consumer.ReconcileAsync(CancellationToken.None);
+
+ Assert.Equal(2, consumer.Sessions.Count);
+ Assert.Contains(consumer.Sessions, static session => session.Backbone == "Abavia");
+ Assert.Contains(consumer.Sessions, static session => session.Backbone == "Giganews");
+ Assert.All(consumer.Sessions, static session => Assert.Equal(ArticleWorkConsumerState.Running, session.State));
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// An already-running healthy consumer is retained while another backbone's topology failure is isolated.
+ ///
+ [Fact]
+ public async Task Running_consumer_is_retained_when_another_backbone_declare_fails()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ var capacity = CreateCapacity([("Giganews", 1)]);
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Abavia", 1), ("Giganews", 1)]),
+ capacity);
+
+ await consumer.StartAsync(CancellationToken.None);
+ var healthy = Assert.Single(consumer.Sessions);
+ Assert.Equal("Giganews", healthy.Backbone);
+ var healthyChannel = Assert.IsType(healthy.Channel);
+
+ ArmQueueDeclareFailure(
+ factory,
+ queue => queue.Equals("backfiller.abavia", StringComparison.Ordinal)
+ ? CreatePreconditionFailed("backfiller.abavia")
+ : null);
+ capacity.PublishSnapshot(new Dictionary(StringComparer.OrdinalIgnoreCase)
+ {
+ ["Giganews"] = 1,
+ ["Abavia"] = 1,
+ });
+ await consumer.ReconcileAsync(CancellationToken.None);
+
+ Assert.Same(healthy, Assert.Single(consumer.Sessions));
+ Assert.Equal(ArticleWorkConsumerState.Running, healthy.State);
+ Assert.Same(healthyChannel, healthy.Channel);
+ Assert.Equal(0, healthyChannel.DisposeCount);
+ Assert.Contains(
+ factory.LastConnection!.Channels,
+ static channel => channel.ExchangeDeclarations.Any(static e => e.Name == "backfiller.abavia")
+ && channel.QueueDeclarations.Count == 0);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Topology channels opened for the pass are disposed after success and after failure.
+ ///
+ [Fact]
+ public async Task Topology_channels_are_disposed_after_success_and_failure()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ ArmQueueDeclareFailure(
+ factory,
+ queue => queue.Equals("backfiller.abavia", StringComparison.Ordinal)
+ ? CreatePreconditionFailed("backfiller.abavia")
+ : null);
+
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Abavia", 1), ("Giganews", 1)]),
+ CreateCapacity([("Abavia", 1), ("Giganews", 1)]));
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ var topologyChannels = factory.LastConnection!.Channels
+ .Where(static channel => channel.ConsumeCount == 0)
+ .ToArray();
+ Assert.All(topologyChannels, static channel => Assert.Equal(1, channel.DisposeCount));
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Cancellation during topology declaration stops further backbone attempts and propagates.
+ ///
+ [Fact]
+ public async Task Cancellation_stops_further_topology_declarations()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ using var cts = new CancellationTokenSource();
+ factory.LastConnection!.OnManualAckChannelCreated = _ => cts.Cancel();
+
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Abavia", 1), ("Giganews", 1)]),
+ CreateCapacity([("Abavia", 1), ("Giganews", 1)]));
+
+ await Assert.ThrowsAnyAsync(
+ () => consumer.StartAsync(cts.Token));
+
+ Assert.Empty(consumer.Sessions);
+ Assert.True(factory.LastConnection.Channels.Count <= 1);
+ Assert.DoesNotContain(
+ factory.LastConnection.Channels.SelectMany(static c => c.QueueDeclarations),
+ static q => q.Name == "backfiller.giganews");
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Successful declarations for several backbones share one open topology channel for the pass.
+ ///
+ [Fact]
+ public async Task Successful_declarations_reuse_one_open_topology_channel()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Eweka", 1), ("Giganews", 1)]),
+ CreateCapacity([("Eweka", 1), ("Giganews", 1)]));
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ Assert.Equal(2, consumer.Sessions.Count);
+ var topology = Assert.Single(
+ factory.LastConnection!.Channels,
+ static channel => channel.ConsumeCount == 0);
+ Assert.Equal(2, topology.QueueDeclarations.Count);
+ Assert.Equal(1, topology.DisposeCount);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ private static void ArmQueueDeclareFailure(
+ FakeBackFillerRabbitMqConnectionFactory factory,
+ Func failure)
+ {
+ factory.LastConnection!.OnManualAckChannelCreated = channel =>
+ {
+ channel.QueueDeclareFailure = failure;
+ };
+ }
+
+ private static OperationInterruptedException CreatePreconditionFailed(string queue)
+ {
+ var shutdown = new ShutdownEventArgs(
+ ShutdownInitiator.Peer,
+ PreconditionFailedReplyCode,
+ $"PRECONDITION_FAILED - inequivalent arg 'x-queue-type' for queue '{queue}'",
+ cause: null!,
+ CancellationToken.None);
+ return new OperationInterruptedException(shutdown);
+ }
+
+ private static ArticleWorkConsumerService CreateConsumer(
+ IRabbitMqService connections,
+ IBackFillerProviderCatalog catalog,
+ BackboneUsableCapacityState capacity,
+ ILogger? logger = null) =>
+ new(
+ connections,
+ RabbitMqServiceTests.CreateFastRuntime(),
+ new DeferredArticleWorkHandler(),
+ new RecordingArticleWorkResponsePublisher(),
+ logger ?? NullLogger.Instance,
+ catalog,
+ capacity);
+
+ private static StaticBackFillerProviderCatalog CreateCatalog(
+ IReadOnlyList<(string Backbone, int MaxSessions)> providers) =>
+ new(
+ [
+ ..providers.Select(static p => new BackFillerProviderDefinition(
+ p.Backbone,
+ "127.0.0.1",
+ 119,
+ false,
+ "nntp-user",
+ "p",
+ 0,
+ p.MaxSessions)),
+ ]);
+
+ private static BackboneUsableCapacityState CreateCapacity(
+ IReadOnlyList<(string Backbone, int Active)> capacities)
+ {
+ var state = new BackboneUsableCapacityState();
+ var snapshot = capacities
+ .Where(static pair => pair.Active > 0)
+ .ToDictionary(
+ static pair => pair.Backbone,
+ static pair => pair.Active,
+ StringComparer.OrdinalIgnoreCase);
+ if (snapshot.Count > 0)
+ {
+ state.PublishSnapshot(snapshot);
+ }
+
+ return state;
+ }
+ }
+}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
index 32f3d91f..1eca084e 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
@@ -411,6 +411,12 @@ internal sealed class FakeBackFillerRabbitMqChannel(long generation) : IRabbitMq
/// When set, thrown from queue declare instead of recording the declaration.
public Exception? QueueDeclareException { get; set; }
+ ///
+ /// When it returns an exception, that queue name fails and clears to model a broker channel close.
+ /// Checked before .
+ ///
+ public Func? QueueDeclareFailure { get; set; }
+
/// When set, thrown from queue bind instead of recording the binding.
public Exception? QueueBindException { get; set; }
@@ -493,8 +499,16 @@ public Task QueueDeclareAsync(
{
ArgumentException.ThrowIfNullOrWhiteSpace(queue);
cancellationToken.ThrowIfCancellationRequested();
+ if (QueueDeclareFailure?.Invoke(queue) is { } selectiveFailure)
+ {
+ // Broker PRECONDITION_FAILED and similar declare failures close the AMQP channel.
+ IsOpen = false;
+ throw selectiveFailure;
+ }
+
if (QueueDeclareException is not null)
{
+ IsOpen = false;
throw QueueDeclareException;
}