diff --git a/docs/architecture.md b/docs/architecture.md
index f5d34438..615a178f 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -22,11 +22,13 @@ Article Work shutdown uses `BackFillerShutdownOptions`:
## Article Work reconcile
-The consumer membership loop waits 15 seconds after a pass that started every desired consumer, including a pass with nothing to start. A pass that cannot declare topology because RabbitMQ is not ready, or because channel creation or declaration failed for a desired backbone, does not reset that delay. The next wait doubles, up to 2 minutes. An unexpected exception uses the same delay and does not end the loop. Shutdown cancellation ends the loop and is not a failure. The host stays healthy while RabbitMQ is temporarily unavailable. Capacity snapshots and connection replacement reconcile once on the shared gate and do not change the delay. If replacement cannot restore a session, the same periodic loop retries it. A later successful pass resets the delay to 15 seconds.
+The consumer membership loop waits 15 seconds after a pass that started every desired consumer, including a pass with nothing to start. A pass that cannot declare topology because RabbitMQ is not ready, because channel creation or declaration failed for a desired backbone, or because Management-plane quorum membership is deficient or unverifiable, does not reset that delay. The next wait doubles, up to 2 minutes. An unexpected exception uses the same delay and does not end the loop. Shutdown cancellation ends the loop and is not a failure. The host stays healthy while RabbitMQ is temporarily unavailable. Capacity snapshots and connection replacement reconcile once on the shared gate and do not change the delay. If replacement cannot restore a session, the same periodic loop retries it. A later successful pass resets the delay to 15 seconds.
+
+Provider queue type is derived from the configured unique `RabbitMQ:Hosts` count (never from reachable hosts or live cluster members). Exactly one host declares a durable classic queue (`x-queue-type=classic`) without quorum arguments or Management quorum health. Two or more hosts declare a durable quorum queue with `x-quorum-initial-group-size` and readiness membership floor equal to that host count, plus configured `x-delivery-limit`. All configured hosts must belong to the same RabbitMQ cluster; the client host list does not establish membership. Successful AMQP quorum declaration alone does not mark a backbone ready: readiness also requires Management HTTP verification of members, online members, and leader. Pre-existing queues with inequivalent type or immutable arguments are not deleted or recreated; operators must migrate or grow membership explicitly. Changing `Hosts` from one entry to multiple (or the reverse) changes the desired queue type and is not a safe in-place conversion. Management outages and timeouts leave multi-host backbones unready and follow the same reconcile backoff.
## Article Work settlement
-`ArticleWorkDispositionPlanner` maps one outcome to ACK or NACK, the NACK requeue flag, and whether a terminal response is published. `ArticleWorkSettlementTests` locks this map for replyable deliveries when cancellation is not already requested.
+`ArticleWorkDispositionPlanner` maps one outcome to ACK or NACK, the NACK requeue flag, and whether a terminal response is published. `ArticleWorkSettlementTests` locks this map for replyable deliveries when cancellation is not already requested. Broker `x-delivery-count` is compared to `ArticleWorkDeliveryLimit`: when the effective count is greater than or equal to the limit, otherwise-requeue outcomes (including cancellation override) settle without requeue so poison work cannot retry indefinitely. On quorum topology, a redelivery missing `x-delivery-count` is treated as exhausted; classic topology does not apply that conservative missing-count rule because classic queues do not provide the header. Negative settlement uses AMQP `basic.reject` via `IRabbitMqManualAckChannel.BasicRejectAsync` so quorum-queue delivery counts increment under RabbitMQ 4.3+ rules; `BasicNackAsync` remains true `basic.nack` for other callers and would not increment the count on 4.3+. The broker itself discards when the post-increment count exceeds `x-delivery-limit` on quorum queues. No dead-letter exchange is declared; exhausted messages are dropped. On multi-host (quorum) topology, a Management outage leaves new or recovering backbones unready even when AMQP is healthy (fail closed). Single-host classic readiness does not call the quorum Management probe.
| Outcome | Broker action | Publish terminal response |
|---|---|---|
@@ -34,12 +36,12 @@ The consumer membership loop waits 15 seconds after a pass that started every de
| ArticleNotFound | Basic.Ack | Yes |
| InvalidArticle | Basic.Ack | Yes. Includes an article `ArticleWorkDeliveryFraming` cannot send. That rejection happens before retention, so the article is not stored. |
| InvalidRequest | Basic.Ack | Yes when both `CorrelationId` and `ReplyTo` are present. Otherwise ACK without publication |
-| ProviderFailure | Basic.Nack, requeue true | No |
-| Cancelled | Basic.Nack, requeue true | No |
-| UnexpectedFailure | Basic.Nack, requeue true | No |
-| RetentionRejected | Basic.Nack, requeue true | No |
+| ProviderFailure | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No |
+| Cancelled | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No |
+| UnexpectedFailure | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No |
+| RetentionRejected | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No |
-If cancellation is already requested, the planner returns NACK requeue and does not publish, including when the handler outcome was terminal.
+If cancellation is already requested, the planner returns NACK (requeue while retries remain) and does not publish, including when the handler outcome was terminal.
`ArticleWorkSettlementLease` is one settlement attempt for one delivery tag on the original consumer channel and connection generation. It sets the settled flag before the broker RPC. If that RPC throws, the flag stays set and the lease does not retry. If the channel is stale, closed, or not the original channel, `TrySettleAsync` returns false without ACK or NACK, and the broker may redeliver. Pipeline settlement calls use `CancellationToken.None`.
diff --git a/docs/common/configuration.md b/docs/common/configuration.md
index 0cba3fd4..d6b47f71 100644
--- a/docs/common/configuration.md
+++ b/docs/common/configuration.md
@@ -26,7 +26,7 @@ There is no application-level RabbitMQ connection pool. `MinConnections` and `Ma
| Key | Type | Default | Required? | Description |
|-----|------|---------|-----------|-------------|
-| `Hosts` | string array | _(none)_ | **yes** | Broker hostnames or IP addresses (no URI scheme, credentials, path, or query). When TLS is enabled, each host is dialed with that hostname as the TLS SNI / certificate identity (not `Hosts[0]` for every endpoint). |
+| `Hosts` | string array | _(none)_ | **yes** | Broker hostnames or IP addresses (no URI scheme, credentials, path, or query). Entries must be unique (case-insensitive); unique count is `1–7`. When TLS is enabled, each host is dialed with that hostname as the TLS SNI / certificate identity (not `Hosts[0]` for every endpoint). Article Work queue type is derived from unique count: one host → classic queue; two or more → quorum queue with `x-quorum-initial-group-size` equal to that count. All entries must belong to the same RabbitMQ cluster; this list does not establish cluster membership. Changing from one host to multiple (or the reverse) changes the desired queue type and may require an explicit operator migration—queues are never deleted or recreated automatically. |
| `Port` | int | `5672` | **yes** | AMQP TCP port (`1–65535`) |
| `Username` | string | _(none)_ | **yes** | Broker username. Maximum 45 characters. Never log |
| `Password` | string | _(none)_ | **yes** | Broker password. Maximum 45 characters. Never commit or log |
@@ -42,6 +42,7 @@ There is no application-level RabbitMQ connection pool. `MinConnections` and `Ma
| `PoolReconnectMaxDelayMs` | int | `30000` | **yes** | Application reconnect max delay (`50–300000`; ≥ base) |
| `ChannelLeaseTimeoutSeconds` | int | `60` | **yes** | Validated; reserved for later channel work (`1–3600`; ≥ `RpcTimeoutSeconds`) |
| `WorkRequestMaxPayloadBytes` | int | `1024` | **yes** | Maximum admitted Article Work request payload bytes (`1–4096`) |
+| `ArticleWorkDeliveryLimit` | int | `10` | **yes** | Declared as `x-delivery-limit` on quorum Article Work queues and used as the application requeue cap when broker delivery-count metadata is available (`1–1000`). Not declared on classic single-host queues (broker does not accept that argument there). With count `0..(limit-1)` the app may requeue; at `limit` it rejects without requeue. No DLX is declared; exhausted messages are dropped |
| `ChannelPoolSize` | int | `512` | **yes** | Validated; reserved for later consumer buffering (`1–8192`) |
| `MinConnections` | int | `4` | **yes** | Validated; reserved (`1–512`; ≤ `MaxConnections`) |
| `MaxConnections` | int | `16` | **yes** | Validated; reserved (`1–512`) |
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
index 1326cdcf..34d37315 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs
@@ -46,6 +46,9 @@ internal sealed class ArticleWorkConsumerService : IHostedService, IAsyncDisposa
/// Consumer logger. This type does not write payloads or credentials.
private readonly ILogger _logger;
+ /// Management-plane quorum membership probe. Separate from AMQP declaration.
+ private readonly IArticleWorkQuorumHealthProbe _quorumHealth;
+
/// Provider catalogue read when building desired consume slots. An omitted catalogue is empty.
private readonly IBackFillerProviderCatalog _catalog;
@@ -78,6 +81,12 @@ internal sealed class ArticleWorkConsumerService : IHostedService, IAsyncDisposa
/// Tracked sessions keyed by using ordinal comparison.
private readonly Dictionary _sessions = new(StringComparer.Ordinal);
+ ///
+ /// Last logged quorum-health kind per backbone. Suppresses repeated Warning/Information on unchanged membership.
+ ///
+ private readonly Dictionary _lastQuorumHealthKind =
+ new(StringComparer.OrdinalIgnoreCase);
+
/// Serializes start, reconcile, connection replacement, and shutdown session replacement.
private readonly SemaphoreSlim _replaceGate = new(1, 1);
@@ -119,6 +128,10 @@ internal sealed class ArticleWorkConsumerService : IHostedService, IAsyncDisposa
///
/// Optional loop body. Null reconciles membership. A thrown exception is supervised inside the same loop task.
///
+ ///
+ /// Quorum membership probe. When omitted, tests use .
+ /// Production DI registers .
+ ///
///
/// Thrown when , , ,
/// , or is null.
@@ -133,7 +146,8 @@ internal ArticleWorkConsumerService(
IBackFillerProviderCatalog? catalog = null,
IBackboneUsableCapacityProvider? capacity = null,
TimeProvider? time = null,
- Func? reconcileLoopBody = null)
+ Func? reconcileLoopBody = null,
+ IArticleWorkQuorumHealthProbe? quorumHealth = null)
{
ArgumentNullException.ThrowIfNull(connections);
ArgumentNullException.ThrowIfNull(runtime);
@@ -149,6 +163,7 @@ internal ArticleWorkConsumerService(
_capacity = capacity ?? new BackboneUsableCapacityState();
_time = time ?? TimeProvider.System;
_reconcileLoopBody = reconcileLoopBody;
+ _quorumHealth = quorumHealth ?? AlwaysHealthyArticleWorkQuorumHealthProbe.Instance;
}
///
@@ -597,7 +612,9 @@ private async Task ReconcileSessionsAsync(CancellationToken cance
var pipeline = new ArticleWorkDeliveryPipeline(
_handler,
_publisher,
- _runtime.RabbitMq.WorkRequestMaxPayloadBytes);
+ _runtime.RabbitMq.WorkRequestMaxPayloadBytes,
+ _runtime.RabbitMq.DeliveryLimit,
+ treatMissingRedeliveryCountAsExhausted: _runtime.RabbitMq.QueueTopology.RequiresQuorumHealth);
var prefetch = _runtime.RabbitMq.ConsumerPrefetchCount is { } configured and > 0
? configured
: (ushort)1;
@@ -722,18 +739,23 @@ private enum ReconcilePass
}
///
- /// Declares durable quorum fanout topology for each backbone that is about to receive
+ /// Declares durable Hosts-derived classic or quorum fanout topology for each backbone that is about to receive
/// consumers. Topology is never deleted when capacity later drops to zero.
///
/// Backbones that still need consumers. Blank names are ignored. Comparison is ordinal ignore-case.
/// Cancels channel creation and each declaration. Cancellation propagates.
///
- /// Backbones whose declaration succeeded. Empty when there is nothing to declare, the connection is not ready, or every declaration failed.
+ /// Backbones whose declaration succeeded and, when quorum topology is selected, whose Management-plane
+ /// membership verification also succeeded. Classic single-host topology becomes ready after successful declaration.
+ /// Empty when there is nothing to declare, the connection is not ready, every declaration failed,
+ /// or (for quorum) membership could not be verified as healthy.
///
///
/// Declarations reuse one manual-ack channel while it remains open. A failed declaration disposes that channel
/// before the next backbone is attempted, because RabbitMQ closes the channel on broker-side declare failures
/// such as PRECONDITION_FAILED. A later backbone therefore never reuses an unusable channel.
+ /// For quorum topology, successful declaration alone does not mark a backbone ready; Management-plane membership
+ /// must also be healthy. Classic topology does not call the quorum Management probe.
/// The connection generation is rechecked before each channel open. is not logged.
///
private async Task> EnsureProviderTopologyAsync(
@@ -754,12 +776,15 @@ private async Task> EnsureProviderTopologyAsync(
if (!_connections.TryGetCurrent(out _))
{
+ var topology = _runtime.RabbitMq.QueueTopology;
foreach (var backbone in unique)
{
ArticleWorkLogMessages.ProviderTopologyDeclareFailed(
_logger,
backbone,
BackFillerRabbitMqTopology.ComposeProviderEntity(backbone),
+ topology.Kind.ToString(),
+ topology.ConfiguredHostCount,
"RabbitMQ connection is not ready for topology declaration.");
}
@@ -779,26 +804,52 @@ private async Task> EnsureProviderTopologyAsync(
.ConfigureAwait(false);
if (channel is null)
{
+ var notReadyTopology = _runtime.RabbitMq.QueueTopology;
ArticleWorkLogMessages.ProviderTopologyDeclareFailed(
_logger,
backbone,
queue,
+ notReadyTopology.Kind.ToString(),
+ notReadyTopology.ConfiguredHostCount,
"RabbitMQ connection is not ready for topology declaration.");
continue;
}
await BackFillerArticleWorkTopology
- .DeclareProviderEndpointAsync(channel, backbone, cancellationToken)
+ .DeclareProviderEndpointAsync(channel, backbone, _runtime, cancellationToken)
.ConfigureAwait(false);
- ready.Add(backbone);
- ArticleWorkLogMessages.ProviderTopologyDeclared(_logger, backbone, queue);
+ var topology = _runtime.RabbitMq.QueueTopology;
+ ArticleWorkLogMessages.ProviderTopologyDeclared(
+ _logger,
+ backbone,
+ queue,
+ topology.Kind.ToString(),
+ topology.ConfiguredHostCount);
+
+ if (!topology.RequiresQuorumHealth)
+ {
+ ready.Add(backbone);
+ continue;
+ }
+
+ var health = await _quorumHealth
+ .InspectAsync(queue, topology.QuorumInitialGroupSize!.Value, cancellationToken)
+ .ConfigureAwait(false);
+ LogQuorumHealthTransition(backbone, queue, health);
+ if (health.Kind == ArticleWorkQuorumHealthKind.Healthy)
+ {
+ ready.Add(backbone);
+ }
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
+ var topology = _runtime.RabbitMq.QueueTopology;
ArticleWorkLogMessages.ProviderTopologyDeclareFailed(
_logger,
backbone,
queue,
+ topology.Kind.ToString(),
+ topology.ConfiguredHostCount,
FormatTopologyDeclareFailure(ex));
channel = await DisposeTopologyChannelAsync(channel).ConfigureAwait(false);
}
@@ -854,6 +905,50 @@ await BackFillerArticleWorkTopology
return null;
}
+ ///
+ /// Logs quorum membership only when the classification changes for .
+ ///
+ /// Provider backbone label.
+ /// Provider queue name.
+ /// Latest membership observation.
+ private void LogQuorumHealthTransition(
+ string backbone,
+ string queue,
+ ArticleWorkQuorumHealthResult health)
+ {
+ lock (_gate)
+ {
+ if (_lastQuorumHealthKind.TryGetValue(backbone, out var previous) && previous == health.Kind)
+ {
+ return;
+ }
+
+ _lastQuorumHealthKind[backbone] = health.Kind;
+ }
+
+ if (health.Kind == ArticleWorkQuorumHealthKind.Healthy)
+ {
+ ArticleWorkLogMessages.QuorumMembershipHealthy(
+ _logger,
+ backbone,
+ queue,
+ health.RequiredMembers,
+ health.ObservedMembers ?? -1,
+ health.OnlineMembers ?? -1);
+ return;
+ }
+
+ ArticleWorkLogMessages.QuorumMembershipNotReady(
+ _logger,
+ backbone,
+ queue,
+ health.Kind,
+ health.RequiredMembers,
+ health.ObservedMembers ?? -1,
+ health.OnlineMembers ?? -1,
+ health.Reason);
+ }
+
///
/// Formats a topology declaration failure, preferring structured AMQP shutdown details when present.
///
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs
index 54c46502..37304b69 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs
@@ -730,6 +730,38 @@ bool ChannelStillCurrent() =>
delivery.DeliveryTag,
reason);
}
+ else if (IsDeliveryLimitExhausted(delivery, outcome))
+ {
+ ArticleWorkLogMessages.DeliveryLimitExhausted(
+ _logger,
+ _backbone,
+ delivery.Generation,
+ delivery.DeliveryTag,
+ delivery.DeliveryCount.GetValueOrDefault(),
+ _pipeline.DeliveryLimit,
+ outcome.ToString());
+ }
+ }
+
+ ///
+ /// True when a retryable outcome was settled after the broker delivery count reached the configured limit.
+ ///
+ private bool IsDeliveryLimitExhausted(RabbitMqManualAckDelivery delivery, ArticleWorkOutcome outcome)
+ {
+ var effectiveCount = ArticleWorkDispositionPlanner.ResolveEffectiveDeliveryCount(
+ delivery.DeliveryCount,
+ delivery.Redelivered,
+ _pipeline.DeliveryLimit,
+ _pipeline.TreatMissingRedeliveryCountAsExhausted);
+ if (effectiveCount < _pipeline.DeliveryLimit)
+ {
+ return false;
+ }
+
+ return outcome is ArticleWorkOutcome.ProviderFailure
+ or ArticleWorkOutcome.Cancelled
+ or ArticleWorkOutcome.UnexpectedFailure
+ or ArticleWorkOutcome.RetentionRejected;
}
///
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs
index 60468ea5..06046423 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs
@@ -17,28 +17,59 @@ internal sealed class ArticleWorkDeliveryPipeline
/// Maximum JSON body size passed to .
private readonly int _maxPayloadBytes;
+ /// Configured queue x-delivery-limit; application requeue cap uses the same value.
+ private readonly int _deliveryLimit;
+
+ ///
+ /// When , a redelivery without x-delivery-count is treated as exhausted (quorum).
+ /// Classic queues pass .
+ ///
+ private readonly bool _treatMissingRedeliveryCountAsExhausted;
+
/// Maximum JSON body size passed to .
internal int MaxPayloadBytes => _maxPayloadBytes;
+ /// Finite delivery limit applied to retryable dispositions.
+ internal int DeliveryLimit => _deliveryLimit;
+
+ ///
+ /// Whether missing broker delivery-count on redelivery is treated as exhausted.
+ ///
+ internal bool TreatMissingRedeliveryCountAsExhausted => _treatMissingRedeliveryCountAsExhausted;
+
///
/// Initializes a new pipeline.
///
/// Admitted-work handler.
/// Response-publish seam.
/// Maximum accepted JSON body size. Must be at least 1.
+ ///
+ /// Finite delivery limit mirrored from queue declaration. Must be at least 1.
+ /// Defaults to 10 so existing tests retain the production default.
+ ///
+ ///
+ /// Quorum topology passes (default). Classic topology passes .
+ ///
/// Thrown when or is null.
- /// Thrown when is less than 1.
+ ///
+ /// Thrown when or is less than 1.
+ ///
internal ArticleWorkDeliveryPipeline(
IArticleWorkHandler handler,
IArticleWorkResponsePublisher publisher,
- int maxPayloadBytes)
+ int maxPayloadBytes,
+ int deliveryLimit = 10,
+ bool treatMissingRedeliveryCountAsExhausted = true)
{
ArgumentNullException.ThrowIfNull(handler);
ArgumentNullException.ThrowIfNull(publisher);
ArgumentOutOfRangeException.ThrowIfLessThan(maxPayloadBytes, 1);
+ ArgumentOutOfRangeException.ThrowIfLessThan(deliveryLimit, 1);
_handler = handler;
_publisher = publisher;
_maxPayloadBytes = maxPayloadBytes;
+ _deliveryLimit = deliveryLimit;
+ _treatMissingRedeliveryCountAsExhausted = treatMissingRedeliveryCountAsExhausted;
}
///
@@ -104,7 +135,11 @@ internal async Task ProcessAsync(
var invalid = ArticleWorkDispositionPlanner.Create(
ArticleWorkOutcome.InvalidRequest,
replyable,
- cancellationRequested: false);
+ cancellationRequested: false,
+ delivery.DeliveryCount,
+ delivery.Redelivered,
+ _deliveryLimit,
+ _treatMissingRedeliveryCountAsExhausted);
if (!channelStillCurrent() || !lease.IsOriginalChannel(channel))
{
return ArticleWorkOutcome.InvalidRequest;
@@ -130,7 +165,9 @@ internal async Task ProcessAsync(
ArticleWorkOutcome.InvalidRequest,
lease,
channel,
- channelStillCurrent)
+ channelStillCurrent,
+ delivery.DeliveryCount,
+ delivery.Redelivered)
.ConfigureAwait(false);
return publishOutcome;
}
@@ -181,7 +218,11 @@ internal async Task ProcessAsync(
var disposition = ArticleWorkDispositionPlanner.Create(
outcome,
replyable: true,
- cancellationToken.IsCancellationRequested || outcome == ArticleWorkOutcome.Cancelled);
+ cancellationToken.IsCancellationRequested || outcome == ArticleWorkOutcome.Cancelled,
+ delivery.DeliveryCount,
+ delivery.Redelivered,
+ _deliveryLimit,
+ _treatMissingRedeliveryCountAsExhausted);
if (!channelStillCurrent() || !lease.IsOriginalChannel(channel))
{
@@ -190,7 +231,7 @@ internal async Task ProcessAsync(
if (outcome == ArticleWorkOutcome.Success && !_publisher.CompletesSuccessPublication)
{
- var pending = new ArticleWorkDisposition(Acknowledge: false, Requeue: true, PublishResponse: false);
+ var pending = CreateRetryDisposition(delivery.DeliveryCount, delivery.Redelivered);
await lease.TrySettleAsync(
pending,
channelStillCurrent() && lease.IsOriginalChannel(channel),
@@ -224,24 +265,30 @@ await lease.TrySettleAsync(
outcome,
lease,
channel,
- channelStillCurrent)
+ channelStillCurrent,
+ delivery.DeliveryCount,
+ delivery.Redelivered)
.ConfigureAwait(false);
return finalOutcome;
}
- /// NACK-requeues the lease when the original channel is still current.
+ /// NACK with requeue while retries remain; NACK without requeue when the delivery limit is exhausted.
/// Settlement lease for this delivery.
/// Channel originally bound to .
/// Rechecked immediately before the NACK.
+ /// Broker x-delivery-count for this delivery.
+ /// AMQP redelivered flag for missing-count handling.
///
/// A task that completes when the NACK attempt finishes. The lease swallows broker failures and does not retry them.
///
- private static async Task SettleRetryableAsync(
+ private async Task SettleRetryableAsync(
ArticleWorkSettlementLease lease,
IRabbitMqManualAckChannel channel,
- Func channelStillCurrent)
+ Func channelStillCurrent,
+ int? deliveryCount,
+ bool redelivered)
{
- var retry = new ArticleWorkDisposition(Acknowledge: false, Requeue: true, PublishResponse: false);
+ var retry = CreateRetryDisposition(deliveryCount, redelivered);
await lease.TrySettleAsync(
retry,
channelStillCurrent() && lease.IsOriginalChannel(channel),
@@ -249,8 +296,20 @@ await lease.TrySettleAsync(
.ConfigureAwait(false);
}
+ /// Builds the retry-or-discard disposition for the current delivery count.
+ private ArticleWorkDisposition CreateRetryDisposition(int? deliveryCount, bool redelivered) =>
+ ArticleWorkDispositionPlanner.Create(
+ ArticleWorkOutcome.UnexpectedFailure,
+ replyable: false,
+ cancellationRequested: false,
+ deliveryCount,
+ redelivered,
+ _deliveryLimit,
+ _treatMissingRedeliveryCountAsExhausted);
+
///
- /// Settles after a publication attempt. A required publication that is cancelled or fails NACK-requeues.
+ /// Settles after a publication attempt. A required publication that is cancelled or fails NACK-requeues
+ /// while retries remain.
///
/// Result of the publication attempt. includes attempts that were not required.
/// Plan produced before publication. Submitted only when is confirmed.
@@ -258,6 +317,8 @@ await lease.TrySettleAsync(
/// Settlement lease for this delivery. One attempt; a failed NACK does not settle again.
/// Channel originally bound to .
/// Rechecked immediately before settlement. A stale or replacement channel is not settled.
+ /// Broker x-delivery-count for this delivery.
+ /// AMQP redelivered flag for missing-count handling.
///
/// when the original disposition is submitted.
/// when publication was cancelled or failed.
@@ -266,17 +327,20 @@ await lease.TrySettleAsync(
/// Broker RPCs use . A lost publisher confirm is reported as failure, so the
/// requeue can publish another reply with the same correlation id. That is not exactly-once delivery.
///
- private static async Task CompleteAfterPublishAttemptAsync(
+ private async Task CompleteAfterPublishAttemptAsync(
ArticleWorkPublishAttempt published,
ArticleWorkDisposition disposition,
ArticleWorkOutcome outcome,
ArticleWorkSettlementLease lease,
IRabbitMqManualAckChannel channel,
- Func channelStillCurrent)
+ Func channelStillCurrent,
+ int? deliveryCount,
+ bool redelivered)
{
if (published != ArticleWorkPublishAttempt.Confirmed)
{
- await SettleRetryableAsync(lease, channel, channelStillCurrent).ConfigureAwait(false);
+ await SettleRetryableAsync(lease, channel, channelStillCurrent, deliveryCount, redelivered)
+ .ConfigureAwait(false);
return ArticleWorkOutcome.UnexpectedFailure;
}
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs
index 840c2bc1..6b6a6f81 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs
@@ -5,6 +5,49 @@ namespace VectorNNTPD.BackFillerService.ArticleWork
///
internal static class ArticleWorkDispositionPlanner
{
+ ///
+ /// Resolves the delivery-count value used for the finite retry policy.
+ ///
+ /// Broker x-delivery-count when present and non-negative.
+ /// AMQP redelivered flag from the delivery.
+ /// Configured finite limit. Used when metadata is missing on a quorum redelivery.
+ ///
+ /// When (quorum), a redelivery without a usable count is treated as exhausted so
+ /// missing metadata cannot bypass the finite retry policy. When (classic),
+ /// missing count remains 0 because classic queues do not provide x-delivery-count.
+ ///
+ ///
+ /// when present; 0 on a first delivery without the header;
+ /// when is
+ /// and a redelivery omits a usable count; otherwise 0.
+ ///
+ ///
+ /// Quorum queues include x-delivery-count on redelivered messages. The broker discards when
+ /// the post-increment count is greater than x-delivery-limit. Application exhaustion uses
+ /// count >= limit on the count observed at delivery time, which aligns with that boundary:
+ /// counts 0..(limit-1) may requeue; a delivery that already shows limit is not requeued.
+ ///
+ /// Thrown when is less than 1.
+ internal static int ResolveEffectiveDeliveryCount(
+ int? deliveryCount,
+ bool redelivered,
+ int deliveryLimit,
+ bool treatMissingRedeliveryCountAsExhausted = true)
+ {
+ ArgumentOutOfRangeException.ThrowIfLessThan(deliveryLimit, 1);
+ if (deliveryCount is { } count && count >= 0)
+ {
+ return count;
+ }
+
+ if (treatMissingRedeliveryCountAsExhausted && redelivered)
+ {
+ return deliveryLimit;
+ }
+
+ return 0;
+ }
+
///
/// Creates the settlement plan for one classified result.
///
@@ -15,19 +58,45 @@ internal static class ArticleWorkDispositionPlanner
///
///
/// When , forces settlement
- /// (NACK requeue, no terminal response) and ignores .
+ /// (NACK requeue when retries remain, no terminal response) and ignores .
+ ///
+ ///
+ /// Broker x-delivery-count when present. Pass the raw header value; this method applies
+ /// with .
+ ///
+ /// AMQP redelivered flag for conservative missing-count handling.
+ ///
+ /// Configured finite delivery limit. When the effective count is greater than or equal
+ /// to this value, an otherwise-requeue disposition becomes NACK without requeue so poison work cannot
+ /// retry indefinitely. Must be at least 1.
+ ///
+ ///
+ /// Quorum queues pass ; classic queues pass .
///
///
- /// The disposition to apply. An unrecognized is NACK requeue without publication.
+ /// The disposition to apply. An unrecognized is NACK requeue without publication
+ /// unless the delivery limit is exhausted.
///
+ /// Thrown when is less than 1.
internal static ArticleWorkDisposition Create(
ArticleWorkOutcome outcome,
bool replyable,
- bool cancellationRequested)
+ bool cancellationRequested,
+ int? deliveryCount,
+ bool redelivered,
+ int deliveryLimit,
+ bool treatMissingRedeliveryCountAsExhausted = true)
{
+ var effectiveCount = ResolveEffectiveDeliveryCount(
+ deliveryCount,
+ redelivered,
+ deliveryLimit,
+ treatMissingRedeliveryCountAsExhausted);
+ var retriesExhausted = effectiveCount >= deliveryLimit;
+
if (cancellationRequested)
{
- return new ArticleWorkDisposition(Acknowledge: false, Requeue: true, PublishResponse: false);
+ return CreateRetryable(retriesExhausted);
}
return outcome switch
@@ -44,9 +113,13 @@ internal static ArticleWorkDisposition Create(
or ArticleWorkOutcome.Cancelled
or ArticleWorkOutcome.UnexpectedFailure
or ArticleWorkOutcome.RetentionRejected =>
- new ArticleWorkDisposition(Acknowledge: false, Requeue: true, PublishResponse: false),
- _ => new ArticleWorkDisposition(Acknowledge: false, Requeue: true, PublishResponse: false),
+ CreateRetryable(retriesExhausted),
+ _ => CreateRetryable(retriesExhausted),
};
}
+
+ /// NACK requeue while retries remain; NACK without requeue when the delivery limit is exhausted.
+ private static ArticleWorkDisposition CreateRetryable(bool retriesExhausted) =>
+ new(Acknowledge: false, Requeue: !retriesExhausted, PublishResponse: false);
}
}
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs
index c41a316b..443aa6d4 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs
@@ -290,16 +290,23 @@ internal static partial void PublicationConfirmed(
internal static partial void ConsumerReconcileCompleted(ILogger logger, int sessionCount);
///
- /// Writes an Information event after one backbone's provider exchange, quorum queue, and binding are declared.
+ /// Writes an Information event after one backbone's provider exchange, queue, and binding are declared.
///
/// Logger that receives the event.
/// Backbone whose topology declaration succeeded.
/// Provider entity name that was declared.
+ /// Hosts-derived queue type (Classic or Quorum).
+ /// Unique configured RabbitMQ:Hosts count used to derive topology.
[LoggerMessage(
EventId = 5319,
Level = LogLevel.Information,
- Message = "Article Work provider topology declared backbone={Backbone} queue={Queue}")]
- internal static partial void ProviderTopologyDeclared(ILogger logger, string backbone, string queue);
+ Message = "Article Work provider topology declared backbone={Backbone} queue={Queue} desiredQueueType={DesiredQueueType} configuredHostCount={ConfiguredHostCount}")]
+ internal static partial void ProviderTopologyDeclared(
+ ILogger logger,
+ string backbone,
+ string queue,
+ string desiredQueueType,
+ int configuredHostCount);
///
/// Writes an Error event when provider topology cannot be declared because the connection is not ready, channel creation fails, or that backbone's declaration fails.
@@ -307,6 +314,8 @@ internal static partial void PublicationConfirmed(
/// Logger that receives the event.
/// Backbone whose declaration did not succeed.
/// Provider entity name that was not left ready.
+ /// Hosts-derived queue type that was attempted.
+ /// Unique configured host count that selected that queue type.
///
/// Caller-supplied failure text. The service passes or a fixed not-ready message.
/// is not logged here. This method does not throw.
@@ -314,11 +323,13 @@ internal static partial void PublicationConfirmed(
[LoggerMessage(
EventId = 5320,
Level = LogLevel.Error,
- Message = "Article Work provider topology declaration failed backbone={Backbone} queue={Queue}: {Reason}")]
+ Message = "Article Work provider topology declaration failed backbone={Backbone} queue={Queue} desiredQueueType={DesiredQueueType} configuredHostCount={ConfiguredHostCount}: {Reason}")]
internal static partial void ProviderTopologyDeclareFailed(
ILogger logger,
string backbone,
string queue,
+ string desiredQueueType,
+ int configuredHostCount,
string reason);
///
@@ -391,5 +402,74 @@ internal static partial void PublisherChannelFaulted(
long generation,
RabbitMqChannelFaultKind kind,
string reason);
+
+ ///
+ /// Writes a Warning event when topology declaration succeeded but quorum membership is not ready.
+ ///
+ /// Logger that receives the event.
+ /// Backbone whose membership is insufficient or unverifiable.
+ /// Provider queue name.
+ /// Membership health classification.
+ /// Configured membership floor.
+ /// Observed configured members, or -1 when unknown.
+ /// Observed online members, or -1 when unknown.
+ /// Stable diagnostic text. Never credentials.
+ [LoggerMessage(
+ EventId = 5325,
+ Level = LogLevel.Warning,
+ Message = "Article Work quorum membership not ready backbone={Backbone} queue={Queue} kind={Kind} requiredMembers={RequiredMembers} observedMembers={ObservedMembers} onlineMembers={OnlineMembers}: {Reason}")]
+ internal static partial void QuorumMembershipNotReady(
+ ILogger logger,
+ string backbone,
+ string queue,
+ ArticleWorkQuorumHealthKind kind,
+ int requiredMembers,
+ int observedMembers,
+ int onlineMembers,
+ string reason);
+
+ ///
+ /// Writes an Information event when a previously unready queue now meets the membership policy.
+ ///
+ /// Logger that receives the event.
+ /// Backbone whose membership became healthy.
+ /// Provider queue name.
+ /// Configured membership floor.
+ /// Verified configured members.
+ /// Verified online members.
+ [LoggerMessage(
+ EventId = 5326,
+ Level = LogLevel.Information,
+ Message = "Article Work quorum membership healthy backbone={Backbone} queue={Queue} requiredMembers={RequiredMembers} observedMembers={ObservedMembers} onlineMembers={OnlineMembers}")]
+ internal static partial void QuorumMembershipHealthy(
+ ILogger logger,
+ string backbone,
+ string queue,
+ int requiredMembers,
+ int observedMembers,
+ int onlineMembers);
+
+ ///
+ /// Writes a Warning event when a delivery is NACKed without requeue because the delivery limit is exhausted.
+ ///
+ /// Logger that receives the event.
+ /// Provider backbone of the session.
+ /// Delivery connection generation.
+ /// Channel-scoped delivery tag.
+ /// Broker x-delivery-count for this delivery.
+ /// Configured finite delivery limit.
+ /// Outcome that would have requeued if retries remained.
+ [LoggerMessage(
+ EventId = 5327,
+ Level = LogLevel.Warning,
+ Message = "Article Work delivery limit exhausted backbone={Backbone} generation={Generation} deliveryTag={DeliveryTag} deliveryCount={DeliveryCount} deliveryLimit={DeliveryLimit} outcome={Outcome}")]
+ internal static partial void DeliveryLimitExhausted(
+ ILogger logger,
+ string backbone,
+ long generation,
+ ulong deliveryTag,
+ int deliveryCount,
+ int deliveryLimit,
+ string outcome);
}
}
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumHealth.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumHealth.cs
new file mode 100644
index 00000000..061e9332
--- /dev/null
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumHealth.cs
@@ -0,0 +1,89 @@
+namespace VectorNNTPD.BackFillerService.ArticleWork
+{
+ ///
+ /// Classification of an Article Work quorum-membership health observation.
+ ///
+ internal enum ArticleWorkQuorumHealthKind
+ {
+ /// Configured membership and online availability both meet the required policy.
+ Healthy = 0,
+
+ /// Queue exists but has fewer configured members than required.
+ InsufficientMembers = 1,
+
+ /// Queue has enough configured members, but too few are online.
+ InsufficientOnlineMembers = 2,
+
+ /// Management reported the queue as missing.
+ QueueMissing = 3,
+
+ /// Membership could not be verified (Management unavailable, timeout, or malformed response).
+ Unverifiable = 4,
+ }
+
+ ///
+ /// Verified (or unverifiable) quorum membership observation for one provider queue.
+ ///
+ /// Readiness classification.
+ /// Configured membership floor that must be met.
+ /// Configured quorum members when reported; otherwise .
+ /// Online quorum members when reported; otherwise .
+ /// Leader node name when reported; otherwise .
+ /// Stable diagnostic text suitable for structured logs. Never credentials.
+ internal readonly record struct ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind Kind,
+ int RequiredMembers,
+ int? ObservedMembers,
+ int? OnlineMembers,
+ string? Leader,
+ string Reason);
+
+ ///
+ /// Management-plane probe that verifies Article Work quorum membership independently of AMQP declaration.
+ ///
+ internal interface IArticleWorkQuorumHealthProbe
+ {
+ ///
+ /// Inspects quorum membership for against .
+ ///
+ /// Provider queue name (for example backfiller.Giganews).
+ /// Configured membership floor from runtime options.
+ /// Cancels the Management HTTP request.
+ /// A verified or unverifiable health result. Never throws for broker/HTTP faults.
+ Task InspectAsync(
+ string queueName,
+ int requiredMembers,
+ CancellationToken cancellationToken);
+ }
+
+ ///
+ /// Test and DI stand-in that always reports healthy membership.
+ ///
+ ///
+ /// Production DI registers . Unit tests that do not
+ /// exercise membership use this stand-in so declaration success remains the readiness gate under test.
+ ///
+ internal sealed class AlwaysHealthyArticleWorkQuorumHealthProbe : IArticleWorkQuorumHealthProbe
+ {
+ /// Shared instance for tests.
+ internal static AlwaysHealthyArticleWorkQuorumHealthProbe Instance { get; } = new();
+
+ ///
+ public Task InspectAsync(
+ string queueName,
+ int requiredMembers,
+ CancellationToken cancellationToken)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(queueName);
+ ArgumentOutOfRangeException.ThrowIfLessThan(requiredMembers, 1);
+ return Task.FromResult(
+ new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.Healthy,
+ requiredMembers,
+ ObservedMembers: requiredMembers,
+ OnlineMembers: requiredMembers,
+ Leader: "test-leader",
+ Reason: "Test stand-in reports healthy quorum membership."));
+ }
+ }
+}
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementHealthProbe.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementHealthProbe.cs
new file mode 100644
index 00000000..8ea79bbd
--- /dev/null
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementHealthProbe.cs
@@ -0,0 +1,237 @@
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+using Microsoft.Extensions.Options;
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
+namespace VectorNNTPD.BackFillerService.ArticleWork
+{
+ ///
+ /// Verifies Article Work quorum membership through the RabbitMQ Management HTTP API.
+ ///
+ ///
+ /// Uses GET /api/queues/{vhost}/{name}. AMQP queue.declare-ok is never treated as membership proof.
+ /// When is unset, every inspection is
+ /// .
+ /// Deserialization uses for Native AOT.
+ ///
+ internal sealed class ArticleWorkQuorumManagementHealthProbe : IArticleWorkQuorumHealthProbe
+ {
+ /// Named client for Management calls.
+ internal const string HttpClientName = "VectorNNTPD.ArticleWork.QuorumManagement";
+
+ private readonly IHttpClientFactory _httpClientFactory;
+ private readonly IOptions _options;
+
+ ///
+ /// Initializes a new Management-backed probe.
+ ///
+ /// Creates the named Management client.
+ /// RabbitMQ options including Management base URL and credentials.
+ /// Thrown when a required dependency is null.
+ public ArticleWorkQuorumManagementHealthProbe(
+ IHttpClientFactory httpClientFactory,
+ IOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(httpClientFactory);
+ ArgumentNullException.ThrowIfNull(options);
+ _httpClientFactory = httpClientFactory;
+ _options = options;
+ }
+
+ ///
+ public async Task InspectAsync(
+ string queueName,
+ int requiredMembers,
+ CancellationToken cancellationToken)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(queueName);
+ ArgumentOutOfRangeException.ThrowIfLessThan(requiredMembers, 1);
+
+ var rabbitMq = _options.Value;
+ var management = rabbitMq.Management;
+ if (management?.BaseUrl is not { Length: > 0 } baseUrl)
+ {
+ return Unverifiable(
+ requiredMembers,
+ "RabbitMQ Management BaseUrl is not configured; quorum membership cannot be verified.");
+ }
+
+ if (string.IsNullOrWhiteSpace(rabbitMq.Username) || rabbitMq.Password is null)
+ {
+ return Unverifiable(
+ requiredMembers,
+ "RabbitMQ credentials are not available for Management authentication.");
+ }
+
+ try
+ {
+ var client = _httpClientFactory.CreateClient(HttpClientName);
+ var timeoutSeconds = management.RequestTimeoutSeconds is { } configured and > 0
+ ? configured
+ : 5;
+ using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ timeoutCts.CancelAfter(TimeSpan.FromSeconds(timeoutSeconds));
+
+ var vhost = string.IsNullOrWhiteSpace(rabbitMq.VirtualHost) ? "/" : rabbitMq.VirtualHost;
+ var requestUri = BuildQueueUri(baseUrl, vhost, queueName);
+ using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+ var credential = Convert.ToBase64String(
+ Encoding.ASCII.GetBytes(rabbitMq.Username + ":" + rabbitMq.Password));
+ request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credential);
+
+ using var response = await client
+ .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutCts.Token)
+ .ConfigureAwait(false);
+
+ if (response.StatusCode == HttpStatusCode.NotFound)
+ {
+ return new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.QueueMissing,
+ requiredMembers,
+ ObservedMembers: null,
+ OnlineMembers: null,
+ Leader: null,
+ Reason: "Management API reported the provider queue as missing.");
+ }
+
+ if (!response.IsSuccessStatusCode)
+ {
+ return Unverifiable(
+ requiredMembers,
+ $"Management API returned HTTP {(int)response.StatusCode}.");
+ }
+
+ await using var stream = await response.Content
+ .ReadAsStreamAsync(timeoutCts.Token)
+ .ConfigureAwait(false);
+ var document = await JsonSerializer
+ .DeserializeAsync(
+ stream,
+ ArticleWorkQuorumManagementJsonContext.Default.ArticleWorkQuorumManagementQueueDocument,
+ timeoutCts.Token)
+ .ConfigureAwait(false);
+
+ if (document is null)
+ {
+ return Unverifiable(requiredMembers, "Management API returned an empty queue document.");
+ }
+
+ return Evaluate(document, requiredMembers);
+ }
+ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
+ {
+ return Unverifiable(requiredMembers, "Management API request timed out.");
+ }
+ catch (OperationCanceledException)
+ {
+ throw;
+ }
+ catch (HttpRequestException ex)
+ {
+ return Unverifiable(
+ requiredMembers,
+ "Management API request failed: " + ex.GetType().Name + ".");
+ }
+ catch (JsonException ex)
+ {
+ return Unverifiable(
+ requiredMembers,
+ "Management API request failed: " + ex.GetType().Name + ".");
+ }
+ catch (InvalidOperationException ex)
+ {
+ return Unverifiable(
+ requiredMembers,
+ "Management API request failed: " + ex.GetType().Name + ".");
+ }
+ catch (IOException ex)
+ {
+ return Unverifiable(
+ requiredMembers,
+ "Management API request failed: " + ex.GetType().Name + ".");
+ }
+ }
+
+ /// Maps a Management queue document to a health result.
+ private static ArticleWorkQuorumHealthResult Evaluate(
+ ArticleWorkQuorumManagementQueueDocument document,
+ int requiredMembers)
+ {
+ var members = document.Members?.Count;
+ var online = document.Online?.Count;
+ var leader = string.IsNullOrWhiteSpace(document.Leader) ? null : document.Leader;
+
+ if (members is null || online is null)
+ {
+ return new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.Unverifiable,
+ requiredMembers,
+ members,
+ online,
+ leader,
+ "Management queue document did not expose quorum members/online arrays.");
+ }
+
+ if (leader is null)
+ {
+ return new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.Unverifiable,
+ requiredMembers,
+ members,
+ online,
+ Leader: null,
+ Reason: "Management queue document did not report a quorum leader.");
+ }
+
+ if (members.Value < requiredMembers)
+ {
+ return new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.InsufficientMembers,
+ requiredMembers,
+ members,
+ online,
+ leader,
+ "Configured quorum membership is below the required group size. Operator membership growth is required for pre-existing queues.");
+ }
+
+ if (online.Value < requiredMembers)
+ {
+ return new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.InsufficientOnlineMembers,
+ requiredMembers,
+ members,
+ online,
+ leader,
+ "Online quorum membership is below the required availability floor.");
+ }
+
+ return new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.Healthy,
+ requiredMembers,
+ members,
+ online,
+ leader,
+ "Quorum membership, online availability, and leader meet the configured policy.");
+ }
+
+ /// Builds an absolute Management queue URL without credentials.
+ internal static Uri BuildQueueUri(string baseUrl, string vhost, string queueName)
+ {
+ var root = baseUrl.TrimEnd('/');
+ var encodedVhost = Uri.EscapeDataString(vhost);
+ var encodedQueue = Uri.EscapeDataString(queueName);
+ return new Uri($"{root}/api/queues/{encodedVhost}/{encodedQueue}");
+ }
+
+ private static ArticleWorkQuorumHealthResult Unverifiable(int requiredMembers, string reason) =>
+ new(
+ ArticleWorkQuorumHealthKind.Unverifiable,
+ requiredMembers,
+ ObservedMembers: null,
+ OnlineMembers: null,
+ Leader: null,
+ Reason: reason);
+ }
+}
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementJsonContext.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementJsonContext.cs
new file mode 100644
index 00000000..b3972657
--- /dev/null
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementJsonContext.cs
@@ -0,0 +1,32 @@
+using System.Text.Json.Serialization;
+
+namespace VectorNNTPD.BackFillerService.ArticleWork
+{
+ ///
+ /// Source-generated JSON metadata for RabbitMQ Management queue documents used by quorum health.
+ ///
+ ///
+ /// Mirrors the historical case-insensitive property names used by
+ /// . Required for Native AOT publish
+ /// (IL2026/IL3050).
+ ///
+ [JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)]
+ [JsonSerializable(typeof(ArticleWorkQuorumManagementQueueDocument))]
+ [JsonSerializable(typeof(List))]
+ internal sealed partial class ArticleWorkQuorumManagementJsonContext : JsonSerializerContext;
+
+ ///
+ /// Subset of the Management GET /api/queues/{vhost}/{name} JSON used for quorum health.
+ ///
+ internal sealed class ArticleWorkQuorumManagementQueueDocument
+ {
+ /// Configured quorum member node names.
+ public List? Members { get; set; }
+
+ /// Online quorum member node names.
+ public List? Online { get; set; }
+
+ /// Current Raft leader node name.
+ public string? Leader { get; set; }
+ }
+}
diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkSettlementLease.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkSettlementLease.cs
index 65acd904..dbce8a35 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkSettlementLease.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkSettlementLease.cs
@@ -22,7 +22,7 @@ internal sealed class ArticleWorkSettlementLease
/// Original consumer channel. Settlement never moves to a replacement channel.
private readonly IRabbitMqManualAckChannel _channel;
- /// Channel-scoped delivery tag passed to Basic.Ack or Basic.Nack.
+ /// Channel-scoped delivery tag passed to Basic.Ack or Basic.Reject.
private readonly ulong _deliveryTag;
/// Connection generation the channel must still report for the settlement to proceed.
@@ -102,7 +102,9 @@ internal Task TrySettleAsync(
}
else
{
- await _channel.BasicNackAsync(_deliveryTag, disposition.Requeue, CancellationToken.None).ConfigureAwait(false);
+ // basic.reject (not basic.nack): quorum x-delivery-count must increment on RMQ 4.3+.
+ await _channel.BasicRejectAsync(_deliveryTag, disposition.Requeue, CancellationToken.None)
+ .ConfigureAwait(false);
}
return true;
diff --git a/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptions.cs b/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptions.cs
index 399ba903..aee791e3 100644
--- a/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptions.cs
+++ b/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptions.cs
@@ -198,8 +198,17 @@ internal sealed record NntpDbRuntimeOptions(
/// Maximum admitted work-request envelope size.
/// Publisher-confirm wait for response publications.
/// Optional Basic.Qos prefetch for Article Work consumers.
+ ///
+ /// Classic or quorum selection derived from configured unique RabbitMQ:Hosts count.
+ ///
+ ///
+ /// Declared x-delivery-limit on quorum queues and application requeue cap when broker delivery-count
+ /// metadata is available.
+ ///
internal sealed record BackFillerRabbitMqRuntimeOptions(
int WorkRequestMaxPayloadBytes,
int PublishConfirmTimeoutSeconds,
- ushort? ConsumerPrefetchCount);
+ ushort? ConsumerPrefetchCount,
+ VectorNNTPD.Common.Messaging.RabbitMq.ArticleWorkQueueTopology QueueTopology,
+ int DeliveryLimit);
}
diff --git a/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptionsFactory.cs b/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptionsFactory.cs
index fe7ec643..27f787db 100644
--- a/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptionsFactory.cs
+++ b/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptionsFactory.cs
@@ -57,7 +57,7 @@ internal static BackFillerRuntimeOptions Create(
CloudFlareZoneId = string.Empty,
DnsSuffix = options.DnsSuffix,
};
- return Create(options, nntpDb, acme, new RabbitMqOptions(), contentRootPath);
+ return Create(options, nntpDb, acme, CreateDefaultRabbitMqForProjection(), contentRootPath);
}
///
@@ -77,7 +77,7 @@ internal static BackFillerRuntimeOptions Create(
NntpDbOptions nntpDb,
AcmeCloudflareOptions acme,
string? contentRootPath = null) =>
- Create(options, nntpDb, acme, new RabbitMqOptions(), contentRootPath);
+ Create(options, nntpDb, acme, CreateDefaultRabbitMqForProjection(), contentRootPath);
///
/// Projects validated options, ACME settings, and RabbitMQ Article Work knobs into the immutable runtime snapshot.
@@ -186,7 +186,9 @@ internal static BackFillerRuntimeOptions Create(
RabbitMq: new BackFillerRabbitMqRuntimeOptions(
WorkRequestMaxPayloadBytes: rabbitMq.WorkRequestMaxPayloadBytes ?? 1024,
PublishConfirmTimeoutSeconds: rabbitMq.PublishConfirmTimeoutSeconds ?? 10,
- ConsumerPrefetchCount: rabbitMq.ConsumerPrefetchCount),
+ ConsumerPrefetchCount: rabbitMq.ConsumerPrefetchCount,
+ QueueTopology: ArticleWorkQueueTopology.FromConfiguredHosts(rabbitMq.Hosts),
+ DeliveryLimit: rabbitMq.ArticleWorkDeliveryLimit ?? 10),
NntpDb: new NntpDbRuntimeOptions(
nntpDb.ConnectionString.Trim(),
nntpDbBuilder.Server ?? string.Empty,
@@ -202,6 +204,17 @@ internal static BackFillerRuntimeOptions Create(
tcpKeepAlive.RetryCount));
}
+ ///
+ /// RabbitMQ projection defaults when callers omit .
+ /// Supplies a single host so Article Work topology derives classic queues for tests and partial Create overloads.
+ /// Production hosting always passes validated options after .
+ ///
+ private static RabbitMqOptions CreateDefaultRabbitMqForProjection() =>
+ new()
+ {
+ Hosts = ["127.0.0.1"],
+ };
+
/// Resolves the file-log directory, or returns empty when that directory is blank.
/// Options whose logging file target is read. A null logging section or file target uses defaults.
/// Base directory forwarded to .
diff --git a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs
index 21953de7..bc56eb27 100644
--- a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs
+++ b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs
@@ -48,7 +48,7 @@ internal static class BackFillerServiceCollectionExtensions
/// ,
/// then . Article Work consume
/// is reconciled from usable NNTP capacity. Before consumers start for a usable
- /// backbone, BackFiller declares that backbone's quorum backfiller.* topology.
+ /// backbone, BackFiller declares that backbone's Hosts-derived classic or quorum backfiller.* topology.
/// RabbitMQ is not registered into BackFiller .
///
///
@@ -281,6 +281,12 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication
provider.GetRequiredService());
builder.Services.AddSingleton(static provider =>
provider.GetRequiredService());
+ builder.Services.AddHttpClient(ArticleWorkQuorumManagementHealthProbe.HttpClientName, static client =>
+ {
+ client.Timeout = Timeout.InfiniteTimeSpan;
+ client.DefaultRequestHeaders.ExpectContinue = false;
+ });
+ builder.Services.AddSingleton();
builder.Services.AddSingleton(static provider => new ArticleWorkConsumerService(
provider.GetRequiredService(),
provider.GetRequiredService(),
@@ -288,7 +294,8 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication
provider.GetRequiredService(),
provider.GetRequiredService>(),
provider.GetRequiredService(),
- provider.GetRequiredService()));
+ provider.GetRequiredService(),
+ quorumHealth: provider.GetRequiredService()));
builder.Services.AddSingleton(static provider =>
provider.GetRequiredService());
builder.Services.AddSingleton(static provider =>
diff --git a/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerArticleWorkTopology.cs b/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerArticleWorkTopology.cs
index ac22f40f..62499bea 100644
--- a/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerArticleWorkTopology.cs
+++ b/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerArticleWorkTopology.cs
@@ -1,49 +1,71 @@
+using VectorNNTPD.BackFillerService.Configuration;
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
namespace VectorNNTPD.BackFillerService.RabbitMq
{
///
- /// Declares the durable quorum fanout ArticleWork topology for one BackFiller backbone.
+ /// Declares the durable fanout Article Work topology for one BackFiller backbone.
///
///
/// BackFiller owns per-backbone backfiller.{backbone} exchange, queue, and binding.
+ /// Queue type is derived from configured unique RabbitMQ:Hosts count: one host declares a
+ /// classic queue; two or more declare a quorum queue with initial group size equal to that count.
/// Declaration is idempotent and never deletes or mutates incompatible existing entities.
/// Topology is created when a backbone becomes usable (before Article Work consumers start)
/// and is retained when the backbone later becomes inactive.
+ /// Successful quorum declaration does not prove quorum membership; membership is verified separately.
///
internal static class BackFillerArticleWorkTopology
{
/// Broker argument that selects the RabbitMQ queue type.
internal const string QueueTypeArgumentName = "x-queue-type";
- /// Required queue type for durable BackFiller ArticleWork provider queues.
+ /// Queue type for single-host durable BackFiller Article Work provider queues.
+ internal const string ClassicQueueType = "classic";
+
+ /// Queue type for multi-host durable BackFiller Article Work provider queues.
internal const string QuorumQueueType = "quorum";
+ /// Broker argument for the initial quorum replica group size at queue creation.
+ internal const string QuorumInitialGroupSizeArgumentName = "x-quorum-initial-group-size";
+
+ /// Broker argument that caps how many times a quorum message may be delivered before dead-letter/drop.
+ internal const string DeliveryLimitArgumentName = "x-delivery-limit";
+
/// Fanout exchange type used for provider endpoints.
private const string FanoutExchangeType = "fanout";
///
- /// Declares the fanout exchange, durable quorum queue, and binding for
+ /// Declares the fanout exchange, durable provider queue, and binding for
/// using existing name/routing semantics.
///
/// Caller-owned manual-ack channel on the current connection generation.
/// Provider backbone label (for example Giganews).
+ /// Runtime snapshot supplying Hosts-derived queue topology and delivery limit.
/// Token passed to each exchange, queue, and bind call.
- /// A task that completes when the exchange, quorum queue, and binding have been declared.
- /// is .
+ /// A task that completes when the exchange, queue, and binding have been declared.
+ /// or is .
/// is null or whitespace.
///
/// Exchange name, queue name, and binding routing key are all
/// of .
/// The exchange is durable with no arguments and is not auto-delete.
- /// The queue is durable, not exclusive, and not auto-delete, with
- /// set to . This method does not delete or rewrite an existing entity.
- /// Broker failures, including cancellation, propagate from the channel.
+ /// The queue is durable, not exclusive, and not auto-delete. Arguments follow
+ /// :
+ /// classic queues set to only;
+ /// quorum queues also set and
+ /// . Classic brokers do not accept quorum-only arguments.
+ /// This method does not delete or rewrite an existing entity. Inequivalent arguments surface as
+ /// broker PRECONDITION_FAILED. Broker failures, including cancellation, propagate from the channel.
///
internal static async Task DeclareProviderEndpointAsync(
VectorNNTPD.Common.Messaging.RabbitMq.IRabbitMqManualAckChannel channel,
string backbone,
+ BackFillerRuntimeOptions runtime,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(channel);
+ ArgumentNullException.ThrowIfNull(runtime);
ArgumentException.ThrowIfNullOrWhiteSpace(backbone);
var name = BackFillerRabbitMqTopology.ComposeProviderEntity(backbone);
@@ -61,10 +83,7 @@ await channel.QueueDeclareAsync(
durable: true,
exclusive: false,
autoDelete: false,
- arguments: new Dictionary(StringComparer.Ordinal)
- {
- [QueueTypeArgumentName] = QuorumQueueType,
- },
+ arguments: BuildQueueArguments(runtime.RabbitMq),
cancellationToken)
.ConfigureAwait(false);
@@ -76,5 +95,32 @@ await channel.QueueBindAsync(
cancellationToken)
.ConfigureAwait(false);
}
+
+ ///
+ /// Builds durable provider-queue arguments from the Hosts-derived topology and delivery limit.
+ ///
+ /// Projected Article Work RabbitMQ knobs.
+ /// Argument dictionary for Queue.Declare.
+ internal static Dictionary BuildQueueArguments(BackFillerRabbitMqRuntimeOptions rabbitMq)
+ {
+ ArgumentNullException.ThrowIfNull(rabbitMq);
+
+ var topology = rabbitMq.QueueTopology;
+ if (topology.Kind == ArticleWorkQueueKind.Classic)
+ {
+ return new Dictionary(StringComparer.Ordinal)
+ {
+ [QueueTypeArgumentName] = ClassicQueueType,
+ };
+ }
+
+ return new Dictionary(StringComparer.Ordinal)
+ {
+ [QueueTypeArgumentName] = QuorumQueueType,
+ // RabbitMQ AMQP field tables accept long for these integer arguments.
+ [QuorumInitialGroupSizeArgumentName] = (long)topology.QuorumInitialGroupSize!.Value,
+ [DeliveryLimitArgumentName] = (long)rabbitMq.DeliveryLimit,
+ };
+ }
}
}
diff --git a/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerRabbitMqTopology.cs b/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerRabbitMqTopology.cs
index fe970cb7..56171831 100644
--- a/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerRabbitMqTopology.cs
+++ b/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerRabbitMqTopology.cs
@@ -4,8 +4,8 @@ namespace VectorNNTPD.BackFillerService.RabbitMq
/// Canonical backfiller.* topology names consumed by Article Work.
///
///
- /// BackFiller declares durable fanout exchanges and quorum queues for each backbone that
- /// becomes usable. This type does not declare, bind, or delete broker entities.
+ /// BackFiller declares durable fanout exchanges and Hosts-derived classic or quorum queues for each
+ /// backbone that becomes usable. This type does not declare, bind, or delete broker entities.
/// cache.requests is the NNTPD→StorageServer fleet article-presence fanout exchange
/// (not a BackFiller consume target and not a shared work queue).
/// Legacy grabbers.* names are out of scope.
diff --git a/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json b/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json
index ab25e6d3..a5c40d0e 100644
--- a/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json
+++ b/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json
@@ -30,6 +30,7 @@
"SocketTimeoutSeconds": 30,
"UnhealthyThreshold": 5,
"VirtualHost": "/",
- "WorkRequestMaxPayloadBytes": 1024
+ "WorkRequestMaxPayloadBytes": 1024,
+ "ArticleWorkDeliveryLimit": 10
}
}
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/ArticleWorkQueueTopology.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/ArticleWorkQueueTopology.cs
new file mode 100644
index 00000000..0f6b1dcf
--- /dev/null
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/ArticleWorkQueueTopology.cs
@@ -0,0 +1,101 @@
+namespace VectorNNTPD.Common.Messaging.RabbitMq
+{
+ ///
+ /// Article Work provider queue kind derived from the configured count.
+ ///
+ internal enum ArticleWorkQueueKind
+ {
+ /// Single configured host: durable classic queue without quorum arguments.
+ Classic = 0,
+
+ /// Two or more configured hosts: durable quorum queue with initial group size equal to host count.
+ Quorum = 1,
+ }
+
+ ///
+ /// Derived Article Work queue topology from the configured RabbitMQ host list.
+ ///
+ ///
+ /// Uses the configured unique host count only. Reachable hosts, live connections, and live cluster
+ /// membership never change this projection. All configured hosts must belong to the same RabbitMQ
+ /// cluster; the client host list does not establish cluster membership.
+ /// Changing from one entry to multiple (or the reverse) changes
+ /// the desired queue type and may require an explicit operator queue migration; it is not a safe
+ /// in-place conversion.
+ ///
+ /// Classic or quorum selection.
+ /// Unique trimmed host count from configuration.
+ ///
+ /// Declared x-quorum-initial-group-size and readiness membership floor when is
+ /// ; otherwise .
+ ///
+ internal readonly record struct ArticleWorkQueueTopology(
+ ArticleWorkQueueKind Kind,
+ int ConfiguredHostCount,
+ int? QuorumInitialGroupSize)
+ {
+ ///
+ /// Maximum unique entries accepted by validation.
+ /// Also the RabbitMQ quorum initial group size upper bound used by this product.
+ ///
+ internal const int MaxConfiguredHosts = 7;
+
+ ///
+ /// Whether Management-plane quorum membership must be verified before consumer readiness.
+ ///
+ internal bool RequiresQuorumHealth => Kind == ArticleWorkQueueKind.Quorum;
+
+ ///
+ /// Counts unique configured hosts (case-insensitive, trimmed) and derives classic vs quorum topology.
+ ///
+ /// Configured host list. Must contain at least one unique non-empty entry.
+ /// Derived topology.
+ /// Thrown when no unique host remains after trimming.
+ internal static ArticleWorkQueueTopology FromConfiguredHosts(IEnumerable? hosts)
+ {
+ var unique = CountUniqueHosts(hosts);
+ if (unique < 1)
+ {
+ throw new InvalidOperationException(
+ "RabbitMQ:Hosts must contain at least one unique entry to derive Article Work queue topology.");
+ }
+
+ if (unique == 1)
+ {
+ return new ArticleWorkQueueTopology(
+ ArticleWorkQueueKind.Classic,
+ ConfiguredHostCount: 1,
+ QuorumInitialGroupSize: null);
+ }
+
+ return new ArticleWorkQueueTopology(
+ ArticleWorkQueueKind.Quorum,
+ ConfiguredHostCount: unique,
+ QuorumInitialGroupSize: unique);
+ }
+
+ ///
+ /// Counts unique non-empty trimmed hosts using ordinal case-insensitive comparison.
+ ///
+ /// Configured host list. Null is treated as empty.
+ /// Unique host count.
+ ///
+ /// Filters blank entries before unique counting. Validation remains the authoritative rejection
+ /// of empty and whitespace-only lists; this helper only
+ /// projects topology from already-trimmed non-empty values.
+ ///
+ internal static int CountUniqueHosts(IEnumerable? hosts)
+ {
+ if (hosts is null)
+ {
+ return 0;
+ }
+
+ return hosts
+ .Where(static host => !string.IsNullOrWhiteSpace(host))
+ .Select(static host => host!.Trim())
+ .Distinct(StringComparer.OrdinalIgnoreCase)
+ .Count();
+ }
+ }
+}
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs
index 646689b8..ec99611b 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqManualAckChannel.cs
@@ -84,10 +84,29 @@ Task BasicConsumeAsync(
/// Token used to cancel the ACK.
Task BasicAckAsync(ulong deliveryTag, CancellationToken cancellationToken);
- /// Negatively acknowledges on this channel only.
+ ///
+ /// Negatively acknowledges on this channel only via AMQP basic.nack
+ /// with multiple=false.
+ ///
/// Channel-scoped delivery tag.
/// Whether the broker should requeue the message.
/// Token used to cancel the NACK.
+ ///
+ /// On RabbitMQ quorum queues from 4.3, basic.nack does not increment x-delivery-count.
+ /// Article Work poison-message settlement uses instead.
+ ///
Task BasicNackAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken);
+
+ ///
+ /// Rejects on this channel only via AMQP basic.reject.
+ ///
+ /// Channel-scoped delivery tag.
+ /// Whether the broker should requeue the message.
+ /// Token used to cancel the reject.
+ ///
+ /// Marks the delivery attempt as failed for quorum-queue poison-message handling so
+ /// x-delivery-count increments under RabbitMQ 4.3+ rules.
+ ///
+ Task BasicRejectAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken);
}
}
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs
index f55622e3..e692a048 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientManualAckChannel.cs
@@ -222,6 +222,13 @@ public Task BasicNackAsync(ulong deliveryTag, bool requeue, CancellationToken ca
return _channel.BasicNackAsync(deliveryTag, multiple: false, requeue, cancellationToken).AsTask();
}
+ ///
+ public Task BasicRejectAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken)
+ {
+ ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this);
+ return _channel.BasicRejectAsync(deliveryTag, requeue, cancellationToken).AsTask();
+ }
+
///
public async ValueTask DisposeAsync()
{
@@ -448,7 +455,8 @@ private async Task OnReceivedAsync(object sender, BasicDeliverEventArgs eventArg
eventArgs.RoutingKey,
eventArgs.Exchange,
eventArgs.ConsumerTag,
- Generation);
+ Generation,
+ ReadDeliveryCount(eventArgs.BasicProperties.Headers));
await handler(delivery).ConfigureAwait(false);
}
@@ -475,6 +483,32 @@ private async Task OnReceivedAsync(object sender, BasicDeliverEventArgs eventArg
};
}
+ ///
+ /// Reads the quorum-queue x-delivery-count header as a non-negative integer.
+ ///
+ /// AMQP headers. Missing or unsupported encodings yield .
+ /// The delivery count, or when absent.
+ internal static int? ReadDeliveryCount(IDictionary? headers)
+ {
+ if (headers is null
+ || !headers.TryGetValue("x-delivery-count", out var value)
+ || value is null)
+ {
+ return null;
+ }
+
+ return value switch
+ {
+ int i when i >= 0 => i,
+ long l when l >= 0 && l <= int.MaxValue => (int)l,
+ short s when s >= 0 => s,
+ byte b => b,
+ uint ui when ui <= int.MaxValue => (int)ui,
+ ulong ul when ul <= int.MaxValue => (int)ul,
+ _ => null,
+ };
+ }
+
/// Copies into a new ordinal dictionary for RabbitMQ.Client.
/// Declare or bind arguments. Not mutated.
/// when is ; otherwise a new dictionary.
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqManualAckDelivery.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqManualAckDelivery.cs
index 1d449921..6d5992e4 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqManualAckDelivery.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqManualAckDelivery.cs
@@ -14,6 +14,10 @@ namespace VectorNNTPD.Common.Messaging.RabbitMq
/// Exchange, when present.
/// Consumer tag, when present.
/// Connection generation of the consuming channel.
+ ///
+ /// Quorum-queue x-delivery-count header when present. on classic queues
+ /// or the first delivery when the broker omits the header (treated as 0 by disposition planning).
+ ///
internal readonly record struct RabbitMqManualAckDelivery(
ulong DeliveryTag,
ReadOnlyMemory Body,
@@ -25,5 +29,6 @@ internal readonly record struct RabbitMqManualAckDelivery(
string? RoutingKey,
string? Exchange,
string? ConsumerTag,
- long Generation);
+ long Generation,
+ int? DeliveryCount = null);
}
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs
index bb127a60..9a567cbd 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs
@@ -68,6 +68,15 @@ public sealed class RabbitMqOptions
///
/// RabbitMQ broker host endpoints used for connection establishment.
///
+ ///
+ /// Entries must be unique (case-insensitive). Validated unique count is 1–7.
+ /// Article Work queue type is derived from that count: one host declares classic queues;
+ /// two or more declare quorum queues with x-quorum-initial-group-size equal to the
+ /// configured unique host count. All entries must belong to the same RabbitMQ cluster;
+ /// this list is a client connection target list and does not establish cluster membership.
+ /// Changing from one host to multiple (or the reverse) changes the desired queue type and may
+ /// require an explicit operator migration; queues are never deleted or recreated automatically.
+ ///
public string[]? Hosts { get; set; } = [];
///
@@ -213,6 +222,16 @@ public sealed class RabbitMqOptions
///
public ushort? ConsumerPrefetchCount { get; set; }
+ ///
+ /// Finite delivery limit declared on Article Work quorum queues (x-delivery-limit) and used as the
+ /// application requeue cap when broker delivery-count metadata is available.
+ ///
+ ///
+ /// Validated range is 1–1000. Default 10. Declared only when Article Work uses quorum queues
+ /// ( count ≥ 2). Classic single-host queues do not support this broker argument.
+ ///
+ public int? ArticleWorkDeliveryLimit { get; set; } = 10;
+
///
/// Optional AMQP CorrelationId used to gate temporary payload diagnostics.
///
diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptionsValidator.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptionsValidator.cs
index c39ab920..6765f5ca 100644
--- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptionsValidator.cs
+++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptionsValidator.cs
@@ -167,6 +167,14 @@ internal static void ValidateConnection(RabbitMqOptions rabbitMq, List f
}
}
+ // Unique host count selects Article Work classic vs quorum topology and, for quorum,
+ // is also x-quorum-initial-group-size. Reject oversized lists rather than clamping.
+ if (normalizedHosts.Count > ArticleWorkQueueTopology.MaxConfiguredHosts)
+ {
+ failures.Add(
+ $"RabbitMQ:Hosts must contain at most {ArticleWorkQueueTopology.MaxConfiguredHosts} unique entries (Article Work quorum initial group size limit).");
+ }
+
var hasUsername = !string.IsNullOrWhiteSpace(rabbitMq.Username);
if (rabbitMq.Username is not null && string.IsNullOrWhiteSpace(rabbitMq.Username))
{
@@ -494,6 +502,15 @@ internal static void ValidateConnection(RabbitMqOptions rabbitMq, List f
failures.Add("RabbitMQ:ConsumerPrefetchCount must be between 1 and 65535.");
}
+ if (rabbitMq.ArticleWorkDeliveryLimit is null)
+ {
+ failures.Add("RabbitMQ:ArticleWorkDeliveryLimit is required.");
+ }
+ else if (rabbitMq.ArticleWorkDeliveryLimit is < 1 or > 1000)
+ {
+ failures.Add("RabbitMQ:ArticleWorkDeliveryLimit must be between 1 and 1000.");
+ }
+
if (rabbitMq.RequestedChannelMax is null)
{
failures.Add("RabbitMQ:RequestedChannelMax is required.");
diff --git a/src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json b/src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json
index ab25e6d3..a5c40d0e 100644
--- a/src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json
+++ b/src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json
@@ -30,6 +30,7 @@
"SocketTimeoutSeconds": 30,
"UnhealthyThreshold": 5,
"VirtualHost": "/",
- "WorkRequestMaxPayloadBytes": 1024
+ "WorkRequestMaxPayloadBytes": 1024,
+ "ArticleWorkDeliveryLimit": 10
}
}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkConsumerLifecycleTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkConsumerLifecycleTests.cs
index 54a125a6..353d2d6a 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkConsumerLifecycleTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkConsumerLifecycleTests.cs
@@ -179,7 +179,7 @@ public async Task Consumer_service_starts_one_session_per_desired_nntp_slot_when
Assert.True(Assert.Single(topologyChannel.QueueDeclarations).Arguments!.TryGetValue(
BackFillerArticleWorkTopology.QueueTypeArgumentName,
out var queueType));
- Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, queueType);
+ Assert.Equal(BackFillerArticleWorkTopology.ClassicQueueType, queueType);
Assert.Equal(1, topologyChannel.DisposeCount);
Assert.Equal(0, topologyChannel.ConsumeCount);
Assert.Equal(1, factory.ConnectCount);
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs
index bd61b600..679d3e66 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkOversizedSecurityTests.cs
@@ -131,6 +131,9 @@ public event EventHandler? ChannelFaulted
///
public Task BasicNackAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken) => Task.CompletedTask;
+
+ ///
+ public Task BasicRejectAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken) => Task.CompletedTask;
}
}
}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkProviderTopologyTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkProviderTopologyTests.cs
index 544851b5..196aafe3 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkProviderTopologyTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkProviderTopologyTests.cs
@@ -16,10 +16,10 @@ namespace VectorNNTPD.BackFillerService.Tests.ArticleWork
public sealed class ArticleWorkProviderTopologyTests
{
///
- /// Usable capacity starts consumers only after a dedicated topology channel declares exchange, durable quorum queue, and bind, then disposes.
+ /// Usable capacity starts consumers only after a dedicated topology channel declares exchange, durable classic queue, and bind, then disposes.
///
[Fact]
- public async Task Active_backbone_declares_quorum_fanout_topology_before_consumers()
+ public async Task Active_backbone_declares_classic_fanout_topology_before_consumers_for_single_host()
{
var factory = new FakeBackFillerRabbitMqConnectionFactory();
var connections = RabbitMqServiceTests.CreateService(factory);
@@ -43,7 +43,9 @@ public async Task Active_backbone_declares_quorum_fanout_topology_before_consume
Assert.False(queue.AutoDelete);
Assert.NotNull(queue.Arguments);
Assert.True(queue.Arguments.TryGetValue(BackFillerArticleWorkTopology.QueueTypeArgumentName, out var queueType));
- Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, queueType);
+ Assert.Equal(BackFillerArticleWorkTopology.ClassicQueueType, queueType);
+ Assert.False(queue.Arguments.ContainsKey(BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName));
+ Assert.False(queue.Arguments.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName));
Assert.False(queue.Arguments.ContainsKey("x-message-ttl"));
Assert.Single(queue.Arguments);
Assert.Equal(
@@ -57,6 +59,35 @@ public async Task Active_backbone_declares_quorum_fanout_topology_before_consume
await connections.DisposeAsync();
}
+ ///
+ /// Multi-host configuration declares quorum queue arguments with initial group size equal to host count.
+ ///
+ [Fact]
+ public async Task Active_backbone_declares_quorum_fanout_topology_for_multi_host()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Giganews", 1)]),
+ CreateCapacity([("Giganews", 1)]),
+ hostCount: 3);
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ Assert.Single(consumer.Sessions);
+ var topology = factory.LastConnection!.Channels[0];
+ var queue = Assert.Single(topology.QueueDeclarations);
+ Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, queue.Arguments![BackFillerArticleWorkTopology.QueueTypeArgumentName]);
+ Assert.Equal(3L, queue.Arguments[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]);
+ Assert.Equal(10L, queue.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]);
+ Assert.Equal(3, queue.Arguments.Count);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
///
/// Queue declare precondition failure leaves zero running sessions and no BasicConsume on any channel.
///
@@ -193,7 +224,7 @@ await ArticleWorkTestDeliveries.WaitUntilAsync(
Assert.True(queue.Arguments.TryGetValue(
BackFillerArticleWorkTopology.QueueTypeArgumentName,
out var queueType));
- Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, queueType);
+ Assert.Equal(BackFillerArticleWorkTopology.ClassicQueueType, queueType);
});
await consumer.DisposeAsync();
@@ -231,24 +262,31 @@ public async Task Only_usable_backbones_are_declared()
}
///
- /// Repeated on one channel records a single declare triple with quorum queue arguments.
+ /// Repeated on one channel records a single declare triple with classic queue arguments.
///
[Fact]
public async Task DeclareProviderEndpoint_is_idempotent_on_same_channel()
{
var channel = new FakeBackFillerRabbitMqChannel(1);
- await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync(channel, "Giganews", CancellationToken.None);
+ var runtime = RabbitMqServiceTests.CreateFastRuntime();
+ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync(
+ channel,
+ "Giganews",
+ runtime,
+ CancellationToken.None);
Assert.Single(channel.ExchangeDeclarations);
Assert.Single(channel.QueueDeclarations);
Assert.Single(channel.BindingDeclarations);
Assert.All(channel.QueueDeclarations, static q => Assert.True(q.Durable));
- Assert.All(channel.QueueDeclarations, static q =>
+ Assert.All(channel.QueueDeclarations, q =>
{
Assert.NotNull(q.Arguments);
Assert.True(q.Arguments.TryGetValue(BackFillerArticleWorkTopology.QueueTypeArgumentName, out var type));
- Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, type);
+ Assert.Equal(BackFillerArticleWorkTopology.ClassicQueueType, type);
+ Assert.False(q.Arguments.ContainsKey(BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName));
+ Assert.False(q.Arguments.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName));
});
}
@@ -258,14 +296,16 @@ public async Task DeclareProviderEndpoint_is_idempotent_on_same_channel()
/// Started RabbitMQ service.
/// Provider definitions driving queue names.
/// Usable capacity state consumed at start.
+ /// Unique configured host count selecting classic (1) or quorum (≥ 2).
/// New .
private static ArticleWorkConsumerService CreateConsumer(
IRabbitMqService connections,
IBackFillerProviderCatalog catalog,
- BackboneUsableCapacityState capacity) =>
+ BackboneUsableCapacityState capacity,
+ int hostCount = 1) =>
new(
connections,
- RabbitMqServiceTests.CreateFastRuntime(),
+ RabbitMqServiceTests.CreateFastRuntime(hostCount),
new DeferredArticleWorkHandler(),
new RecordingArticleWorkResponsePublisher(),
NullLogger.Instance,
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs
new file mode 100644
index 00000000..6a692110
--- /dev/null
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs
@@ -0,0 +1,904 @@
+using System.Net;
+using System.Text;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using VectorNNTPD.BackFillerService.ArticleWork;
+using VectorNNTPD.BackFillerService.Configuration;
+using VectorNNTPD.BackFillerService.Nntp;
+using VectorNNTPD.BackFillerService.RabbitMq;
+using VectorNNTPD.BackFillerService.Tests.Fixtures;
+using VectorNNTPD.BackFillerService.Tests.RabbitMq;
+using VectorNNTPD.BackFillerService.Tests.TestDoubles;
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
+namespace VectorNNTPD.BackFillerService.Tests.ArticleWork
+{
+ ///
+ /// Quorum membership readiness and finite delivery-limit settlement for Article Work (#195).
+ ///
+ public sealed class ArticleWorkQuorumHealthAndDeliveryLimitTests
+ {
+ private const int QuorumMembershipNotReadyEventId = 5325;
+ private const int QuorumMembershipHealthyEventId = 5326;
+
+ ///
+ /// One configured host declares classic topology without quorum or delivery-limit arguments.
+ ///
+ [Fact]
+ public async Task One_host_declares_classic_topology()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var runtime = CreateRuntime(hostCount: 1, deliveryLimit: 7);
+
+ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync(
+ channel,
+ "Giganews",
+ runtime,
+ CancellationToken.None);
+
+ var declaration = Assert.Single(channel.QueueDeclarations);
+ Assert.Equal(
+ BackFillerArticleWorkTopology.ClassicQueueType,
+ declaration.Arguments![BackFillerArticleWorkTopology.QueueTypeArgumentName]);
+ Assert.False(declaration.Arguments.ContainsKey(BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName));
+ Assert.False(declaration.Arguments.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName));
+ Assert.Equal(ArticleWorkQueueKind.Classic, runtime.RabbitMq.QueueTopology.Kind);
+ Assert.Equal(1, runtime.RabbitMq.QueueTopology.ConfiguredHostCount);
+ Assert.Null(runtime.RabbitMq.QueueTopology.QuorumInitialGroupSize);
+ Assert.False(runtime.RabbitMq.QueueTopology.RequiresQuorumHealth);
+ }
+
+ ///
+ /// Two configured hosts declare quorum topology with initial group size 2 and delivery limit.
+ ///
+ [Fact]
+ public async Task Two_hosts_declare_quorum_topology_with_group_size_2()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var runtime = CreateRuntime(hostCount: 2, deliveryLimit: 7);
+
+ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync(
+ channel,
+ "Giganews",
+ runtime,
+ CancellationToken.None);
+
+ var declaration = Assert.Single(channel.QueueDeclarations);
+ Assert.Equal(
+ BackFillerArticleWorkTopology.QuorumQueueType,
+ declaration.Arguments![BackFillerArticleWorkTopology.QueueTypeArgumentName]);
+ Assert.Equal(2L, declaration.Arguments[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]);
+ Assert.Equal(7L, declaration.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]);
+ Assert.Equal(2, runtime.RabbitMq.QueueTopology.QuorumInitialGroupSize);
+ }
+
+ ///
+ /// Three configured hosts declare quorum topology with initial group size 3.
+ ///
+ [Fact]
+ public async Task Three_hosts_declare_quorum_topology_with_group_size_3()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var runtime = CreateRuntime(hostCount: 3, deliveryLimit: 10);
+
+ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync(
+ channel,
+ "Giganews",
+ runtime,
+ CancellationToken.None);
+
+ var declaration = Assert.Single(channel.QueueDeclarations);
+ Assert.Equal(3L, declaration.Arguments![BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]);
+ Assert.Equal(3, runtime.RabbitMq.QueueTopology.ConfiguredHostCount);
+ }
+
+ ///
+ /// A larger supported host count maps to the same initial group size without clamping.
+ ///
+ [Fact]
+ public async Task Seven_hosts_declare_quorum_topology_with_group_size_7()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var runtime = CreateRuntime(hostCount: 7, deliveryLimit: 10);
+
+ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync(
+ channel,
+ "Giganews",
+ runtime,
+ CancellationToken.None);
+
+ var declaration = Assert.Single(channel.QueueDeclarations);
+ Assert.Equal(7L, declaration.Arguments![BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]);
+ Assert.Equal(7, runtime.RabbitMq.QueueTopology.QuorumInitialGroupSize);
+ }
+
+ ///
+ /// Configured host count (not live reachability) remains the topology source when a host is unavailable.
+ ///
+ [Fact]
+ public void Unavailable_configured_host_does_not_change_queue_type_or_group_size()
+ {
+ var rabbit = BackFillerTestOptions.CreateValidRabbitMqWithHostCount(3);
+ // Simulate an outage: configuration still lists three hosts; topology must not shrink.
+ var topology = ArticleWorkQueueTopology.FromConfiguredHosts(rabbit.Hosts);
+ Assert.Equal(ArticleWorkQueueKind.Quorum, topology.Kind);
+ Assert.Equal(3, topology.ConfiguredHostCount);
+ Assert.Equal(3, topology.QuorumInitialGroupSize);
+ Assert.True(topology.RequiresQuorumHealth);
+ }
+
+ ///
+ /// First delivery (no x-delivery-count) may requeue within the configured limit.
+ ///
+ [Fact]
+ public async Task First_delivery_without_count_requeues_on_retryable_failure()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure },
+ new RecordingArticleWorkResponsePublisher(),
+ maxPayloadBytes: 1024,
+ deliveryLimit: 3);
+
+ var outcome = await pipeline.ProcessAsync(
+ ArticleWorkTestDeliveries.Canonical(),
+ "Giganews",
+ channel,
+ static () => true,
+ CancellationToken.None);
+
+ Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.True(settlement.Requeue);
+ }
+
+ ///
+ /// The final permitted retry (count == limit - 1) still requeues.
+ ///
+ [Fact]
+ public async Task Final_permitted_retry_still_requeues()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure },
+ new RecordingArticleWorkResponsePublisher(),
+ maxPayloadBytes: 1024,
+ deliveryLimit: 5);
+
+ var outcome = await pipeline.ProcessAsync(
+ ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson, deliveryCount: 4),
+ "Giganews",
+ channel,
+ static () => true,
+ CancellationToken.None);
+
+ Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.True(settlement.Requeue);
+ }
+
+ ///
+ /// Single-host classic readiness does not call the quorum Management probe.
+ ///
+ [Fact]
+ public async Task Single_host_readiness_does_not_call_quorum_management_probe()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+
+ var probe = new ControllableArticleWorkQuorumHealthProbe
+ {
+ Next = Deficient(required: 3, observed: 1, online: 1),
+ };
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Giganews", 1)]),
+ CreateCapacity([("Giganews", 1)]),
+ quorumHealth: probe,
+ hostCount: 1);
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ var session = Assert.Single(consumer.Sessions);
+ Assert.Equal(ArticleWorkConsumerState.Running, session.State);
+ Assert.Equal(0, probe.InspectCount);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Successful declaration with insufficient membership leaves the backbone unready.
+ ///
+ [Fact]
+ public async Task Insufficient_membership_does_not_start_consumer()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+
+ var probe = new ControllableArticleWorkQuorumHealthProbe
+ {
+ Next = Deficient(required: 3, observed: 1, online: 1),
+ };
+ var logger = new CollectingLogger();
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Giganews", 1)]),
+ CreateCapacity([("Giganews", 1)]),
+ logger,
+ probe,
+ hostCount: 3);
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ Assert.Empty(consumer.Sessions);
+ Assert.Equal(1, probe.InspectCount);
+ Assert.Contains(
+ logger.Entries,
+ static entry => entry.EventId.Id == QuorumMembershipNotReadyEventId
+ && entry.Message.Contains("InsufficientMembers", StringComparison.Ordinal));
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Sufficient configured and online membership allows readiness after declaration.
+ ///
+ [Fact]
+ public async Task Healthy_membership_allows_consumer_start()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+
+ var probe = new ControllableArticleWorkQuorumHealthProbe
+ {
+ Next = Healthy(required: 3, observed: 3, online: 3),
+ };
+ var logger = new CollectingLogger();
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Giganews", 1)]),
+ CreateCapacity([("Giganews", 1)]),
+ logger,
+ probe,
+ hostCount: 3);
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ var session = Assert.Single(consumer.Sessions);
+ Assert.Equal(ArticleWorkConsumerState.Running, session.State);
+ Assert.Contains(
+ logger.Entries,
+ static entry => entry.EventId.Id == QuorumMembershipHealthyEventId);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Unverifiable membership keeps the backbone unready and does not claim false readiness.
+ ///
+ [Fact]
+ public async Task Unverifiable_membership_keeps_backbone_unready()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+
+ var probe = new ControllableArticleWorkQuorumHealthProbe
+ {
+ Next = new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.Unverifiable,
+ RequiredMembers: 3,
+ ObservedMembers: null,
+ OnlineMembers: null,
+ Leader: null,
+ Reason: "Management API request timed out."),
+ };
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Giganews", 1)]),
+ CreateCapacity([("Giganews", 1)]),
+ quorumHealth: probe,
+ hostCount: 3);
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ Assert.Empty(consumer.Sessions);
+ Assert.Equal(1, probe.InspectCount);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// A later healthy membership observation makes the backbone ready through reconciliation.
+ ///
+ [Fact]
+ public async Task Later_healthy_membership_allows_readiness_via_reconcile()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+
+ var probe = new ControllableArticleWorkQuorumHealthProbe
+ {
+ Next = Deficient(required: 3, observed: 1, online: 1),
+ };
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Giganews", 1)]),
+ CreateCapacity([("Giganews", 1)]),
+ quorumHealth: probe,
+ hostCount: 3);
+
+ await consumer.StartAsync(CancellationToken.None);
+ Assert.Empty(consumer.Sessions);
+
+ probe.Next = Healthy(required: 3, observed: 3, online: 3);
+ await consumer.ReconcileAsync(CancellationToken.None);
+
+ var session = Assert.Single(consumer.Sessions);
+ Assert.Equal(ArticleWorkConsumerState.Running, session.State);
+ Assert.True(probe.InspectCount >= 2);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// A healthy running consumer is kept when membership remains healthy on reconcile.
+ ///
+ [Fact]
+ public async Task Healthy_consumer_is_not_recreated_when_membership_stays_healthy()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+
+ var probe = new ControllableArticleWorkQuorumHealthProbe
+ {
+ Next = Healthy(required: 3, observed: 3, online: 3),
+ };
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Giganews", 1)]),
+ CreateCapacity([("Giganews", 1)]),
+ quorumHealth: probe,
+ hostCount: 3);
+
+ await consumer.StartAsync(CancellationToken.None);
+ var original = Assert.Single(consumer.Sessions);
+
+ await consumer.ReconcileAsync(CancellationToken.None);
+
+ var after = Assert.Single(consumer.Sessions);
+ Assert.Same(original, after);
+ Assert.Equal(ArticleWorkConsumerState.Running, after.State);
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Within the delivery limit, retryable failures still requeue.
+ ///
+ [Fact]
+ public async Task Retryable_failure_requeues_within_delivery_limit()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure },
+ new RecordingArticleWorkResponsePublisher(),
+ maxPayloadBytes: 1024,
+ deliveryLimit: 10);
+
+ var outcome = await pipeline.ProcessAsync(
+ ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson, deliveryCount: 3),
+ "Giganews",
+ channel,
+ static () => true,
+ CancellationToken.None);
+
+ Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.True(settlement.Requeue);
+ }
+
+ ///
+ /// When delivery count reaches the configured limit, work is NACKed without requeue.
+ ///
+ [Fact]
+ public async Task Exhausted_delivery_limit_nacks_without_requeue()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure },
+ new RecordingArticleWorkResponsePublisher(),
+ maxPayloadBytes: 1024,
+ deliveryLimit: 5);
+
+ var outcome = await pipeline.ProcessAsync(
+ ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson, deliveryCount: 5),
+ "Giganews",
+ channel,
+ static () => true,
+ CancellationToken.None);
+
+ Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.False(settlement.Requeue);
+ }
+
+ ///
+ /// Cancellation that has exhausted the delivery limit does not bypass the finite policy.
+ ///
+ [Fact]
+ public async Task Cancelled_delivery_at_limit_does_not_requeue()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Success },
+ new RecordingArticleWorkResponsePublisher(),
+ maxPayloadBytes: 1024,
+ deliveryLimit: 4);
+
+ using var cts = new CancellationTokenSource();
+ await cts.CancelAsync();
+ var outcome = await pipeline.ProcessAsync(
+ ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson, deliveryCount: 4),
+ "Giganews",
+ channel,
+ static () => true,
+ cts.Token);
+
+ Assert.Equal(ArticleWorkOutcome.Cancelled, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.False(settlement.Requeue);
+ }
+
+ ///
+ /// Successful processing still ACKs when delivery count is high.
+ ///
+ [Fact]
+ public async Task Successful_processing_still_acks_at_high_delivery_count()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var publisher = new RecordingArticleWorkResponsePublisher { CompletesSuccessPublication = true };
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Success },
+ publisher,
+ maxPayloadBytes: 1024,
+ deliveryLimit: 3);
+
+ var outcome = await pipeline.ProcessAsync(
+ ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson, deliveryCount: 3),
+ "Giganews",
+ channel,
+ static () => true,
+ CancellationToken.None);
+
+ Assert.Equal(ArticleWorkOutcome.Success, outcome);
+ Assert.Single(publisher.Published);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.True(settlement.Acknowledge);
+ Assert.False(settlement.Requeue);
+ }
+
+ ///
+ /// Publisher failure at the limit settles without requeue so confirm loss cannot bypass the cap.
+ ///
+ [Fact]
+ public async Task Publisher_failure_at_limit_does_not_requeue()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var publisher = new RecordingArticleWorkResponsePublisher
+ {
+ CompletesSuccessPublication = true,
+ PublishException = new InvalidOperationException("confirm failed"),
+ };
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Success },
+ publisher,
+ maxPayloadBytes: 1024,
+ deliveryLimit: 2);
+
+ var outcome = await pipeline.ProcessAsync(
+ ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson, deliveryCount: 2),
+ "Giganews",
+ channel,
+ static () => true,
+ CancellationToken.None);
+
+ Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.False(settlement.Requeue);
+ }
+
+ ///
+ /// On quorum topology, a redelivery missing x-delivery-count is treated as exhausted so missing metadata cannot disable the cap.
+ ///
+ [Fact]
+ public async Task Quorum_redelivery_without_delivery_count_does_not_requeue()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure },
+ new RecordingArticleWorkResponsePublisher(),
+ maxPayloadBytes: 1024,
+ deliveryLimit: 5,
+ treatMissingRedeliveryCountAsExhausted: true);
+
+ var delivery = ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson) with
+ {
+ Redelivered = true,
+ DeliveryCount = null,
+ };
+
+ var outcome = await pipeline.ProcessAsync(
+ delivery,
+ "Giganews",
+ channel,
+ static () => true,
+ CancellationToken.None);
+
+ Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.False(settlement.Requeue);
+ }
+
+ ///
+ /// Classic queues lack x-delivery-count; missing count on redelivery must not force discard.
+ ///
+ [Fact]
+ public async Task Classic_redelivery_without_delivery_count_still_requeues()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure },
+ new RecordingArticleWorkResponsePublisher(),
+ maxPayloadBytes: 1024,
+ deliveryLimit: 5,
+ treatMissingRedeliveryCountAsExhausted: false);
+
+ var delivery = ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson) with
+ {
+ Redelivered = true,
+ DeliveryCount = null,
+ };
+
+ var outcome = await pipeline.ProcessAsync(
+ delivery,
+ "Giganews",
+ channel,
+ static () => true,
+ CancellationToken.None);
+
+ Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.True(settlement.Requeue);
+ }
+
+ ///
+ /// Cancellation below the limit still requeues.
+ ///
+ [Fact]
+ public async Task Cancelled_delivery_below_limit_still_requeues()
+ {
+ var channel = new FakeBackFillerRabbitMqChannel(1);
+ var pipeline = new ArticleWorkDeliveryPipeline(
+ new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Success },
+ new RecordingArticleWorkResponsePublisher(),
+ maxPayloadBytes: 1024,
+ deliveryLimit: 4);
+
+ using var cts = new CancellationTokenSource();
+ await cts.CancelAsync();
+ var outcome = await pipeline.ProcessAsync(
+ ArticleWorkTestDeliveries.Create(ArticleWorkTestDeliveries.CanonicalRequestJson, deliveryCount: 2),
+ "Giganews",
+ channel,
+ static () => true,
+ cts.Token);
+
+ Assert.Equal(ArticleWorkOutcome.Cancelled, outcome);
+ var settlement = Assert.Single(channel.Settlements);
+ Assert.False(settlement.Acknowledge);
+ Assert.True(settlement.Requeue);
+ }
+
+ ///
+ /// Insufficient online members is distinct from unverifiable health and blocks readiness.
+ ///
+ [Fact]
+ public async Task Insufficient_online_members_blocks_readiness()
+ {
+ var factory = new FakeBackFillerRabbitMqConnectionFactory();
+ var connections = RabbitMqServiceTests.CreateService(factory);
+ await connections.StartAsync(CancellationToken.None);
+
+ var probe = new ControllableArticleWorkQuorumHealthProbe
+ {
+ Next = new ArticleWorkQuorumHealthResult(
+ ArticleWorkQuorumHealthKind.InsufficientOnlineMembers,
+ RequiredMembers: 3,
+ ObservedMembers: 3,
+ OnlineMembers: 1,
+ Leader: "rabbit@node-a",
+ Reason: "Online quorum membership is below the required availability floor."),
+ };
+ var logger = new CollectingLogger();
+ var consumer = CreateConsumer(
+ connections,
+ CreateCatalog([("Giganews", 1)]),
+ CreateCapacity([("Giganews", 1)]),
+ logger,
+ probe,
+ hostCount: 3);
+
+ await consumer.StartAsync(CancellationToken.None);
+
+ Assert.Empty(consumer.Sessions);
+ Assert.Contains(
+ logger.Entries,
+ static entry => entry.EventId.Id == QuorumMembershipNotReadyEventId
+ && entry.Message.Contains("InsufficientOnlineMembers", StringComparison.Ordinal));
+
+ await consumer.DisposeAsync();
+ await connections.DisposeAsync();
+ }
+
+ ///
+ /// Effective count resolution treats first delivery and redelivery-without-count distinctly.
+ ///
+ [Fact]
+ public void Effective_delivery_count_is_conservative_for_redelivery_without_header()
+ {
+ Assert.Equal(0, ArticleWorkDispositionPlanner.ResolveEffectiveDeliveryCount(null, redelivered: false, deliveryLimit: 10));
+ Assert.Equal(10, ArticleWorkDispositionPlanner.ResolveEffectiveDeliveryCount(null, redelivered: true, deliveryLimit: 10));
+ Assert.Equal(4, ArticleWorkDispositionPlanner.ResolveEffectiveDeliveryCount(4, redelivered: true, deliveryLimit: 10));
+ }
+
+ ///
+ /// Management URI builder encodes vhost and queue without credentials.
+ ///
+ [Fact]
+ public void Management_queue_uri_encodes_vhost_and_queue()
+ {
+ var uri = ArticleWorkQuorumManagementHealthProbe.BuildQueueUri(
+ "http://198.18.0.3:15672",
+ "/",
+ "backfiller.Giganews");
+
+ Assert.Equal(
+ "http://198.18.0.3:15672/api/queues/%2F/backfiller.Giganews",
+ uri.AbsoluteUri);
+ }
+
+ ///
+ /// Management probe maps HTTP auth failure to Unverifiable, not insufficient membership.
+ ///
+ [Fact]
+ public async Task Management_probe_auth_failure_is_unverifiable()
+ {
+ var handler = new StubHttpMessageHandler(HttpStatusCode.Unauthorized, "{}");
+ var (probe, lifetime) = CreateManagementProbe(handler, baseUrl: "http://127.0.0.1:15672");
+ using (lifetime)
+ {
+ var result = await probe.InspectAsync("backfiller.Giganews", requiredMembers: 3, CancellationToken.None);
+
+ Assert.Equal(ArticleWorkQuorumHealthKind.Unverifiable, result.Kind);
+ Assert.Contains("401", result.Reason, StringComparison.Ordinal);
+ Assert.Null(result.ObservedMembers);
+ }
+ }
+
+ ///
+ /// Management probe maps a healthy queue document to Healthy when membership policy is met.
+ ///
+ [Fact]
+ public async Task Management_probe_parses_healthy_membership_document()
+ {
+ var body = """{"members":["a","b","c"],"online":["a","b","c"],"leader":"a"}""";
+ var handler = new StubHttpMessageHandler(HttpStatusCode.OK, body);
+ var (probe, lifetime) = CreateManagementProbe(handler, baseUrl: "http://127.0.0.1:15672");
+ using (lifetime)
+ {
+ var result = await probe.InspectAsync("backfiller.Giganews", requiredMembers: 3, CancellationToken.None);
+
+ Assert.Equal(ArticleWorkQuorumHealthKind.Healthy, result.Kind);
+ Assert.Equal(3, result.ObservedMembers);
+ Assert.Equal(3, result.OnlineMembers);
+ Assert.Equal("a", result.Leader);
+ }
+ }
+
+ ///
+ /// Missing leader in an otherwise complete document is Unverifiable.
+ ///
+ [Fact]
+ public async Task Management_probe_missing_leader_is_unverifiable()
+ {
+ var body = """{"members":["a","b","c"],"online":["a","b","c"]}""";
+ var handler = new StubHttpMessageHandler(HttpStatusCode.OK, body);
+ var (probe, lifetime) = CreateManagementProbe(handler, baseUrl: "http://127.0.0.1:15672");
+ using (lifetime)
+ {
+ var result = await probe.InspectAsync("backfiller.Giganews", requiredMembers: 3, CancellationToken.None);
+
+ Assert.Equal(ArticleWorkQuorumHealthKind.Unverifiable, result.Kind);
+ Assert.Contains("leader", result.Reason, StringComparison.OrdinalIgnoreCase);
+ }
+ }
+
+ private static ArticleWorkQuorumHealthResult Healthy(int required, int observed, int online) =>
+ new(
+ ArticleWorkQuorumHealthKind.Healthy,
+ required,
+ observed,
+ online,
+ Leader: "rabbit@node-a",
+ Reason: "Quorum membership and online availability meet the configured policy.");
+
+ private static ArticleWorkQuorumHealthResult Deficient(int required, int observed, int online) =>
+ new(
+ ArticleWorkQuorumHealthKind.InsufficientMembers,
+ required,
+ observed,
+ online,
+ Leader: "rabbit@node-a",
+ Reason: "Configured quorum membership is below the required group size. Operator membership growth is required for pre-existing queues.");
+
+ private static BackFillerRuntimeOptions CreateRuntime(int hostCount, int deliveryLimit)
+ {
+ var rabbit = BackFillerTestOptions.CreateValidRabbitMqWithHostCount(hostCount);
+ rabbit.ArticleWorkDeliveryLimit = deliveryLimit;
+ return BackFillerRuntimeOptionsFactory.Create(
+ BackFillerTestOptions.CreateValid(),
+ BackFillerTestOptions.CreateValidNntpDb(),
+ BackFillerTestOptions.CreateValidAcme(),
+ rabbit);
+ }
+
+ private static ArticleWorkConsumerService CreateConsumer(
+ IRabbitMqService connections,
+ IBackFillerProviderCatalog catalog,
+ BackboneUsableCapacityState capacity,
+ ILogger? logger = null,
+ IArticleWorkQuorumHealthProbe? quorumHealth = null,
+ int hostCount = 3) =>
+ new(
+ connections,
+ RabbitMqServiceTests.CreateFastRuntime(hostCount),
+ new DeferredArticleWorkHandler(),
+ new RecordingArticleWorkResponsePublisher(),
+ logger ?? NullLogger.Instance,
+ catalog,
+ capacity,
+ quorumHealth: quorumHealth);
+
+ private static StaticBackFillerProviderCatalog CreateCatalog(
+ IReadOnlyList<(string Backbone, int MaxSessions)> providers) =>
+ new(
+ [
+ ..providers.Select(static p => new BackFillerProviderDefinition(
+ p.Backbone,
+ "127.0.0.1",
+ 119,
+ false,
+ "nntp-user",
+ "p",
+ 0,
+ p.MaxSessions)),
+ ]);
+
+ private static BackboneUsableCapacityState CreateCapacity(
+ IReadOnlyList<(string Backbone, int Active)> capacities)
+ {
+ var state = new BackboneUsableCapacityState();
+ var snapshot = capacities
+ .Where(static pair => pair.Active > 0)
+ .ToDictionary(
+ static pair => pair.Backbone,
+ static pair => pair.Active,
+ StringComparer.OrdinalIgnoreCase);
+ if (snapshot.Count > 0)
+ {
+ state.PublishSnapshot(snapshot);
+ }
+
+ return state;
+ }
+
+ private static (ArticleWorkQuorumManagementHealthProbe Probe, IDisposable Lifetime) CreateManagementProbe(
+ HttpMessageHandler handler,
+ string baseUrl)
+ {
+ var options = Options.Create(new RabbitMqOptions
+ {
+ Hosts = ["127.0.0.1"],
+ Username = "guest",
+ Password = "guest",
+ EnableSsl = false,
+ VirtualHost = "/",
+ Management = new RabbitMqManagementOptions
+ {
+ BaseUrl = baseUrl,
+ RequestTimeoutSeconds = 5,
+ },
+ });
+ var factory = new SingleHandlerHttpClientFactory(handler);
+ return (new ArticleWorkQuorumManagementHealthProbe(factory, options), factory);
+ }
+
+ /// Controllable Management stand-in for membership readiness tests.
+ private sealed class ControllableArticleWorkQuorumHealthProbe : IArticleWorkQuorumHealthProbe
+ {
+ internal ArticleWorkQuorumHealthResult Next { get; set; }
+
+ internal int InspectCount;
+
+ public Task InspectAsync(
+ string queueName,
+ int requiredMembers,
+ CancellationToken cancellationToken)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(queueName);
+ Interlocked.Increment(ref InspectCount);
+ return Task.FromResult(Next);
+ }
+ }
+
+ private sealed class SingleHandlerHttpClientFactory : IHttpClientFactory, IDisposable
+ {
+ private readonly HttpClient _client;
+
+ public SingleHandlerHttpClientFactory(HttpMessageHandler handler) =>
+ _client = new HttpClient(handler, disposeHandler: true);
+
+ public HttpClient CreateClient(string name) => _client;
+
+ public void Dispose() => _client.Dispose();
+ }
+
+ private sealed class StubHttpMessageHandler : HttpMessageHandler
+ {
+ private readonly HttpStatusCode _statusCode;
+ private readonly byte[] _bodyBytes;
+
+ public StubHttpMessageHandler(HttpStatusCode statusCode, string body)
+ {
+ _statusCode = statusCode;
+ _bodyBytes = Encoding.UTF8.GetBytes(body);
+ }
+
+ protected override async Task SendAsync(
+ HttpRequestMessage request,
+ CancellationToken cancellationToken)
+ {
+ await Task.CompletedTask.ConfigureAwait(false);
+ // Returned to HttpClient; InspectAsync disposes the response with using.
+ return new HttpResponseMessage(_statusCode)
+ {
+ Content = new ByteArrayContent(_bodyBytes)
+ {
+ Headers = { ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json") },
+ },
+ };
+ }
+ }
+ }
+}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkSettlementTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkSettlementTests.cs
index b35f2867..e0effb8a 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkSettlementTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkSettlementTests.cs
@@ -34,7 +34,13 @@ public void Planner_maps_terminal_and_retryable_outcomes(
bool publish)
{
var outcome = Enum.Parse(outcomeName);
- var disposition = ArticleWorkDispositionPlanner.Create(outcome, replyable: true, cancellationRequested: false);
+ var disposition = ArticleWorkDispositionPlanner.Create(
+ outcome,
+ replyable: true,
+ cancellationRequested: false,
+ deliveryCount: null,
+ redelivered: false,
+ deliveryLimit: 10);
Assert.Equal(acknowledge, disposition.Acknowledge);
Assert.Equal(requeue, disposition.Requeue);
Assert.Equal(publish, disposition.PublishResponse);
@@ -49,7 +55,10 @@ public void InvalidRequest_without_reply_coordinates_does_not_publish()
var disposition = ArticleWorkDispositionPlanner.Create(
ArticleWorkOutcome.InvalidRequest,
replyable: false,
- cancellationRequested: false);
+ cancellationRequested: false,
+ deliveryCount: null,
+ redelivered: false,
+ deliveryLimit: 10);
Assert.True(disposition.Acknowledge);
Assert.False(disposition.Requeue);
@@ -65,13 +74,54 @@ public void Cancellation_overrides_a_terminal_outcome()
var disposition = ArticleWorkDispositionPlanner.Create(
ArticleWorkOutcome.Success,
replyable: true,
- cancellationRequested: true);
+ cancellationRequested: true,
+ deliveryCount: null,
+ redelivered: false,
+ deliveryLimit: 10);
Assert.False(disposition.Acknowledge);
Assert.True(disposition.Requeue);
Assert.False(disposition.PublishResponse);
}
+ ///
+ /// When broker delivery count reaches the configured limit, retryable outcomes NACK without requeue.
+ ///
+ [Fact]
+ public void Delivery_limit_exhaustion_nacks_without_requeue()
+ {
+ var disposition = ArticleWorkDispositionPlanner.Create(
+ ArticleWorkOutcome.ProviderFailure,
+ replyable: true,
+ cancellationRequested: false,
+ deliveryCount: 10,
+ redelivered: true,
+ deliveryLimit: 10);
+
+ Assert.False(disposition.Acknowledge);
+ Assert.False(disposition.Requeue);
+ Assert.False(disposition.PublishResponse);
+ }
+
+ ///
+ /// Cancellation that has already exhausted the delivery limit also discards without requeue.
+ ///
+ [Fact]
+ public void Cancellation_with_exhausted_delivery_limit_does_not_requeue()
+ {
+ var disposition = ArticleWorkDispositionPlanner.Create(
+ ArticleWorkOutcome.Success,
+ replyable: true,
+ cancellationRequested: true,
+ deliveryCount: 10,
+ redelivered: true,
+ deliveryLimit: 10);
+
+ Assert.False(disposition.Acknowledge);
+ Assert.False(disposition.Requeue);
+ Assert.False(disposition.PublishResponse);
+ }
+
///
/// Deferred handler (not completed) surfaces provider failure with requeue and no published terminal JSON.
///
@@ -154,7 +204,7 @@ public async Task Terminal_handler_outcomes_publish_and_settle_exactly_once(
var first = await pipeline.ProcessAsync(delivery, "Giganews", channel, static () => true, CancellationToken.None);
var lease = handler.LastItem!.Settlement;
var second = await lease.TrySettleAsync(
- ArticleWorkDispositionPlanner.Create(handlerOutcome, true, false),
+ ArticleWorkDispositionPlanner.Create(handlerOutcome, true, false, deliveryCount: null, redelivered: false, deliveryLimit: 10),
channelStillCurrent: true,
CancellationToken.None);
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Audit/BackFillerRealWorldAuditTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Audit/BackFillerRealWorldAuditTests.cs
index bd3029c7..491913de 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Audit/BackFillerRealWorldAuditTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Audit/BackFillerRealWorldAuditTests.cs
@@ -331,9 +331,12 @@ private static async Task RunFailureCasesAsync(
line($"failure malformed outcome={malformed.Outcome} error={malformed.Error}");
Assert.Equal(ArticleWorkOutcome.InvalidArticle, malformed.Outcome);
- var success = ArticleWorkDispositionPlanner.Create(ArticleWorkOutcome.Success, replyable: true, cancellationRequested: false);
- var requeue = ArticleWorkDispositionPlanner.Create(ArticleWorkOutcome.ProviderFailure, replyable: true, cancellationRequested: false);
- var cancelledDisposition = ArticleWorkDispositionPlanner.Create(ArticleWorkOutcome.Success, replyable: true, cancellationRequested: true);
+ var success = ArticleWorkDispositionPlanner.Create(
+ ArticleWorkOutcome.Success, replyable: true, cancellationRequested: false, deliveryCount: null, redelivered: false, deliveryLimit: 10);
+ var requeue = ArticleWorkDispositionPlanner.Create(
+ ArticleWorkOutcome.ProviderFailure, replyable: true, cancellationRequested: false, deliveryCount: null, redelivered: false, deliveryLimit: 10);
+ var cancelledDisposition = ArticleWorkDispositionPlanner.Create(
+ ArticleWorkOutcome.Success, replyable: true, cancellationRequested: true, deliveryCount: null, redelivered: false, deliveryLimit: 10);
line($"disposition success ack={success.Acknowledge} requeue={success.Requeue}");
line($"disposition providerFailure ack={requeue.Acknowledge} requeue={requeue.Requeue}");
line($"disposition cancelled ack={cancelledDisposition.Acknowledge} requeue={cancelledDisposition.Requeue}");
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeOptionsFactoryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeOptionsFactoryTests.cs
index 4d4baced..142f6bf6 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeOptionsFactoryTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeOptionsFactoryTests.cs
@@ -37,6 +37,11 @@ public void Create_projects_normalized_identity_and_paths()
Assert.Equal(2048, runtime.RabbitMq.WorkRequestMaxPayloadBytes);
Assert.Equal(7, runtime.RabbitMq.PublishConfirmTimeoutSeconds);
Assert.Equal((ushort)3, runtime.RabbitMq.ConsumerPrefetchCount);
+ // Unique hosts after trim/case-fold: 127.0.0.1 and localhost → quorum size 2.
+ Assert.Equal(VectorNNTPD.Common.Messaging.RabbitMq.ArticleWorkQueueKind.Quorum, runtime.RabbitMq.QueueTopology.Kind);
+ Assert.Equal(2, runtime.RabbitMq.QueueTopology.ConfiguredHostCount);
+ Assert.Equal(2, runtime.RabbitMq.QueueTopology.QuorumInitialGroupSize);
+ Assert.Equal(10, runtime.RabbitMq.DeliveryLimit);
Assert.True(Path.IsPathRooted(runtime.LogDirectory));
Assert.True(Path.IsPathRooted(runtime.CertificateDirectory));
Assert.Equal("127.0.0.1", runtime.NntpDb.Server);
@@ -54,6 +59,25 @@ public void Create_projects_normalized_identity_and_paths()
Assert.Equal(TimeSpan.FromSeconds(60), runtime.AccountRefreshInterval);
}
+ ///
+ /// A single configured host projects classic Article Work topology without quorum group size.
+ ///
+ [Fact]
+ public void Create_projects_classic_topology_for_single_host()
+ {
+ var rabbit = BackFillerTestOptions.CreateValidRabbitMqWithHostCount(1);
+ var runtime = BackFillerRuntimeOptionsFactory.Create(
+ BackFillerTestOptions.CreateValid(),
+ BackFillerTestOptions.CreateValidNntpDb(),
+ BackFillerTestOptions.CreateValidAcme(),
+ rabbit);
+
+ Assert.Equal(VectorNNTPD.Common.Messaging.RabbitMq.ArticleWorkQueueKind.Classic, runtime.RabbitMq.QueueTopology.Kind);
+ Assert.Equal(1, runtime.RabbitMq.QueueTopology.ConfiguredHostCount);
+ Assert.Null(runtime.RabbitMq.QueueTopology.QuorumInitialGroupSize);
+ Assert.False(runtime.RabbitMq.QueueTopology.RequiresQuorumHealth);
+ }
+
///
/// Contract: Runtime certificate names are only the backfiller fqdn.
///
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/ArticleWorkTestDeliveries.cs b/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/ArticleWorkTestDeliveries.cs
index edba60ef..23d57b67 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/ArticleWorkTestDeliveries.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/ArticleWorkTestDeliveries.cs
@@ -53,6 +53,7 @@ internal static class ArticleWorkTestDeliveries
/// Optional request-id header mirroring the JSON requestId.
/// AMQP delivery tag for ack/nack tests.
/// Channel generation stamped on the delivery.
+ /// Optional quorum-queue x-delivery-count.
internal static RabbitMqManualAckDelivery Create(
string json,
@@ -61,7 +62,8 @@ internal static RabbitMqManualAckDelivery Create(
string? contentType = ArticleWorkRequestParser.JsonContentType,
string? requestIdHeader = CanonicalRequestId,
ulong deliveryTag = 7,
- long generation = 1)
+ long generation = 1,
+ int? deliveryCount = null)
{
return new RabbitMqManualAckDelivery(
deliveryTag,
@@ -70,11 +72,12 @@ internal static RabbitMqManualAckDelivery Create(
replyTo,
contentType,
requestIdHeader,
- Redelivered: false,
+ Redelivered: deliveryCount is > 0,
RoutingKey: "backfiller.giganews",
Exchange: "backfiller.giganews",
ConsumerTag: "ctag-test",
- generation);
+ generation,
+ deliveryCount);
}
///
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/BackFillerTestOptions.cs b/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/BackFillerTestOptions.cs
index 86b34f43..50b22681 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/BackFillerTestOptions.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Fixtures/BackFillerTestOptions.cs
@@ -50,6 +50,21 @@ internal static RabbitMqOptions CreateValidRabbitMq() =>
EnableSsl = false,
};
+ ///
+ /// Valid RabbitMQ options with unique hosts (quorum when ≥ 2).
+ ///
+ /// Unique host count in 1–7.
+ internal static RabbitMqOptions CreateValidRabbitMqWithHostCount(int hostCount)
+ {
+ ArgumentOutOfRangeException.ThrowIfLessThan(hostCount, 1);
+ ArgumentOutOfRangeException.ThrowIfGreaterThan(hostCount, 7);
+ var rabbit = CreateValidRabbitMq();
+ rabbit.Hosts = Enumerable.Range(0, hostCount)
+ .Select(static i => $"127.0.0.{i + 1}")
+ .ToArray();
+ return rabbit;
+ }
+
/// In-memory configuration key/value pairs that pass production validators.
internal static Dictionary CreateValidConfigurationPairs()
{
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/BackFillerRabbitMqServiceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/BackFillerRabbitMqServiceTests.cs
index a79d848d..6eb45eb4 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/BackFillerRabbitMqServiceTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/BackFillerRabbitMqServiceTests.cs
@@ -794,11 +794,18 @@ internal static RabbitMqService CreateService(
logger ?? NullLogger.Instance);
}
- /// Runtime options with RabbitMQ pool reconnect delays minimized for unit tests.
+ /// Runtime options with RabbitMQ pool reconnect delays minimized for unit tests (single-host classic).
+ internal static BackFillerRuntimeOptions CreateFastRuntime() =>
+ CreateFastRuntime(hostCount: 1);
- internal static BackFillerRuntimeOptions CreateFastRuntime()
+ ///
+ /// Runtime options with minimized reconnect delays and the given unique host count
+ /// (classic when 1, quorum when ≥ 2).
+ ///
+ /// Unique configured host count in 1–7.
+ internal static BackFillerRuntimeOptions CreateFastRuntime(int hostCount)
{
- var rabbit = BackFillerTestOptions.CreateValidRabbitMq();
+ var rabbit = BackFillerTestOptions.CreateValidRabbitMqWithHostCount(hostCount);
rabbit.PoolReconnectBaseDelayMs = 1;
rabbit.PoolReconnectMaxDelayMs = 1;
return BackFillerRuntimeOptionsFactory.Create(
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
index 1eca084e..4881ab05 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs
@@ -658,13 +658,27 @@ public async Task BasicAckAsync(ulong deliveryTag, CancellationToken cancellatio
/// Delivery tag recorded when the nack completes.
/// Requeue flag stored on the settlement.
/// Honored before entering settlement logic.
- public Task BasicNackAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken)
+ public Task BasicNackAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken) =>
+ RecordNegativeSettlementAsync(deliveryTag, requeue, cancellationToken, "NACK");
+
+ ///
+ public Task BasicRejectAsync(ulong deliveryTag, bool requeue, CancellationToken cancellationToken) =>
+ RecordNegativeSettlementAsync(deliveryTag, requeue, cancellationToken, "REJECT");
+
+ ///
+ /// Records a negative settlement (NACK or REJECT) with the same gate/exception hooks used by settlement overlap tests.
+ ///
+ private Task RecordNegativeSettlementAsync(
+ ulong deliveryTag,
+ bool requeue,
+ CancellationToken cancellationToken,
+ string operation)
{
cancellationToken.ThrowIfCancellationRequested();
ObjectDisposedException.ThrowIf(DisposeCount > 0, this);
if (!IsOpen)
{
- throw new InvalidOperationException("RabbitMQ channel is not open for NACK.");
+ throw new InvalidOperationException($"RabbitMQ channel is not open for {operation}.");
}
if (NackException is not null)
diff --git a/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs b/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs
index 2f0ba892..147ff9b2 100644
--- a/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs
+++ b/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs
@@ -158,6 +158,7 @@ private static void AssertSharedValues(RabbitMqOptions options)
Assert.Equal(5672, options.Port);
Assert.Equal(512, options.ChannelPoolSize);
Assert.Equal(1024, options.WorkRequestMaxPayloadBytes);
+ Assert.Equal(10, options.ArticleWorkDeliveryLimit);
Assert.Equal("http://198.18.0.3:15672", options.Management?.BaseUrl);
Assert.Equal(5, options.Management?.RequestTimeoutSeconds);
Assert.Equal("/", options.VirtualHost);
diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs
index 0e4e8878..91fb29b4 100644
--- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs
+++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs
@@ -21,6 +21,7 @@ public void Defaults_MatchSharedContract()
var options = new RabbitMqOptions();
Assert.Equal(1024, options.WorkRequestMaxPayloadBytes);
+ Assert.Equal(10, options.ArticleWorkDeliveryLimit);
Assert.Equal(60, options.ChannelLeaseTimeoutSeconds);
Assert.Equal(30, options.RpcTimeoutSeconds);
Assert.Equal(30, options.ConnectionBlockedTimeoutSeconds);
@@ -78,6 +79,78 @@ public void Validate_Succeeds_WithoutManagement()
Assert.True(result.Succeeded);
}
+ ///
+ /// More than seven unique hosts fails validation rather than clamping quorum initial group size.
+ ///
+ [Fact]
+ public void Validate_Fails_WhenHostsExceedQuorumInitialGroupSizeLimit()
+ {
+ var options = CreateValidConnectivityOnly();
+ options.Hosts =
+ [
+ "127.0.0.1",
+ "127.0.0.2",
+ "127.0.0.3",
+ "127.0.0.4",
+ "127.0.0.5",
+ "127.0.0.6",
+ "127.0.0.7",
+ "127.0.0.8",
+ ];
+ var result = new RabbitMqOptionsValidator().Validate(null, options);
+ Assert.True(result.Failed);
+ Assert.Contains(
+ result.Failures!,
+ static failure => failure.Contains("at most 7 unique entries", StringComparison.Ordinal));
+ }
+
+ ///
+ /// Topology derivation maps unique host counts to classic or quorum without using live connectivity.
+ ///
+ [Theory]
+ [InlineData(1, "Classic", null)]
+ [InlineData(2, "Quorum", 2)]
+ [InlineData(3, "Quorum", 3)]
+ [InlineData(7, "Quorum", 7)]
+ public void FromConfiguredHosts_derives_topology_from_unique_host_count(
+ int hostCount,
+ string expectedKindName,
+ int? expectedGroupSize)
+ {
+ var hosts = Enumerable.Range(0, hostCount).Select(static i => $"127.0.0.{i + 1}").ToArray();
+ var topology = ArticleWorkQueueTopology.FromConfiguredHosts(hosts);
+ Assert.Equal(expectedKindName, topology.Kind.ToString());
+ Assert.Equal(hostCount, topology.ConfiguredHostCount);
+ Assert.Equal(expectedGroupSize, topology.QuorumInitialGroupSize);
+ Assert.Equal(expectedKindName == "Quorum", topology.RequiresQuorumHealth);
+ }
+
+ ///
+ /// Empty host lists cannot derive Article Work topology.
+ ///
+ [Fact]
+ public void FromConfiguredHosts_throws_when_hosts_empty()
+ {
+ Assert.Throws(static () => ArticleWorkQueueTopology.FromConfiguredHosts([]));
+ Assert.Throws(static () => ArticleWorkQueueTopology.FromConfiguredHosts(null));
+ }
+
+ ///
+ /// Article Work delivery limit outside 1–1000 fails validation.
+ ///
+ [Fact]
+ public void Validate_Fails_WhenArticleWorkDeliveryLimitOutOfRange()
+ {
+ var options = CreateValidConnectivityOnly();
+ options.ArticleWorkDeliveryLimit = 1001;
+ var result = new RabbitMqOptionsValidator().Validate(null, options);
+ Assert.True(result.Failed);
+ Assert.Contains(
+ result.Failures!,
+ static failure => failure.Contains("ArticleWorkDeliveryLimit", StringComparison.Ordinal)
+ && failure.Contains("between 1 and 1000", StringComparison.Ordinal));
+ }
+
///
/// Asserts that default options fail validation with a failure that mentions Hosts.
///