diff --git a/.idea/.idea.VectorNNTPD/.idea/inspectionProfiles/Project_Default.xml b/.idea/.idea.VectorNNTPD/.idea/inspectionProfiles/Project_Default.xml index 8d66637c..02ee3119 100644 --- a/.idea/.idea.VectorNNTPD/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/.idea.VectorNNTPD/.idea/inspectionProfiles/Project_Default.xml @@ -1,5 +1,36 @@ \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md index 4473c344..20617b67 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -110,7 +110,7 @@ Consequences: Monitor HTTP never enters Article Work dispatch, settlement, NNTP, or ACK/NACK paths. Failed/missing metrics are `Unknown` (never invented as zero). The monitor does not change reconcile, topology, or logging retention. -Provider queue type is derived from the configured unique `RabbitMQ:Hosts` count (never from reachable hosts or live cluster members). Exactly one host declares a durable classic queue (`x-queue-type=classic`) without quorum arguments or Management quorum health. Two or more hosts declare a durable quorum queue with `x-quorum-initial-group-size` and readiness membership floor equal to that host count, plus configured `x-delivery-limit`. All configured hosts must belong to the same RabbitMQ cluster; the client host list does not establish membership. Successful AMQP quorum declaration alone does not mark a backbone ready: readiness also requires Management HTTP verification of members, online members, and leader. Pre-existing queues with inequivalent type or immutable arguments are not deleted or recreated; operators must migrate or grow membership explicitly. Changing `Hosts` from one entry to multiple (or the reverse) changes the desired queue type and is not a safe in-place conversion. Management outages and timeouts leave multi-host backbones unready and follow the same reconcile backoff. +Provider queue type is derived from the configured unique `RabbitMQ:Hosts` count (never from reachable hosts or live cluster members). Exactly one host declares a durable classic queue (`x-queue-type=classic`) without quorum arguments or Management quorum health. Two or more hosts declare a durable quorum queue with `x-quorum-initial-group-size` and readiness membership floor equal to that host count, plus configured `x-delivery-limit` (default `5` via `ArticleWorkDeliveryLimit`). Both classic and quorum declarations include configured `x-max-length` (AMQP `long`, default `10000` via `ArticleWorkQueueMaxLength`) and `x-message-ttl` (AMQP `long`, default `60000` via `ArticleWorkQueueMessageTtlMilliseconds`) so redeclaration matches existing broker policy. Queue TTL is distinct from per-message expiration on Article Work response publications. All configured hosts must belong to the same RabbitMQ cluster; the client host list does not establish membership. Successful AMQP quorum declaration alone does not mark a backbone ready: readiness also requires Management HTTP verification of members, online members, and leader. Pre-existing queues with inequivalent type or immutable arguments are not deleted or recreated; operators must migrate or grow membership explicitly. Changing `Hosts` from one entry to multiple (or the reverse) changes the desired queue type and is not a safe in-place conversion. Management outages and timeouts leave multi-host backbones unready and follow the same reconcile backoff. ## Article Work settlement diff --git a/docs/common/configuration.md b/docs/common/configuration.md index aa6beec1..baef1c4c 100644 --- a/docs/common/configuration.md +++ b/docs/common/configuration.md @@ -42,7 +42,9 @@ There is no application-level RabbitMQ connection pool. `MinConnections` and `Ma | `PoolReconnectMaxDelayMs` | int | `30000` | **yes** | Application reconnect max delay (`50–300000`; ≥ base) | | `ChannelLeaseTimeoutSeconds` | int | `60` | **yes** | Validated; reserved for later channel work (`1–3600`; ≥ `RpcTimeoutSeconds`) | | `WorkRequestMaxPayloadBytes` | int | `1024` | **yes** | Maximum admitted Article Work request payload bytes (`1–4096`) | -| `ArticleWorkDeliveryLimit` | int | `10` | **yes** | Declared as `x-delivery-limit` on quorum Article Work queues and used as the application requeue cap when broker delivery-count metadata is available (`1–1000`). Not declared on classic single-host queues (broker does not accept that argument there). With count `0..(limit-1)` the app may requeue; at `limit` it rejects without requeue. No DLX is declared; exhausted messages are dropped. Confirmed mandatory-unroutable replies use the same discard class (NACK without requeue) with structured `ReplyUnroutableTerminal` logging — not a dead-letter | +| `ArticleWorkDeliveryLimit` | int | `5` | **yes** | Declared as `x-delivery-limit` on quorum Article Work queues and used as the application requeue cap when broker delivery-count metadata is available (`1–1000`). Not declared on classic single-host queues (broker does not accept that argument there). With count `0..(limit-1)` the app may requeue; at `limit` it rejects without requeue. No DLX is declared; exhausted messages are dropped. Confirmed mandatory-unroutable replies use the same discard class (NACK without requeue) with structured `ReplyUnroutableTerminal` logging — not a dead-letter. Must match any value already configured on the queue | +| `ArticleWorkQueueMaxLength` | int | `10000` | **yes** | Declared as `x-max-length` (AMQP `long`) on classic and quorum Article Work provider queues (`1–10000000`). Caps ready messages retained on the queue. Must match any value already configured on the queue; inequivalent values yield `PRECONDITION_FAILED` and are not rewritten in place | +| `ArticleWorkQueueMessageTtlMilliseconds` | int | `60000` | **yes** | Declared as `x-message-ttl` (AMQP `long`) on classic and quorum Article Work provider queues (`1–86400000`). Ready-message lifetime in milliseconds. Distinct from per-message expiration on Article Work response publications and from `BackFiller:ArticleRetention:RetentionTtlSeconds`. Must match any value already configured on the queue; inequivalent values yield `PRECONDITION_FAILED` and are not rewritten in place | | `ChannelPoolSize` | int | `512` | **yes** | Validated; reserved for later consumer buffering (`1–8192`) | | `MinConnections` | int | `4` | **yes** | Validated; reserved (`1–512`; ≤ `MaxConnections`) | | `MaxConnections` | int | `16` | **yes** | Validated; reserved (`1–512`) | diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponseWireProtocol.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponseWireProtocol.cs index 66d77352..0f84f503 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponseWireProtocol.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponseWireProtocol.cs @@ -28,10 +28,10 @@ internal static class ArticleWorkResponseWireProtocol /// Applied when BackFiller publishes a terminal reply to the caller's exclusive ReplyTo queue. /// Leftover replies should not linger if the caller has already abandoned the correlation. /// This is RabbitMQ message TTL on the reply path, not a processing deadline for a work - /// delivery already handed to a consumer, and not a TTL for work requests waiting - /// in backfiller.{backbone}. Work requests must remain eligible for broker buffering - /// until a consumer delivers them; they must not carry this expiration. Queue declarations - /// must not set x-message-ttl that silently drops ready work. + /// delivery already handed to a consumer, and not the provider-queue ready-message TTL + /// declared as x-message-ttl via RabbitMQ:ArticleWorkQueueMessageTtlMilliseconds. + /// Work-request publishes must not carry this reply-path expiration; queue-level TTL is a + /// separate broker policy argument on backfiller.{backbone}. /// internal const string ExpirationMilliseconds = "1000"; diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationStateMachine.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationStateMachine.cs index f2bcc527..971ac3c4 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationStateMachine.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationStateMachine.cs @@ -5,9 +5,11 @@ namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor /// /// /// neither opens nor resolves: an already-open alert stays open - /// so a Management outage is not treated as recovery. Sustained - /// past the configured open window opens; sustained past the - /// resolve window resolves. Persistence clocks reset when the opposite conclusive signal arrives. + /// so a Management outage is not treated as recovery. Unknown also freezes Active/Inactive + /// persistence clocks (shifts clock origins by the Unknown gap) so outage wall time cannot complete an + /// open or resolve window. Sustained past the configured open + /// window opens; sustained past the resolve window resolves. + /// Persistence clocks reset when the opposite conclusive signal arrives. /// internal sealed class ObservationStateMachine { @@ -29,6 +31,11 @@ internal sealed class ObservationStateMachine /// UTC start of the current Inactive persistence clock while open; null otherwise. private DateTimeOffset? _inactiveSinceUtc; + /// + /// UTC of the last used to freeze persistence across gaps. + /// + private DateTimeOffset? _lastApplyUtc; + /// Creates a machine for one observation identity. /// Stable identity (instance session key, local, or shared queue name). /// Minimum sustained Active duration before . @@ -57,11 +64,14 @@ public ObservationResult Apply(ConditionSignal signal, DateTimeOffset atUtc, str switch (signal) { case ConditionSignal.Unknown: + FreezePersistenceClocks(atUtc); + _lastApplyUtc = atUtc; return Result(signal, ObservationTransition.None, CurrentPersistence(atUtc), detail); case ConditionSignal.Active: _inactiveSinceUtc = null; _activeSinceUtc ??= atUtc; + _lastApplyUtc = atUtc; var activeFor = atUtc - _activeSinceUtc.Value; if (_phase == ObservationPhase.Clear) { @@ -81,10 +91,12 @@ public ObservationResult Apply(ConditionSignal signal, DateTimeOffset atUtc, str if (_phase == ObservationPhase.Clear) { _inactiveSinceUtc = null; + _lastApplyUtc = atUtc; return Result(signal, ObservationTransition.None, TimeSpan.Zero, detail); } _inactiveSinceUtc ??= atUtc; + _lastApplyUtc = atUtc; var inactiveFor = atUtc - _inactiveSinceUtc.Value; if (inactiveFor >= _resolveWindow) { @@ -96,10 +108,40 @@ public ObservationResult Apply(ConditionSignal signal, DateTimeOffset atUtc, str return Result(signal, ObservationTransition.Continued, inactiveFor, detail); default: + _lastApplyUtc = atUtc; return Result(ConditionSignal.Unknown, ObservationTransition.None, TimeSpan.Zero, "unhandled"); } } + /// + /// Shifts Active/Inactive clock origins by the gap since so Unknown + /// wall time does not advance open or resolve persistence. + /// + /// Current evaluation instant. + private void FreezePersistenceClocks(DateTimeOffset atUtc) + { + if (_lastApplyUtc is not { } last) + { + return; + } + + var gap = atUtc - last; + if (gap <= TimeSpan.Zero) + { + return; + } + + if (_activeSinceUtc is { } activeSince) + { + _activeSinceUtc = activeSince + gap; + } + + if (_inactiveSinceUtc is { } inactiveSince) + { + _inactiveSinceUtc = inactiveSince + gap; + } + } + /// Active or Inactive persistence relative to , or zero when neither clock is set. /// Evaluation instant. /// Elapsed persistence for the active clock, inactive clock, or . diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs index 853fc82d..f24805ac 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs @@ -177,8 +177,9 @@ public IReadOnlyList ObserveInstance( /// Thrown when is null. /// /// Non-success polls append an invalid sample so shared evaluators return . - /// Unknown neither opens nor resolves; an already-open shared alert remains open until a conclusive - /// Inactive signal, catalog prune (target gone — not broker recovery), or process restart. + /// Unknown neither opens nor resolves and freezes open/resolve persistence clocks; an already-open + /// shared alert remains open until a conclusive Inactive signal, catalog prune (target gone — not + /// broker recovery), or process restart. /// public IReadOnlyList ObserveSharedQueue( string queueName, diff --git a/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptions.cs b/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptions.cs index aee791e3..25bfd2b2 100644 --- a/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptions.cs +++ b/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptions.cs @@ -205,10 +205,18 @@ internal sealed record NntpDbRuntimeOptions( /// Declared x-delivery-limit on quorum queues and application requeue cap when broker delivery-count /// metadata is available. /// + /// + /// Declared x-max-length on classic and quorum Article Work provider queues (ready-message cap). + /// + /// + /// Declared x-message-ttl on classic and quorum Article Work provider queues (ready-message lifetime). + /// internal sealed record BackFillerRabbitMqRuntimeOptions( int WorkRequestMaxPayloadBytes, int PublishConfirmTimeoutSeconds, ushort? ConsumerPrefetchCount, VectorNNTPD.Common.Messaging.RabbitMq.ArticleWorkQueueTopology QueueTopology, - int DeliveryLimit); + int DeliveryLimit, + int QueueMaxLength, + int QueueMessageTtlMilliseconds); } diff --git a/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptionsFactory.cs b/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptionsFactory.cs index 27f787db..50d35556 100644 --- a/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptionsFactory.cs +++ b/src/VectorNNTPD.BackFillerService/Configuration/BackFillerRuntimeOptionsFactory.cs @@ -188,7 +188,9 @@ internal static BackFillerRuntimeOptions Create( PublishConfirmTimeoutSeconds: rabbitMq.PublishConfirmTimeoutSeconds ?? 10, ConsumerPrefetchCount: rabbitMq.ConsumerPrefetchCount, QueueTopology: ArticleWorkQueueTopology.FromConfiguredHosts(rabbitMq.Hosts), - DeliveryLimit: rabbitMq.ArticleWorkDeliveryLimit ?? 10), + DeliveryLimit: rabbitMq.ArticleWorkDeliveryLimit ?? 5, + QueueMaxLength: rabbitMq.ArticleWorkQueueMaxLength ?? 10000, + QueueMessageTtlMilliseconds: rabbitMq.ArticleWorkQueueMessageTtlMilliseconds ?? 60000), NntpDb: new NntpDbRuntimeOptions( nntpDb.ConnectionString.Trim(), nntpDbBuilder.Server ?? string.Empty, diff --git a/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerArticleWorkTopology.cs b/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerArticleWorkTopology.cs index 62499bea..579cebee 100644 --- a/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerArticleWorkTopology.cs +++ b/src/VectorNNTPD.BackFillerService/RabbitMq/BackFillerArticleWorkTopology.cs @@ -32,6 +32,12 @@ internal static class BackFillerArticleWorkTopology /// Broker argument that caps how many times a quorum message may be delivered before dead-letter/drop. internal const string DeliveryLimitArgumentName = "x-delivery-limit"; + /// Broker argument that caps ready messages retained on the provider queue. + internal const string MaxLengthArgumentName = "x-max-length"; + + /// Broker argument that expires ready messages retained on the provider queue. + internal const string MessageTtlArgumentName = "x-message-ttl"; + /// Fanout exchange type used for provider endpoints. private const string FanoutExchangeType = "fanout"; @@ -52,9 +58,11 @@ internal static class BackFillerArticleWorkTopology /// The exchange is durable with no arguments and is not auto-delete. /// The queue is durable, not exclusive, and not auto-delete. Arguments follow /// : - /// classic queues set to only; - /// quorum queues also set and - /// . Classic brokers do not accept quorum-only arguments. + /// classic queues set to , + /// , and ; quorum queues also set + /// and . + /// Classic brokers do not accept quorum-only arguments. Integer queue arguments are declared as AMQP + /// long so redeclaration matches existing broker policy. /// This method does not delete or rewrite an existing entity. Inequivalent arguments surface as /// broker PRECONDITION_FAILED. Broker failures, including cancellation, propagate from the channel. /// @@ -97,7 +105,7 @@ await channel.QueueBindAsync( } /// - /// Builds durable provider-queue arguments from the Hosts-derived topology and delivery limit. + /// Builds durable provider-queue arguments from the Hosts-derived topology and configured queue policy. /// /// Projected Article Work RabbitMQ knobs. /// Argument dictionary for Queue.Declare. @@ -106,20 +114,26 @@ await channel.QueueBindAsync( ArgumentNullException.ThrowIfNull(rabbitMq); var topology = rabbitMq.QueueTopology; + // RabbitMQ AMQP field tables accept long for these integer arguments. + var maxLength = (long)rabbitMq.QueueMaxLength; + var messageTtl = (long)rabbitMq.QueueMessageTtlMilliseconds; if (topology.Kind == ArticleWorkQueueKind.Classic) { return new Dictionary(StringComparer.Ordinal) { [QueueTypeArgumentName] = ClassicQueueType, + [MaxLengthArgumentName] = maxLength, + [MessageTtlArgumentName] = messageTtl, }; } return new Dictionary(StringComparer.Ordinal) { [QueueTypeArgumentName] = QuorumQueueType, - // RabbitMQ AMQP field tables accept long for these integer arguments. [QuorumInitialGroupSizeArgumentName] = (long)topology.QuorumInitialGroupSize!.Value, [DeliveryLimitArgumentName] = (long)rabbitMq.DeliveryLimit, + [MaxLengthArgumentName] = maxLength, + [MessageTtlArgumentName] = messageTtl, }; } } diff --git a/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json b/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json index a5c40d0e..a749f26b 100644 --- a/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json +++ b/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json @@ -7,10 +7,12 @@ "DegradedThreshold": 0.75, "EnableSsl": false, "Hosts": [ - "198.18.0.3" + "rabbitmq1a.usenet.ninja", + "rabbitmq1b.usenet.ninja", + "rabbitmq1c.usenet.ninja" ], "Management": { - "BaseUrl": "http://198.18.0.3:15672", + "BaseUrl": "http://rabbitmq1a.usenet.ninja:15672", "RequestTimeoutSeconds": 5 }, "MaxConnections": 16, @@ -31,6 +33,8 @@ "UnhealthyThreshold": 5, "VirtualHost": "/", "WorkRequestMaxPayloadBytes": 1024, - "ArticleWorkDeliveryLimit": 10 + "ArticleWorkDeliveryLimit": 5, + "ArticleWorkQueueMaxLength": 10000, + "ArticleWorkQueueMessageTtlMilliseconds": 60000 } } diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs index 9a567cbd..8e944633 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs @@ -227,10 +227,36 @@ public sealed class RabbitMqOptions /// application requeue cap when broker delivery-count metadata is available. /// /// - /// Validated range is 1–1000. Default 10. Declared only when Article Work uses quorum queues + /// Validated range is 1–1000. Default 5. Declared only when Article Work uses quorum queues /// ( count ≥ 2). Classic single-host queues do not support this broker argument. + /// Must match any value already configured on the queue; inequivalent values yield + /// PRECONDITION_FAILED and are not rewritten in place. /// - public int? ArticleWorkDeliveryLimit { get; set; } = 10; + public int? ArticleWorkDeliveryLimit { get; set; } = 5; + + /// + /// Maximum ready-message length declared on Article Work provider queues (x-max-length). + /// + /// + /// Validated range is 1–10_000_000. Default 10000. Declared as AMQP long on both + /// classic and quorum Article Work queues so redeclaration matches existing broker policy. + /// Must match the value already configured on the queue; inequivalent values yield + /// PRECONDITION_FAILED and are not rewritten in place. + /// + public int? ArticleWorkQueueMaxLength { get; set; } = 10000; + + /// + /// Ready-message TTL declared on Article Work provider queues (x-message-ttl), in milliseconds. + /// + /// + /// Validated range is 1–86_400_000 (one day). Default 60000. Declared as AMQP long + /// on both classic and quorum Article Work queues so redeclaration matches existing broker policy. + /// Distinct from per-message expiration on Article Work response publications and from + /// BackFiller:ArticleRetention:RetentionTtlSeconds (in-process retained article lifetime). + /// Must match any value already configured on the queue; inequivalent values yield + /// PRECONDITION_FAILED and are not rewritten in place. + /// + public int? ArticleWorkQueueMessageTtlMilliseconds { get; set; } = 60000; /// /// Optional AMQP CorrelationId used to gate temporary payload diagnostics. diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptionsValidator.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptionsValidator.cs index 6765f5ca..e82f6655 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptionsValidator.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptionsValidator.cs @@ -511,6 +511,24 @@ internal static void ValidateConnection(RabbitMqOptions rabbitMq, List f failures.Add("RabbitMQ:ArticleWorkDeliveryLimit must be between 1 and 1000."); } + if (rabbitMq.ArticleWorkQueueMaxLength is null) + { + failures.Add("RabbitMQ:ArticleWorkQueueMaxLength is required."); + } + else if (rabbitMq.ArticleWorkQueueMaxLength is < 1 or > 10_000_000) + { + failures.Add("RabbitMQ:ArticleWorkQueueMaxLength must be between 1 and 10000000."); + } + + if (rabbitMq.ArticleWorkQueueMessageTtlMilliseconds is null) + { + failures.Add("RabbitMQ:ArticleWorkQueueMessageTtlMilliseconds is required."); + } + else if (rabbitMq.ArticleWorkQueueMessageTtlMilliseconds is < 1 or > 86_400_000) + { + failures.Add("RabbitMQ:ArticleWorkQueueMessageTtlMilliseconds must be between 1 and 86400000."); + } + if (rabbitMq.RequestedChannelMax is null) { failures.Add("RabbitMQ:RequestedChannelMax is required."); diff --git a/src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json b/src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json index a5c40d0e..934ea129 100644 --- a/src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json +++ b/src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json @@ -31,6 +31,8 @@ "UnhealthyThreshold": 5, "VirtualHost": "/", "WorkRequestMaxPayloadBytes": 1024, - "ArticleWorkDeliveryLimit": 10 + "ArticleWorkDeliveryLimit": 5, + "ArticleWorkQueueMaxLength": 10000, + "ArticleWorkQueueMessageTtlMilliseconds": 60000 } } diff --git a/src/VectorNNTPD.StorageService/VectorNNTPD.RabbitMQ.json b/src/VectorNNTPD.StorageService/VectorNNTPD.RabbitMQ.json index a5c40d0e..934ea129 100644 --- a/src/VectorNNTPD.StorageService/VectorNNTPD.RabbitMQ.json +++ b/src/VectorNNTPD.StorageService/VectorNNTPD.RabbitMQ.json @@ -31,6 +31,8 @@ "UnhealthyThreshold": 5, "VirtualHost": "/", "WorkRequestMaxPayloadBytes": 1024, - "ArticleWorkDeliveryLimit": 10 + "ArticleWorkDeliveryLimit": 5, + "ArticleWorkQueueMaxLength": 10000, + "ArticleWorkQueueMessageTtlMilliseconds": 60000 } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkProviderTopologyTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkProviderTopologyTests.cs index 196aafe3..cd76be76 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkProviderTopologyTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkProviderTopologyTests.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Logging.Abstractions; using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Configuration; using VectorNNTPD.BackFillerService.Nntp; using VectorNNTPD.BackFillerService.RabbitMq; using VectorNNTPD.BackFillerService.Tests.Fixtures; @@ -44,10 +45,13 @@ public async Task Active_backbone_declares_classic_fanout_topology_before_consum Assert.NotNull(queue.Arguments); Assert.True(queue.Arguments.TryGetValue(BackFillerArticleWorkTopology.QueueTypeArgumentName, out var queueType)); Assert.Equal(BackFillerArticleWorkTopology.ClassicQueueType, queueType); + Assert.Equal(10000L, queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); Assert.False(queue.Arguments.ContainsKey(BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName)); Assert.False(queue.Arguments.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName)); - Assert.False(queue.Arguments.ContainsKey("x-message-ttl")); - Assert.Single(queue.Arguments); + Assert.Equal(3, queue.Arguments.Count); Assert.Equal( ("backfiller.giganews", "backfiller.giganews", "backfiller.giganews"), Assert.Single(topology.BindingDeclarations)); @@ -81,8 +85,155 @@ public async Task Active_backbone_declares_quorum_fanout_topology_for_multi_host var queue = Assert.Single(topology.QueueDeclarations); Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, queue.Arguments![BackFillerArticleWorkTopology.QueueTypeArgumentName]); Assert.Equal(3L, queue.Arguments[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]); - Assert.Equal(10L, queue.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); - Assert.Equal(3, queue.Arguments.Count); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]); + Assert.Equal(5L, queue.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.Equal(10000L, queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.Equal(5, queue.Arguments.Count); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// Both provider queues declare identical classic arguments including max-length and message TTL as AMQP long. + /// + [Fact] + public async Task Eweka_and_giganews_declare_identical_classic_queue_arguments() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + var consumer = CreateConsumer( + connections, + CreateCatalog([("Giganews", 1), ("Eweka", 1)]), + CreateCapacity([("Giganews", 1), ("Eweka", 1)])); + + await consumer.StartAsync(CancellationToken.None); + + var declarations = factory.LastConnection!.Channels + .SelectMany(static channel => channel.QueueDeclarations) + .OrderBy(static q => q.Name, StringComparer.Ordinal) + .ToArray(); + Assert.Equal(["backfiller.eweka", "backfiller.giganews"], declarations.Select(static q => q.Name)); + Assert.All(declarations, static queue => + { + Assert.NotNull(queue.Arguments); + Assert.Equal(BackFillerArticleWorkTopology.ClassicQueueType, queue.Arguments[BackFillerArticleWorkTopology.QueueTypeArgumentName]); + Assert.Equal(10000L, queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.Equal(3, queue.Arguments.Count); + }); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// Both provider queues declare identical quorum arguments for a three-host topology. + /// + [Fact] + public async Task Eweka_and_giganews_declare_identical_quorum_queue_arguments() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + var consumer = CreateConsumer( + connections, + CreateCatalog([("Giganews", 1), ("Eweka", 1)]), + CreateCapacity([("Giganews", 1), ("Eweka", 1)]), + hostCount: 3); + + await consumer.StartAsync(CancellationToken.None); + + var declarations = factory.LastConnection!.Channels + .SelectMany(static channel => channel.QueueDeclarations) + .OrderBy(static q => q.Name, StringComparer.Ordinal) + .ToArray(); + Assert.Equal(["backfiller.eweka", "backfiller.giganews"], declarations.Select(static q => q.Name)); + Assert.All(declarations, static queue => + { + Assert.NotNull(queue.Arguments); + Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, queue.Arguments[BackFillerArticleWorkTopology.QueueTypeArgumentName]); + Assert.Equal(3L, queue.Arguments[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]); + Assert.Equal(5L, queue.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.Equal(10000L, queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.Equal(5, queue.Arguments.Count); + }); + + await consumer.DisposeAsync(); + await connections.DisposeAsync(); + } + + /// + /// Configured queue max length, delivery limit, and message TTL reach declarations as AMQP long. + /// + [Theory] + [InlineData(1, 5000, 5, 45000)] + [InlineData(3, 25000, 7, 90000)] + public async Task Configured_queue_policy_reaches_declaration( + int hostCount, + int maxLength, + int deliveryLimit, + int messageTtlMilliseconds) + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + var rabbit = BackFillerTestOptions.CreateValidRabbitMqWithHostCount(hostCount); + rabbit.PoolReconnectBaseDelayMs = 1; + rabbit.PoolReconnectMaxDelayMs = 1; + rabbit.ArticleWorkQueueMaxLength = maxLength; + rabbit.ArticleWorkDeliveryLimit = deliveryLimit; + rabbit.ArticleWorkQueueMessageTtlMilliseconds = messageTtlMilliseconds; + var runtime = BackFillerRuntimeOptionsFactory.Create( + BackFillerTestOptions.CreateValid(), + BackFillerTestOptions.CreateValidNntpDb(), + BackFillerTestOptions.CreateValidAcme(), + rabbit); + Assert.Equal(maxLength, runtime.RabbitMq.QueueMaxLength); + Assert.Equal(deliveryLimit, runtime.RabbitMq.DeliveryLimit); + Assert.Equal(messageTtlMilliseconds, runtime.RabbitMq.QueueMessageTtlMilliseconds); + var consumer = new ArticleWorkConsumerService( + connections, + runtime, + new DeferredArticleWorkHandler(), + new RecordingArticleWorkResponsePublisher(), + NullLogger.Instance, + CreateCatalog([("Giganews", 1)]), + CreateCapacity([("Giganews", 1)])); + + await consumer.StartAsync(CancellationToken.None); + + var queue = Assert.Single(factory.LastConnection!.Channels[0].QueueDeclarations); + Assert.Equal((long)maxLength, queue.Arguments![BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal((long)messageTtlMilliseconds, queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + if (hostCount == 1) + { + Assert.Equal(BackFillerArticleWorkTopology.ClassicQueueType, queue.Arguments[BackFillerArticleWorkTopology.QueueTypeArgumentName]); + Assert.False(queue.Arguments.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName)); + Assert.Equal(3, queue.Arguments.Count); + } + else + { + Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, queue.Arguments[BackFillerArticleWorkTopology.QueueTypeArgumentName]); + Assert.Equal((long)deliveryLimit, queue.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.Equal((long)hostCount, queue.Arguments[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]); + Assert.IsType(queue.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.Equal(5, queue.Arguments.Count); + } await consumer.DisposeAsync(); await connections.DisposeAsync(); @@ -285,6 +436,10 @@ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync( Assert.NotNull(q.Arguments); Assert.True(q.Arguments.TryGetValue(BackFillerArticleWorkTopology.QueueTypeArgumentName, out var type)); Assert.Equal(BackFillerArticleWorkTopology.ClassicQueueType, type); + Assert.Equal(10000L, q.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(q.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, q.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(q.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); Assert.False(q.Arguments.ContainsKey(BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName)); Assert.False(q.Arguments.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName)); }); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs index b3a5f9e4..e0bc2293 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs @@ -44,8 +44,13 @@ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync( Assert.Equal( BackFillerArticleWorkTopology.ClassicQueueType, declaration.Arguments![BackFillerArticleWorkTopology.QueueTypeArgumentName]); + Assert.Equal(10000L, declaration.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(declaration.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, declaration.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(declaration.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); Assert.False(declaration.Arguments.ContainsKey(BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName)); Assert.False(declaration.Arguments.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName)); + Assert.Equal(3, declaration.Arguments.Count); Assert.Equal(ArticleWorkQueueKind.Classic, runtime.RabbitMq.QueueTopology.Kind); Assert.Equal(1, runtime.RabbitMq.QueueTopology.ConfiguredHostCount); Assert.Null(runtime.RabbitMq.QueueTopology.QuorumInitialGroupSize); @@ -73,6 +78,11 @@ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync( declaration.Arguments![BackFillerArticleWorkTopology.QueueTypeArgumentName]); Assert.Equal(2L, declaration.Arguments[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]); Assert.Equal(7L, declaration.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.Equal(10000L, declaration.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, declaration.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(declaration.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(declaration.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.Equal(5, declaration.Arguments.Count); Assert.Equal(2, runtime.RabbitMq.QueueTopology.QuorumInitialGroupSize); } @@ -83,7 +93,7 @@ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync( public async Task Three_hosts_declare_quorum_topology_with_group_size_3() { var channel = new FakeBackFillerRabbitMqChannel(1); - var runtime = CreateRuntime(hostCount: 3, deliveryLimit: 10); + var runtime = CreateRuntime(hostCount: 3, deliveryLimit: 5); await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync( channel, @@ -93,6 +103,14 @@ await BackFillerArticleWorkTopology.DeclareProviderEndpointAsync( var declaration = Assert.Single(channel.QueueDeclarations); Assert.Equal(3L, declaration.Arguments![BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]); + Assert.Equal(5L, declaration.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.Equal(10000L, declaration.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, declaration.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(declaration.Arguments[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.IsType(declaration.Arguments[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.IsType(declaration.Arguments[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, declaration.Arguments[BackFillerArticleWorkTopology.QueueTypeArgumentName]); + Assert.Equal(5, declaration.Arguments.Count); Assert.Equal(3, runtime.RabbitMq.QueueTopology.ConfiguredHostCount); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs index 02839be3..19ac9b3e 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs @@ -673,7 +673,9 @@ public async Task Unroutable_terminal_settlement_is_discard_without_requeue_not_ ArticleWorkQueueKind.Classic, ConfiguredHostCount: 1, QuorumInitialGroupSize: null), - DeliveryLimit: 10)) + DeliveryLimit: 5, + QueueMaxLength: 10000, + QueueMessageTtlMilliseconds: 60000)) .ContainsKey("x-dead-letter-exchange")); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyLabRedeclarationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyLabRedeclarationTests.cs new file mode 100644 index 00000000..54ad6b6a --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyLabRedeclarationTests.cs @@ -0,0 +1,148 @@ +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using RabbitMQ.Client; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.RabbitMq; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork +{ + /// + /// Opt-in lab verification that application queue arguments redeclare existing provider queues without PRECONDITION_FAILED. + /// + /// + /// Disabled unless VECTORNNTPD_ARTICLE_WORK_TOPOLOGY_LAB=1. Does not purge, delete, or rewrite queue policy; + /// it only redeclares with . + /// + public sealed class ArticleWorkTopologyLabRedeclarationTests + { + /// Master gate. Must be 1; any other value keeps this test a no-op. + internal const string EnableEnvironmentVariable = "VECTORNNTPD_ARTICLE_WORK_TOPOLOGY_LAB"; + + /// + /// Redeclares backfiller.eweka and backfiller.giganews using runtime-projected application arguments. + /// + /// + /// Uses Hosts and Management from shipped VectorNNTPD.RabbitMQ.json only. Does not honor the + /// single-host stress RabbitMQ host override, which targets a different classic-queue lab broker. + /// + [Fact] + public async Task Lab_redeclares_existing_provider_queues_using_application_arguments() + { + if (!IsEnabled()) + { + return; + } + + var username = RequireEnv("VECTOR__RABBITMQ__USERNAME"); + var password = RequireEnv("VECTOR__RABBITMQ__PASSWORD"); + var rabbit = BindShippedRabbitMqOptions(); + rabbit.Username = username; + rabbit.Password = password; + + Assert.NotNull(rabbit.Hosts); + Assert.True(rabbit.Hosts.Length >= 2, "Lab redeclaration requires multi-host quorum topology from shipped RabbitMQ JSON."); + Assert.False(string.IsNullOrWhiteSpace(rabbit.Management?.BaseUrl)); + + var runtime = BackFillerRuntimeOptionsFactory.Create( + BackFillerTestOptions.CreateValid(), + BackFillerTestOptions.CreateValidNntpDb(), + BackFillerTestOptions.CreateValidAcme(), + rabbit); + + var intended = BackFillerArticleWorkTopology.BuildQueueArguments(runtime.RabbitMq); + Assert.Equal(BackFillerArticleWorkTopology.QuorumQueueType, intended[BackFillerArticleWorkTopology.QueueTypeArgumentName]); + Assert.Equal(3L, intended[BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName]); + Assert.Equal(5L, intended[BackFillerArticleWorkTopology.DeliveryLimitArgumentName]); + Assert.Equal(10000L, intended[BackFillerArticleWorkTopology.MaxLengthArgumentName]); + Assert.Equal(60000L, intended[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.All( + new[] + { + BackFillerArticleWorkTopology.QuorumInitialGroupSizeArgumentName, + BackFillerArticleWorkTopology.DeliveryLimitArgumentName, + BackFillerArticleWorkTopology.MaxLengthArgumentName, + BackFillerArticleWorkTopology.MessageTtlArgumentName, + }, + name => Assert.IsType(intended[name])); + + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; + var token = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}")); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", token); + + foreach (var queueName in new[] { "backfiller.eweka", "backfiller.giganews" }) + { + var managementUri = $"{rabbit.Management.BaseUrl!.TrimEnd('/')}/api/queues/%2F/{Uri.EscapeDataString(queueName)}"; + using var response = await http.GetAsync(managementUri); + response.EnsureSuccessStatusCode(); + await using var stream = await response.Content.ReadAsStreamAsync(); + using var document = await JsonDocument.ParseAsync(stream); + var arguments = document.RootElement.GetProperty("arguments"); + Assert.Equal("quorum", arguments.GetProperty("x-queue-type").GetString()); + Assert.Equal(3, arguments.GetProperty("x-quorum-initial-group-size").GetInt32()); + Assert.Equal(5, arguments.GetProperty("x-delivery-limit").GetInt32()); + Assert.Equal(10000, arguments.GetProperty("x-max-length").GetInt32()); + Assert.Equal(60000, arguments.GetProperty("x-message-ttl").GetInt32()); + + var factory = new ConnectionFactory + { + HostName = rabbit.Hosts[0], + Port = rabbit.Port ?? 5672, + UserName = username, + Password = password, + VirtualHost = rabbit.VirtualHost ?? "/", + RequestedHeartbeat = TimeSpan.FromSeconds(30), + }; + await using var connection = await factory.CreateConnectionAsync( + $"vectornntp-topology-lab-{queueName}"); + await using var channel = await connection.CreateChannelAsync(); + var declare = await channel.QueueDeclareAsync( + queue: queueName, + durable: true, + exclusive: false, + autoDelete: false, + arguments: intended, + cancellationToken: CancellationToken.None); + Assert.Equal(queueName, declare.QueueName); + } + } + + /// Returns whether the lab redeclaration probe should execute. + private static bool IsEnabled() => + string.Equals( + Environment.GetEnvironmentVariable(EnableEnvironmentVariable), + "1", + StringComparison.Ordinal); + + /// Binds shipped VectorNNTPD.RabbitMQ.json without hard-coding broker hostnames in this test. + private static RabbitMqOptions BindShippedRabbitMqOptions() + { + var path = RepositoryFiles.Under( + "src", + "VectorNNTPD.BackFillerService", + "VectorNNTPD.RabbitMQ.json"); + Assert.True(File.Exists(path), path); + var configuration = new Microsoft.Extensions.Configuration.ConfigurationBuilder() + .AddJsonFile(path, optional: false, reloadOnChange: false) + .Build(); + var rabbit = new RabbitMqOptions(); + configuration.GetSection(RabbitMqOptions.SectionName).Bind(rabbit); + return rabbit; + } + + /// Reads a required environment variable or throws without echoing the value. + private static string RequireEnv(string name) + { + var value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"{name} is required when {EnableEnvironmentVariable}=1."); + } + + return value; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ObservationStateMachineTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ObservationStateMachineTests.cs index 2ef07f63..7417ff0e 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ObservationStateMachineTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ObservationStateMachineTests.cs @@ -41,5 +41,64 @@ public void Unknown_does_not_open_or_resolve() Assert.Equal(ObservationTransition.Continued, machine.Apply(ConditionSignal.Inactive, t0.AddSeconds(2)).Transition); Assert.Equal(ObservationTransition.Resolved, machine.Apply(ConditionSignal.Inactive, t0.AddSeconds(3)).Transition); } + + /// + /// Unknown during Clear-phase Active accumulation must not count toward the open window + /// (Active bookends with a long Unknown gap must not Open). + /// + [Fact] + public void Unknown_freezes_active_open_persistence_while_clear() + { + var key = new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.DispatchStall, "q"); + var open = TimeSpan.FromSeconds(90); + var machine = new ObservationStateMachine(key, open, TimeSpan.FromSeconds(20)); + var t0 = DateTimeOffset.Parse("2026-10-26T15:00:00Z"); + + Assert.Equal(ObservationTransition.None, machine.Apply(ConditionSignal.Active, t0).Transition); + var unknown = machine.Apply(ConditionSignal.Unknown, t0.AddSeconds(60)); + Assert.Equal(ObservationTransition.None, unknown.Transition); + Assert.Equal(ObservationPhase.Clear, unknown.Phase); + Assert.Equal(TimeSpan.Zero, unknown.Persistence); + + // Wall clock from first Active would be 90s, but Unknown froze the clock — still Clear. + var stillClear = machine.Apply(ConditionSignal.Active, t0.AddSeconds(90)); + Assert.Equal(ObservationTransition.None, stillClear.Transition); + Assert.Equal(ObservationPhase.Clear, stillClear.Phase); + Assert.True(stillClear.Persistence < open); + + Assert.Equal( + ObservationTransition.Opened, + machine.Apply(ConditionSignal.Active, t0.AddSeconds(90) + open).Transition); + } + + /// + /// Unknown during Open-phase Inactive accumulation must not count toward the resolve window. + /// + [Fact] + public void Unknown_freezes_inactive_resolve_persistence_while_open() + { + var key = new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.ConsumerLoss, "q"); + var resolve = TimeSpan.FromSeconds(20); + var machine = new ObservationStateMachine(key, TimeSpan.FromSeconds(1), resolve); + var t0 = DateTimeOffset.Parse("2026-10-26T15:00:00Z"); + + Assert.Equal(ObservationTransition.None, machine.Apply(ConditionSignal.Active, t0).Transition); + Assert.Equal(ObservationTransition.Opened, machine.Apply(ConditionSignal.Active, t0.AddSeconds(1)).Transition); + Assert.Equal(ObservationTransition.Continued, machine.Apply(ConditionSignal.Inactive, t0.AddSeconds(2)).Transition); + + var unknown = machine.Apply(ConditionSignal.Unknown, t0.AddSeconds(2) + resolve); + Assert.Equal(ObservationTransition.None, unknown.Transition); + Assert.Equal(ObservationPhase.Open, unknown.Phase); + Assert.True(unknown.Persistence < resolve); + + // Same wall gap after first Inactive would resolve without freeze; with freeze still Open. + var stillOpen = machine.Apply(ConditionSignal.Inactive, t0.AddSeconds(2) + resolve); + Assert.Equal(ObservationTransition.Continued, stillOpen.Transition); + Assert.Equal(ObservationPhase.Open, stillOpen.Phase); + + Assert.Equal( + ObservationTransition.Resolved, + machine.Apply(ConditionSignal.Inactive, t0.AddSeconds(2) + resolve + resolve).Transition); + } } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluatorAndEngineTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluatorAndEngineTests.cs index 7f5e2954..a3007134 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluatorAndEngineTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluatorAndEngineTests.cs @@ -207,6 +207,48 @@ ObservationResult LastConsumerLoss(IReadOnlyList results) => Assert.DoesNotContain("removed from catalog", resolved.Detail ?? string.Empty, StringComparison.OrdinalIgnoreCase); } + /// + /// Failed Management polls (Unknown) during Clear-phase accumulation must not complete the shared open window. + /// + [Fact] + public void Unknown_management_gap_does_not_complete_shared_open_window() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var unknownPoll = new ManagementPollResult + { + Kind = ManagementPollKind.TransientFailure, + Reason = "timeout", + }; + + ObservationResult LastLoss(IReadOnlyList results) => + results.Single(r => + r.Key.Condition == SilentConsumerCondition.ConsumerLoss + && r.Key.Scope == ObservationScope.SharedQueue); + + Assert.Equal( + ObservationTransition.None, + LastLoss(engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true)).Transition); + + time.Advance(TimeSpan.FromSeconds(60)); + Assert.Equal( + ObservationTransition.None, + LastLoss(engine.ObserveSharedQueue("backfiller.giganews", unknownPoll, true)).Transition); + Assert.Equal(ObservationPhase.Clear, LastLoss( + engine.ObserveSharedQueue("backfiller.giganews", unknownPoll, true)).Phase); + + // Wall time since first Active would meet ConsumerLoss open (45s); freeze keeps Clear. + time.Advance(TimeSpan.FromSeconds(30)); + Assert.Equal( + ObservationTransition.None, + LastLoss(engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true)).Transition); + + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + Assert.Equal( + ObservationTransition.Opened, + LastLoss(engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true)).Transition); + } + /// /// Event 9113 logs once per observation key per open-window interval; distinct queue keys and advanced time emit additional entries. /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/ApplicationJsonConfigurationSourceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/ApplicationJsonConfigurationSourceTests.cs index 83028f12..6703af7e 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/ApplicationJsonConfigurationSourceTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/ApplicationJsonConfigurationSourceTests.cs @@ -240,7 +240,7 @@ public void Shipped_rabbitmq_file_is_copied_and_loads() var configuration = builder.Build(); Assert.Equal("5672", configuration["RabbitMQ:Port"]); - Assert.Equal("198.18.0.3", configuration["RabbitMQ:Hosts:0"]); + Assert.Equal("rabbitmq1a.usenet.ninja", configuration["RabbitMQ:Hosts:0"]); Assert.Equal(bool.FalseString, configuration["RabbitMQ:EnableSsl"]); Assert.Equal("/", configuration["RabbitMQ:VirtualHost"]); @@ -249,13 +249,18 @@ public void Shipped_rabbitmq_file_is_copied_and_loads() var validation = new RabbitMqOptionsValidator().Validate(null, rabbit); Assert.True(validation.Succeeded, string.Join("; ", validation.Failures ?? [])); Assert.NotNull(rabbit.Hosts); - Assert.Equal(["198.18.0.3"], rabbit.Hosts); + Assert.Equal( + ["rabbitmq1a.usenet.ninja", "rabbitmq1b.usenet.ninja", "rabbitmq1c.usenet.ninja"], + rabbit.Hosts); Assert.Equal(5672, rabbit.Port); Assert.Equal(false, rabbit.EnableSsl); Assert.Equal("/", rabbit.VirtualHost); Assert.Equal(512, rabbit.ChannelPoolSize); Assert.Equal(1024, rabbit.WorkRequestMaxPayloadBytes); - Assert.Equal("http://198.18.0.3:15672", rabbit.Management.BaseUrl); + Assert.Equal(5, rabbit.ArticleWorkDeliveryLimit); + Assert.Equal(10000, rabbit.ArticleWorkQueueMaxLength); + Assert.Equal(60000, rabbit.ArticleWorkQueueMessageTtlMilliseconds); + Assert.Equal("http://rabbitmq1a.usenet.ninja:15672", rabbit.Management.BaseUrl); Assert.Equal(5, rabbit.Management.RequestTimeoutSeconds); } finally diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeOptionsFactoryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeOptionsFactoryTests.cs index 142f6bf6..6f896a5b 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeOptionsFactoryTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeOptionsFactoryTests.cs @@ -41,7 +41,9 @@ public void Create_projects_normalized_identity_and_paths() Assert.Equal(VectorNNTPD.Common.Messaging.RabbitMq.ArticleWorkQueueKind.Quorum, runtime.RabbitMq.QueueTopology.Kind); Assert.Equal(2, runtime.RabbitMq.QueueTopology.ConfiguredHostCount); Assert.Equal(2, runtime.RabbitMq.QueueTopology.QuorumInitialGroupSize); - Assert.Equal(10, runtime.RabbitMq.DeliveryLimit); + Assert.Equal(5, runtime.RabbitMq.DeliveryLimit); + Assert.Equal(10000, runtime.RabbitMq.QueueMaxLength); + Assert.Equal(60000, runtime.RabbitMq.QueueMessageTtlMilliseconds); Assert.True(Path.IsPathRooted(runtime.LogDirectory)); Assert.True(Path.IsPathRooted(runtime.CertificateDirectory)); Assert.Equal("127.0.0.1", runtime.NntpDb.Server); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs index 8ea11720..be317227 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs @@ -58,9 +58,11 @@ public void Response_ttl_remains_one_second_and_is_not_applied_to_stress_request props.Expiration ?? string.Empty); } - /// Classic and quorum queue arguments must not declare x-message-ttl. + /// + /// Classic and quorum provider queues declare configured x-message-ttl; response-path expiration remains separate. + /// [Fact] - public void Provider_work_queue_arguments_do_not_declare_x_message_ttl() + public void Provider_work_queue_arguments_declare_configured_x_message_ttl() { var classic = BackFillerArticleWorkTopology.BuildQueueArguments( new BackFillerRabbitMqRuntimeOptions( @@ -71,8 +73,14 @@ public void Provider_work_queue_arguments_do_not_declare_x_message_ttl() ArticleWorkQueueKind.Classic, ConfiguredHostCount: 1, QuorumInitialGroupSize: null), - DeliveryLimit: 10)); - Assert.False(classic.ContainsKey("x-message-ttl")); + DeliveryLimit: 5, + QueueMaxLength: 10000, + QueueMessageTtlMilliseconds: 60000)); + Assert.Equal(60000L, classic[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(classic[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.NotEqual( + ArticleWorkResponseWireProtocol.ExpirationMilliseconds, + classic[BackFillerArticleWorkTopology.MessageTtlArgumentName]!.ToString()); var quorum = BackFillerArticleWorkTopology.BuildQueueArguments( new BackFillerRabbitMqRuntimeOptions( @@ -83,8 +91,11 @@ public void Provider_work_queue_arguments_do_not_declare_x_message_ttl() ArticleWorkQueueKind.Quorum, ConfiguredHostCount: 3, QuorumInitialGroupSize: 3), - DeliveryLimit: 10)); - Assert.False(quorum.ContainsKey("x-message-ttl")); + DeliveryLimit: 5, + QueueMaxLength: 10000, + QueueMessageTtlMilliseconds: 60000)); + Assert.Equal(60000L, quorum[BackFillerArticleWorkTopology.MessageTtlArgumentName]); + Assert.IsType(quorum[BackFillerArticleWorkTopology.MessageTtlArgumentName]); Assert.True(quorum.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName)); } diff --git a/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs b/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs index 9b1b37ba..c187ba3a 100644 --- a/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Configuration/SharedRabbitMqJsonTests.cs @@ -158,7 +158,9 @@ private static void AssertSharedValues(RabbitMqOptions options) Assert.Equal(5672, options.Port); Assert.Equal(512, options.ChannelPoolSize); Assert.Equal(1024, options.WorkRequestMaxPayloadBytes); - Assert.Equal(10, options.ArticleWorkDeliveryLimit); + Assert.Equal(5, options.ArticleWorkDeliveryLimit); + Assert.Equal(10000, options.ArticleWorkQueueMaxLength); + Assert.Equal(60000, options.ArticleWorkQueueMessageTtlMilliseconds); Assert.Equal("http://198.18.0.3:15672", options.Management?.BaseUrl); Assert.Equal(5, options.Management?.RequestTimeoutSeconds); Assert.Equal("/", options.VirtualHost); diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs index 91fb29b4..0f8f6acb 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs @@ -21,7 +21,9 @@ public void Defaults_MatchSharedContract() var options = new RabbitMqOptions(); Assert.Equal(1024, options.WorkRequestMaxPayloadBytes); - Assert.Equal(10, options.ArticleWorkDeliveryLimit); + Assert.Equal(5, options.ArticleWorkDeliveryLimit); + Assert.Equal(10000, options.ArticleWorkQueueMaxLength); + Assert.Equal(60000, options.ArticleWorkQueueMessageTtlMilliseconds); Assert.Equal(60, options.ChannelLeaseTimeoutSeconds); Assert.Equal(30, options.RpcTimeoutSeconds); Assert.Equal(30, options.ConnectionBlockedTimeoutSeconds); @@ -151,6 +153,38 @@ public void Validate_Fails_WhenArticleWorkDeliveryLimitOutOfRange() && failure.Contains("between 1 and 1000", StringComparison.Ordinal)); } + /// + /// Article Work queue max length outside 1–10000000 fails validation. + /// + [Fact] + public void Validate_Fails_WhenArticleWorkQueueMaxLengthOutOfRange() + { + var options = CreateValidConnectivityOnly(); + options.ArticleWorkQueueMaxLength = 0; + var result = new RabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("ArticleWorkQueueMaxLength", StringComparison.Ordinal) + && failure.Contains("between 1 and 10000000", StringComparison.Ordinal)); + } + + /// + /// Article Work queue message TTL outside 1–86400000 fails validation. + /// + [Fact] + public void Validate_Fails_WhenArticleWorkQueueMessageTtlOutOfRange() + { + var options = CreateValidConnectivityOnly(); + options.ArticleWorkQueueMessageTtlMilliseconds = 0; + var result = new RabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("ArticleWorkQueueMessageTtlMilliseconds", StringComparison.Ordinal) + && failure.Contains("between 1 and 86400000", StringComparison.Ordinal)); + } + /// /// Asserts that default options fail validation with a failure that mentions Hosts. ///