From 2e445650d7fa855b600941dd662b58a762cc73f8 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 21 Oct 2026 19:17:35 +0200 Subject: [PATCH 1/3] Fix RabbitMQ blocked connection failover --- docs/common/configuration.md | 6 +- .../Messaging/RabbitMq/IRabbitMqConnection.cs | 18 + .../RabbitMq/RabbitMqClientConnection.cs | 21 +- .../RabbitMqClientConnectionFactory.cs | 4 +- .../RabbitMqConnectionBlockedEventArgs.cs | 6 + .../Messaging/RabbitMq/RabbitMqLogMessages.cs | 18 + .../Messaging/RabbitMq/RabbitMqOptions.cs | 8 +- .../RabbitMq/RabbitMqRuntimeOptions.cs | 4 +- .../Messaging/RabbitMq/RabbitMqService.cs | 226 ++++++- .../RabbitMqBlockedConnectionTimeoutTests.cs | 577 ++++++++++++++++++ .../TestDoubles/FakeBackFillerRabbitMq.cs | 19 + .../RabbitMqClientTlsEndpointTests.cs | 22 + .../RabbitMq/RabbitMqLogEventSinkTests.cs | 27 + .../RabbitMq/RabbitMqOptionsValidatorTests.cs | 37 ++ 14 files changed, 970 insertions(+), 23 deletions(-) create mode 100644 src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqConnectionBlockedEventArgs.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqBlockedConnectionTimeoutTests.cs diff --git a/docs/common/configuration.md b/docs/common/configuration.md index 6a54d616..0cba3fd4 100644 --- a/docs/common/configuration.md +++ b/docs/common/configuration.md @@ -20,7 +20,7 @@ Top-level `RabbitMQ` section. The Common sample is `src/VectorNNTPD.Common/Vecto NNTPD and StorageServer are not projects in this repository, so their build output is not part of the contract here. BackFiller ships connectivity in `src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json`. Its primary file, `VectorNNTPD.BackFillerService.json`, still must not repeat a top-level `RabbitMQ` section. `BackFiller:Logging:RabbitMQ` is the log sink, not broker connectivity. -`RabbitMqService` owns one process-wide connection, verifies that it is open, and replaces it on connectivity loss while incrementing a monotonic generation. Client automatic recovery is disabled. The AMQP client name comes from `IRabbitMqConnectionNameProvider`. Callers obtain the current connection with `TryGetCurrent` and do not dispose it. +`RabbitMqService` owns one process-wide connection, verifies that it is open, and replaces it on connectivity loss or when a broker block lasts longer than `ConnectionBlockedTimeoutSeconds`, while incrementing a monotonic generation. Client automatic recovery is disabled. The AMQP client name comes from `IRabbitMqConnectionNameProvider`. Callers obtain the current connection with `TryGetCurrent` and do not dispose it. There is no application-level RabbitMQ connection pool. `MinConnections` and `MaxConnections` are validated and are not enforced as a pool. @@ -33,10 +33,10 @@ There is no application-level RabbitMQ connection pool. `MinConnections` and `Ma | `VirtualHost` | string | `/` | **yes** | RabbitMQ virtual host | | `EnableSsl` | bool | `true` | **yes** | Whether the connection uses TLS (TLS 1.2 and 1.3) | | `RequestedHeartbeatSeconds` | int | `60` | **yes** | AMQP heartbeat (`0–3600`; `0` disables) | -| `SocketTimeoutSeconds` | int | `30` | **yes** | Socket read/write timeout (`5–600`) | +| `SocketTimeoutSeconds` | int | `30` | **yes** | Socket read/write timeout and AMQP connection establishment (`RequestedConnectionTimeout`) (`5–600`) | | `RequestedChannelMax` | int | `2047` | **yes** | Requested channel limit per connection (`1–65535`) | | `RpcTimeoutSeconds` | int | `30` | **yes** | Client continuation/handshake timeout (`1–3600`) | -| `ConnectionBlockedTimeoutSeconds` | int | `30` | **yes** | Client connection timeout (`5–3600`; must be ≥ `RpcTimeoutSeconds`) | +| `ConnectionBlockedTimeoutSeconds` | int | `30` | **yes** | Maximum continuous broker-blocked duration before the connection generation is retired and multi-host reconnect runs (`5–3600`; must be ≥ `RpcTimeoutSeconds`). Not the AMQP dial timeout. | | `NetworkRecoveryIntervalSeconds` | int | `5` | **yes** | Client recovery-interval setting (`1–3600`; unused while automatic recovery is disabled) | | `PoolReconnectBaseDelayMs` | int | `250` | **yes** | Application reconnect base delay (`50–60000`) | | `PoolReconnectMaxDelayMs` | int | `30000` | **yes** | Application reconnect max delay (`50–300000`; ≥ base) | diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqConnection.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqConnection.cs index e1169d24..3511a5f6 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqConnection.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqConnection.cs @@ -33,6 +33,24 @@ internal interface IRabbitMqConnection : IAsyncDisposable /// event EventHandler? ConnectionLost; + /// + /// Raised when the broker blocks the connection (for example under resource alarm). + /// + /// + /// is the only production subscriber. The connection remains + /// open; the service owns the blocked-duration timeout and recovery decision. + /// + event EventHandler? ConnectionBlocked; + + /// + /// Raised when the broker clears a prior block on this connection. + /// + /// + /// is the only production subscriber. Cancels a pending + /// blocked-duration timeout for the current generation when the sender is still current. + /// + event EventHandler? ConnectionUnblocked; + /// /// Opens a short-lived channel that can declare exchanges, queues, and bindings. /// diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs index f095a876..52ee40a1 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs @@ -72,6 +72,12 @@ internal RabbitMqClientConnection(IConnection connection, string virtualHost, IL /// public event EventHandler? ConnectionLost; + /// + public event EventHandler? ConnectionBlocked; + + /// + public event EventHandler? ConnectionUnblocked; + /// public async Task CreateTopologyChannelAsync(CancellationToken cancellationToken) { @@ -222,23 +228,32 @@ private Task OnCallbackExceptionAsync(object sender, CallbackExceptionEventArgs return Task.CompletedTask; } - /// Logs the broker block reason. Does not pause channels or request recovery. + /// + /// Logs the broker block reason and raises . + /// Does not pause channels or request recovery; owns the timeout. + /// /// Unused. /// Broker block reason. /// A completed task. private Task OnBlockedAsync(object sender, ConnectionBlockedEventArgs eventArgs) { - RabbitMqLogMessages.ConnectionBlocked(_logger, eventArgs.Reason); + var reason = eventArgs.Reason ?? string.Empty; + RabbitMqLogMessages.ConnectionBlocked(_logger, reason); + ConnectionBlocked?.Invoke(this, new RabbitMqConnectionBlockedEventArgs(reason)); return Task.CompletedTask; } - /// Logs that the broker unblocked the connection. is unused. + /// + /// Logs that the broker unblocked the connection and raises . + /// is unused. + /// /// Unused. /// Unused client event args. /// A completed task. private Task OnUnblockedAsync(object sender, AsyncEventArgs eventArgs) { RabbitMqLogMessages.ConnectionUnblocked(_logger); + ConnectionUnblocked?.Invoke(this, EventArgs.Empty); return Task.CompletedTask; } } diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnectionFactory.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnectionFactory.cs index 3b152722..ebe29705 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnectionFactory.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnectionFactory.cs @@ -85,7 +85,9 @@ internal static ConnectionFactory CreateClientFactory( TopologyRecoveryEnabled = false, NetworkRecoveryInterval = TimeSpan.FromSeconds(options.NetworkRecoveryIntervalSeconds), RequestedHeartbeat = TimeSpan.FromSeconds(options.RequestedHeartbeatSeconds), - RequestedConnectionTimeout = TimeSpan.FromSeconds(options.ConnectionBlockedTimeoutSeconds), + // Handshake/dial timeout is independent of ConnectionBlockedTimeoutSeconds, which + // RabbitMqService uses as the maximum continuous broker-blocked duration. + RequestedConnectionTimeout = TimeSpan.FromSeconds(options.SocketTimeoutSeconds), ContinuationTimeout = TimeSpan.FromSeconds(options.RpcTimeoutSeconds), HandshakeContinuationTimeout = TimeSpan.FromSeconds(options.RpcTimeoutSeconds), SocketReadTimeout = TimeSpan.FromSeconds(options.SocketTimeoutSeconds), diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqConnectionBlockedEventArgs.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqConnectionBlockedEventArgs.cs new file mode 100644 index 00000000..2768ce32 --- /dev/null +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqConnectionBlockedEventArgs.cs @@ -0,0 +1,6 @@ +namespace VectorNNTPD.Common.Messaging.RabbitMq +{ + /// Broker block details for a single AMQP connection. + /// Block reason supplied by the broker. + internal sealed record RabbitMqConnectionBlockedEventArgs(string Reason); +} diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqLogMessages.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqLogMessages.cs index 2a1e79d6..3f86326b 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqLogMessages.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqLogMessages.cs @@ -107,6 +107,24 @@ internal static partial void ConnectionLost( Message = "RabbitMQ broker unblocked the connection")] internal static partial void ConnectionUnblocked(ILogger logger); + /// + /// Logs event 2807 (warning): a connection remained broker-blocked through + /// ConnectionBlockedTimeoutSeconds; recovery is requested for that generation. + /// + /// Logger that receives the event. + /// Connection generation that stayed blocked. + /// Configured blocked-duration timeout that elapsed. + /// Block reason from the start of this blocked episode. + [LoggerMessage( + EventId = 2807, + Level = LogLevel.Warning, + Message = "RabbitMQ connection generation {Generation} remained blocked for {TimeoutSeconds}s ({Reason}); requesting recovery")] + internal static partial void ConnectionBlockedTimeout( + ILogger logger, + long Generation, + int TimeoutSeconds, + string Reason); + /// Logs event 2808 (information): a reconnect attempt is about to wait. /// Logger that receives the event. /// One-based attempt number for the current recovery. diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs index 87b7e339..bb127a60 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqOptions.cs @@ -59,8 +59,9 @@ public sealed class RabbitMqOptions public int? RpcTimeoutSeconds { get; set; } = 30; /// - /// Maximum duration, in seconds, that a broker-blocked RabbitMQ connection may remain - /// blocked before recovery evaluation. Also used as the client connection timeout. + /// Maximum continuous duration, in seconds, that a broker-blocked RabbitMQ connection may + /// remain blocked before retires that generation and reconnects. + /// Not the AMQP handshake timeout; handshake uses . /// public int? ConnectionBlockedTimeoutSeconds { get; set; } = 30; @@ -197,7 +198,8 @@ public sealed class RabbitMqOptions public int? RequestedHeartbeatSeconds { get; set; } = 60; /// - /// RabbitMQ socket operation timeout, in seconds, for low-level network I/O. + /// RabbitMQ socket read/write timeout and AMQP connection-establishment timeout + /// (RequestedConnectionTimeout), in seconds. /// public int? SocketTimeoutSeconds { get; set; } = 30; diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqRuntimeOptions.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqRuntimeOptions.cs index 6aecbcdb..9048ef36 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqRuntimeOptions.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqRuntimeOptions.cs @@ -11,7 +11,7 @@ namespace VectorNNTPD.Common.Messaging.RabbitMq /// Whether TLS is enabled for RabbitMQ connectivity. /// Maximum channel lease timeout in seconds. /// Maximum RabbitMQ RPC timeout in seconds. - /// Maximum blocked-connection timeout in seconds. + /// Maximum continuous broker-blocked duration before generation retirement, in seconds. /// Configured RabbitMQ channel pool size. /// Configured minimum pooled RabbitMQ connections. /// Configured maximum pooled RabbitMQ connections. @@ -27,7 +27,7 @@ namespace VectorNNTPD.Common.Messaging.RabbitMq /// Degraded health threshold ratio. /// Consecutive unhealthy evaluations threshold. /// Requested AMQP heartbeat in seconds. - /// Socket read/write timeout in seconds. + /// Socket read/write and AMQP connection-establishment timeout in seconds. /// Requested channel max for AMQP negotiation. /// Optional RabbitMQ consumer prefetch count. /// Optional AMQP CorrelationId used to gate temporary payload diagnostics. diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs index c58a06aa..7b3eb9f4 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs @@ -13,7 +13,10 @@ namespace VectorNNTPD.Common.Messaging.RabbitMq /// fails if an initial usable connection cannot be established, and it /// disposes any partial connect before returning. After start, a lost connection is not terminal: /// one watch task reconnects with backoff until a new generation is installed or shutdown cancels - /// the loop. RabbitMQ.Client automatic recovery is disabled; this service is the only lifecycle owner. + /// the loop. A connection that remains broker-blocked longer than + /// requests the same recovery + /// path while the AMQP session may still be open. RabbitMQ.Client automatic recovery is disabled; + /// this service is the only lifecycle owner. /// /// /// Callers obtain a generation snapshot with . A handle does not @@ -80,6 +83,12 @@ internal sealed class RabbitMqService : IRabbitMqService, IApplicationService, I /// private bool _stopping; + /// + /// Cancels the pending blocked-duration timeout for the current generation. + /// Owned under ; replaced only when a new blocked episode starts. + /// + private CancellationTokenSource? _blockedTimeoutCts; + /// /// Creates the process connection owner, using for reconnect delays. /// @@ -252,6 +261,7 @@ public async ValueTask DisposeAsync() lock (_gate) { _stopping = true; + CancelBlockedTimeoutLocked(); } await _runCts.CancelAsync().ConfigureAwait(false); @@ -417,7 +427,7 @@ private async Task ConnectAndInstallAsync( var previous = live.Replaced; if (previous is not null) { - previous.Connection.ConnectionLost -= OnConnectionLost; + DetachConnectionHandlers(previous.Connection); await DisposeConnectionQuietlyAsync(previous.Connection).ConfigureAwait(false); } @@ -531,29 +541,31 @@ private static string DescribeConnectionReplacedSubscriber( } /// - /// Subscribes to connection loss, then publishes as the next generation. + /// Subscribes to connection lifecycle events, then publishes as the next generation. /// /// Open connection to make current. The caller must not dispose it after a successful return. /// The installed generation and the previous generation, if one was still published. - /// The service is stopping. The loss handler is removed and the connection is not published. + /// The service is stopping. Handlers are removed and the connection is not published. /// /// Does not dispose or the previous generation. The caller disposes the previous one. - /// A close that races this method is still observed because the handler is attached before the publish. + /// A close that races this method is still observed because the handlers are attached before the publish. + /// Any pending blocked-duration timeout from a prior generation is cancelled when a new generation is published. /// private PublishedConnection Publish(IRabbitMqConnection connection) { - // Subscribe before making the instance current so a close that races the - // publish is observed. OnConnectionLost still ignores a sender that is not current. - connection.ConnectionLost += OnConnectionLost; + // Subscribe before making the instance current so a close or block that races the + // publish is observed. Handlers still ignore a sender that is not current. + AttachConnectionHandlers(connection); lock (_gate) { if (_stopping) { - connection.ConnectionLost -= OnConnectionLost; + DetachConnectionHandlers(connection); throw new InvalidOperationException("RabbitMQ service is stopping."); } + CancelBlockedTimeoutLocked(); var previous = _current; _generation++; var current = new LiveConnection(connection, _generation); @@ -563,6 +575,24 @@ private PublishedConnection Publish(IRabbitMqConnection connection) } } + /// Attaches loss, blocked, and unblocked handlers for a connection about to be published. + /// Connection whose events this service will observe. + private void AttachConnectionHandlers(IRabbitMqConnection connection) + { + connection.ConnectionLost += OnConnectionLost; + connection.ConnectionBlocked += OnConnectionBlocked; + connection.ConnectionUnblocked += OnConnectionUnblocked; + } + + /// Detaches loss, blocked, and unblocked handlers from a retired or unpublished connection. + /// Connection that must no longer raise recovery work into this service. + private void DetachConnectionHandlers(IRabbitMqConnection connection) + { + connection.ConnectionLost -= OnConnectionLost; + connection.ConnectionBlocked -= OnConnectionBlocked; + connection.ConnectionUnblocked -= OnConnectionUnblocked; + } + /// /// Logs loss of the current connection and requests recovery. A retired sender is ignored. /// @@ -582,6 +612,7 @@ private void OnConnectionLost(object? sender, RabbitMqConnectionLostEventArgs ev return; } + CancelBlockedTimeoutLocked(); generation = current.Generation; } @@ -594,6 +625,177 @@ private void OnConnectionLost(object? sender, RabbitMqConnectionLostEventArgs ev RequestRecovery(); } + /// + /// Starts a single blocked-duration timeout for the current connection generation. + /// Duplicate blocked notifications for the same episode are ignored. + /// + /// Connection that raised the block. Ignored unless it is still . + /// Broker block reason preserved for timeout logging. + /// + /// Does not block the AMQP callback thread on I/O. The timeout uses + /// elapsed-duration delay and requests recovery through only if + /// that same generation is still current and still blocked when the delay completes. + /// + private void OnConnectionBlocked(object? sender, RabbitMqConnectionBlockedEventArgs eventArgs) + { + CancellationTokenSource cts; + long generation; + IRabbitMqConnection connection; + int timeoutSeconds; + lock (_gate) + { + if (_stopping + || _runtime is null + || _current is not { } current + || !ReferenceEquals(current.Connection, sender)) + { + return; + } + + // One timer per continuous blocked episode; duplicates must not stack. + if (_blockedTimeoutCts is not null) + { + return; + } + + timeoutSeconds = _runtime.ConnectionBlockedTimeoutSeconds; + generation = current.Generation; + connection = current.Connection; + cts = CancellationTokenSource.CreateLinkedTokenSource(_runCts.Token); + _blockedTimeoutCts = cts; + } + + _ = ObserveBlockedTimeoutAsync(generation, connection, timeoutSeconds, eventArgs.Reason, cts); + } + + /// + /// Cancels a pending blocked-duration timeout when the current generation unblocks. + /// + /// Connection that raised the unblock. Ignored unless it is still . + /// Unused. + private void OnConnectionUnblocked(object? sender, EventArgs eventArgs) + { + lock (_gate) + { + if (_stopping + || _current is not { } current + || !ReferenceEquals(current.Connection, sender)) + { + return; + } + + CancelBlockedTimeoutLocked(); + } + } + + /// + /// Waits for the configured blocked duration, then requests recovery when that generation is still current. + /// + /// Generation that started the blocked episode. + /// Connection instance that must still be current at expiry. + /// Configured continuous blocked-duration limit. + /// Block reason captured at episode start. + /// Episode cancellation source; must match at expiry. + /// + /// + /// Linearization versus unblock/lost/shutdown: after the delay completes, recovery is claimed only by + /// clearing under while it still references + /// . (unblock, lost, retire, publish, + /// dispose) nulls that field first, so a timeout that loses the race returns without + /// . A timeout that wins cannot be undone by a later unblock: the CTS + /// slot is empty, and is gated again on . + /// + /// + /// Delay completion alone is not the claim; an unblock that cancels between completion and this lock + /// still wins. Stale senders never reach for another + /// generation because blocked/unblocked handlers require ReferenceEquals with . + /// + /// + private async Task ObserveBlockedTimeoutAsync( + long generation, + IRabbitMqConnection connection, + int timeoutSeconds, + string reason, + CancellationTokenSource cts) + { + try + { + await Task.Delay(TimeSpan.FromSeconds(timeoutSeconds), _timeProvider, cts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + // Includes ObjectDisposedException if the CTS was disposed between schedule and Token use. + RabbitMqLogMessages.CallbackException(_logger, ex.Message); + return; + } + + var shouldRecover = false; + lock (_gate) + { + // Claim point: only the CTS still installed for this episode may clear state or recover. + if (!ReferenceEquals(_blockedTimeoutCts, cts)) + { + return; + } + + _blockedTimeoutCts = null; + shouldRecover = !_stopping + && _current is { } current + && current.Generation == generation + && ReferenceEquals(current.Connection, connection); + } + + try + { + cts.Dispose(); + } + catch (ObjectDisposedException) + { + } + + if (!shouldRecover) + { + return; + } + + RabbitMqLogMessages.ConnectionBlockedTimeout(_logger, generation, timeoutSeconds, reason); + RequestRecovery(); + } + + /// + /// Cancels and clears under . + /// + private void CancelBlockedTimeoutLocked() + { + var cts = _blockedTimeoutCts; + _blockedTimeoutCts = null; + if (cts is null) + { + return; + } + + try + { + cts.Cancel(); + } + catch (ObjectDisposedException) + { + } + + try + { + cts.Dispose(); + } + catch (ObjectDisposedException) + { + } + } + /// /// Completes the current recovery signal. Concurrent calls collapse onto the same signal. No-op while stopping. /// @@ -643,7 +845,8 @@ private async Task WaitForRecoveryRequestAsync(CancellationToken cancellationTok } /// - /// Unpublishes the current connection, detaches its loss handler, and disposes it. No-op when none is current. + /// Unpublishes the current connection, cancels any blocked-duration timeout, detaches handlers, and disposes it. + /// No-op when none is current. /// private async Task RetireCurrentAsync() { @@ -652,6 +855,7 @@ private async Task RetireCurrentAsync() { current = _current; _current = null; + CancelBlockedTimeoutLocked(); } if (current is null) @@ -659,7 +863,7 @@ private async Task RetireCurrentAsync() return; } - current.Connection.ConnectionLost -= OnConnectionLost; + DetachConnectionHandlers(current.Connection); await DisposeConnectionQuietlyAsync(current.Connection).ConfigureAwait(false); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqBlockedConnectionTimeoutTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqBlockedConnectionTimeoutTests.cs new file mode 100644 index 00000000..1fd32d49 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqBlockedConnectionTimeoutTests.cs @@ -0,0 +1,577 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.RabbitMq +{ + /// + /// Locks broker-blocked duration timeout retirement onto the existing multi-host reconnect path. + /// + public sealed class RabbitMqBlockedConnectionTimeoutTests + { + /// + /// A connection that stays blocked through the configured timeout is retired and replaced. + /// + [Fact] + public async Task Persistent_block_retires_generation_and_reconnects() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var logger = new CollectingLogger(); + await using var service = CreateService( + factory, + logger, + new ImmediateDelayTimeProvider(), + blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateBlocked("disk alarm"); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + + Assert.Equal(2, service.ConnectionGeneration); + Assert.Equal(1, first.DisposeCount); + Assert.True(service.IsReady); + Assert.Contains( + logger.Entries, + static entry => entry.EventId.Id == 2807 + && entry.Message.Contains("generation 1", StringComparison.Ordinal) + && entry.Message.Contains("disk alarm", StringComparison.Ordinal)); + Assert.DoesNotContain(logger.Entries, static entry => entry.EventId.Id == 2803); + } + + /// + /// Unblocking before the timeout cancels recovery for that episode. + /// + [Fact] + public async Task Unblock_before_timeout_cancels_recovery() + { + var clock = new ControllableTimeProvider(); + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + await using var service = CreateService(factory, timeProvider: clock, blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + + first.SimulateBlocked(); + clock.Advance(TimeSpan.FromSeconds(4)); + first.SimulateUnblocked(); + clock.Advance(TimeSpan.FromSeconds(10)); + await Task.Yield(); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, service.ConnectionGeneration); + Assert.Equal(0, first.DisposeCount); + Assert.Same(first, factory.LastConnection); + } + + /// + /// After the timeout claims recovery (event 2807), a later unblock cannot prevent reconnect. + /// + [Fact] + public async Task Timeout_claim_then_unblock_still_recovers() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var logger = new CollectingLogger(); + await using var service = CreateService( + factory, + logger, + new ImmediateDelayTimeProvider(), + blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateBlocked("claimed-then-unblock"); + await WaitForLogEventAsync(logger, eventId: 2807); + + first.SimulateUnblocked(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + + Assert.Equal(2, service.ConnectionGeneration); + Assert.Equal(1, first.DisposeCount); + Assert.Contains( + logger.Entries, + static entry => entry.EventId.Id == 2807 + && entry.Message.Contains("claimed-then-unblock", StringComparison.Ordinal)); + } + + /// + /// An unblock from a retired connection cannot cancel a blocked-timeout timer on the replacement. + /// + [Fact] + public async Task Stale_unblock_does_not_cancel_replacement_blocked_timer() + { + var clock = new ControllableTimeProvider(); + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + await using var service = CreateService(factory, timeProvider: clock, blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + await WaitForTaskWhileAdvancingAsync(secondConnected.Task, clock); + var second = factory.LastConnection!; + Assert.Equal(2, service.ConnectionGeneration); + + var thirdConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = thirdConnected; + second.SimulateBlocked("replacement-block"); + first.SimulateUnblocked(); + clock.Advance(TimeSpan.FromSeconds(4)); + await Task.Yield(); + Assert.Equal(2, factory.ConnectCount); + + await WaitForTaskWhileAdvancingAsync(thirdConnected.Task, clock); + Assert.Equal(3, service.ConnectionGeneration); + Assert.Equal(1, second.DisposeCount); + } + + /// + /// A new blocked episode after unblock gets a fresh timeout window. + /// + [Fact] + public async Task New_block_episode_starts_a_fresh_timeout() + { + var clock = new ControllableTimeProvider(); + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + await using var service = CreateService(factory, timeProvider: clock, blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + first.SimulateBlocked("episode-1"); + clock.Advance(TimeSpan.FromSeconds(4)); + first.SimulateUnblocked(); + first.SimulateBlocked("episode-2"); + clock.Advance(TimeSpan.FromSeconds(4)); + await Task.Yield(); + Assert.Equal(1, factory.ConnectCount); + + factory.Connected = secondConnected; + await WaitForTaskWhileAdvancingAsync(secondConnected.Task, clock); + + Assert.Equal(2, service.ConnectionGeneration); + } + + /// + /// Duplicate blocked notifications for one episode schedule a single timeout. + /// + [Fact] + public async Task Duplicate_blocked_notifications_do_not_stack_timers() + { + var clock = new ControllableTimeProvider(); + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + await using var service = CreateService(factory, timeProvider: clock, blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateBlocked("first"); + first.SimulateBlocked("duplicate"); + clock.Advance(TimeSpan.FromSeconds(4)); + await Task.Yield(); + Assert.Equal(1, factory.ConnectCount); + + await WaitForTaskWhileAdvancingAsync(secondConnected.Task, clock); + Assert.Equal(2, factory.ConnectCount); + } + + /// + /// Ordinary shutdown racing a blocked timeout still produces one recovery transition. + /// + [Fact] + public async Task Shutdown_racing_blocked_timeout_yields_one_recovery() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + await using var service = CreateService( + factory, + timeProvider: new ImmediateDelayTimeProvider(), + blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateBlocked(); + first.SimulateLost(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + + Assert.Equal(2, factory.ConnectCount); + Assert.Equal(2, service.ConnectionGeneration); + Assert.Equal(1, first.DisposeCount); + } + + /// + /// A stale blocked timeout from a retired generation cannot retire its replacement. + /// + [Fact] + public async Task Stale_generation_blocked_timeout_does_not_retire_replacement() + { + var clock = new ControllableTimeProvider(); + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + await using var service = CreateService(factory, timeProvider: clock, blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateBlocked("stale-episode"); + first.SimulateLost(); + await WaitForTaskWhileAdvancingAsync(secondConnected.Task, clock); + var second = factory.LastConnection!; + Assert.Equal(2, service.ConnectionGeneration); + + clock.Advance(TimeSpan.FromSeconds(10)); + await Task.Yield(); + + Assert.Equal(2, factory.ConnectCount); + Assert.Equal(2, service.ConnectionGeneration); + Assert.Same(second, factory.LastConnection); + Assert.Equal(0, second.DisposeCount); + } + + /// + /// Stopping the service cancels a pending blocked timeout without starting reconnect. + /// + [Fact] + public async Task Service_stop_cancels_pending_blocked_timeout() + { + var clock = new ControllableTimeProvider(); + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var service = CreateService(factory, timeProvider: clock, blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + + first.SimulateBlocked(); + await service.DisposeAsync(); + clock.Advance(TimeSpan.FromSeconds(30)); + await Task.Yield(); + + Assert.Equal(1, factory.ConnectCount); + Assert.Equal(1, first.DisposeCount); + } + + /// + /// Blocked-timeout recovery uses ConnectAsync with the configured multi-host options list. + /// + [Fact] + public async Task Timeout_recovery_reconnects_with_configured_hosts() + { + var factory = new RecordingHostsConnectionFactory(); + var options = BackFillerTestOptions.CreateValidRabbitMq(); + options.Hosts = ["broker-a.example.test", "broker-b.example.test"]; + options.PoolReconnectBaseDelayMs = 1; + options.PoolReconnectMaxDelayMs = 1; + options.ConnectionBlockedTimeoutSeconds = 5; + await using var service = new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.BackFillerService:blocked-hosts"), + NullLogger.Instance, + new ImmediateDelayTimeProvider()); + + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateBlocked(); + + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + + Assert.Equal(2, factory.ConnectCount); + Assert.Equal(2, factory.ConnectOptions.Count); + Assert.Equal( + ["broker-a.example.test", "broker-b.example.test"], + factory.ConnectOptions[1].Hosts!); + } + + /// + /// A delayed blocked callback from a retired connection is ignored by the active generation. + /// + [Fact] + public async Task Stale_blocked_callback_does_not_start_timer_on_replacement() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + await using var service = CreateService( + factory, + timeProvider: new ImmediateDelayTimeProvider(), + blockedTimeoutSeconds: 5); + await service.StartAsync(CancellationToken.None); + var first = factory.LastConnection!; + var secondConnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + factory.Connected = secondConnected; + + first.SimulateLost(); + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await secondConnected.Task.WaitAsync(safety.Token); + var second = factory.LastConnection!; + + first.SimulateBlocked("from-retired"); + await Task.Yield(); + + Assert.Equal(2, factory.ConnectCount); + Assert.Same(second, factory.LastConnection); + Assert.Equal(0, second.DisposeCount); + } + + /// + /// Advances a controllable clock until completes or the safety budget expires. + /// + private static async Task WaitForTaskWhileAdvancingAsync(Task task, ControllableTimeProvider clock) + { + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (!task.IsCompleted) + { + clock.Advance(TimeSpan.FromSeconds(1)); + await Task.Yield(); + safety.Token.ThrowIfCancellationRequested(); + } + + await task.WaitAsync(safety.Token); + } + + /// + /// Waits until records or the safety budget expires. + /// + private static async Task WaitForLogEventAsync(CollectingLogger logger, int eventId) + { + using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + while (!logger.Entries.Any(entry => entry.EventId.Id == eventId)) + { + await Task.Yield(); + safety.Token.ThrowIfCancellationRequested(); + } + } + + private static RabbitMqService CreateService( + IRabbitMqConnectionFactory factory, + ILogger? logger = null, + TimeProvider? timeProvider = null, + int blockedTimeoutSeconds = 30) + { + var options = BackFillerTestOptions.CreateValidRabbitMq(); + options.PoolReconnectBaseDelayMs = 1; + options.PoolReconnectMaxDelayMs = 1; + options.ConnectionBlockedTimeoutSeconds = blockedTimeoutSeconds; + return new RabbitMqService( + factory, + Options.Create(options), + new DelegateRabbitMqConnectionNameProvider(() => "VectorNNTPD.BackFillerService:blocked-timeout"), + logger ?? NullLogger.Instance, + timeProvider ?? TimeProvider.System); + } + + /// + /// Records options passed to each connect so multi-host failover inputs stay observable. + /// + private sealed class RecordingHostsConnectionFactory : IRabbitMqConnectionFactory + { + private readonly FakeBackFillerRabbitMqConnectionFactory _inner = new(); + + internal int ConnectCount => _inner.ConnectCount; + + internal FakeBackFillerRabbitMqConnection? LastConnection => _inner.LastConnection; + + internal TaskCompletionSource? Connected + { + get => _inner.Connected; + set => _inner.Connected = value; + } + + internal List ConnectOptions { get; } = []; + + public Task ConnectAsync( + RabbitMqOptions options, + string connectionName, + CancellationToken cancellationToken) + { + ConnectOptions.Add(options); + return _inner.ConnectAsync(options, connectionName, cancellationToken); + } + } + + /// + /// Completes every immediately. + /// + private sealed class ImmediateDelayTimeProvider : TimeProvider + { + /// + public override ITimer CreateTimer( + TimerCallback callback, + object? state, + TimeSpan dueTime, + TimeSpan period) + { + if (dueTime != Timeout.InfiniteTimeSpan) + { + callback(state); + } + + return new NoOpTimer(); + } + + private sealed class NoOpTimer : ITimer + { + public bool Change(TimeSpan dueTime, TimeSpan period) => true; + + public void Dispose() + { + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + } + + /// + /// Virtual clock that queues timers until . + /// + private sealed class ControllableTimeProvider : TimeProvider + { + private readonly Lock _gate = new(); + private TimeSpan _elapsed; + private readonly List _timers = []; + + /// + public override DateTimeOffset GetUtcNow() => DateTimeOffset.UnixEpoch + _elapsed; + + /// + public override long GetTimestamp() => _elapsed.Ticks; + + /// + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + /// Advances the virtual clock and fires due one-shot timers. + /// Non-negative duration to add to the virtual clock. + public void Advance(TimeSpan delta) + { + ArgumentOutOfRangeException.ThrowIfLessThan(delta, TimeSpan.Zero); + + List<(TimerCallback Callback, object? State)> due = []; + lock (_gate) + { + _elapsed += delta; + for (var i = _timers.Count - 1; i >= 0; i--) + { + var timer = _timers[i]; + if (timer.Cancelled || timer.DueAt > _elapsed) + { + continue; + } + + due.Add((timer.Callback, timer.State)); + _timers.RemoveAt(i); + } + } + + foreach (var (callback, state) in due) + { + callback(state); + } + } + + /// + public override ITimer CreateTimer( + TimerCallback callback, + object? state, + TimeSpan dueTime, + TimeSpan period) + { + ArgumentNullException.ThrowIfNull(callback); + var timer = new PendingTimer(this, callback, state); + timer.Change(dueTime, period); + return timer; + } + + private void Schedule(PendingTimer timer, TimeSpan dueTime) + { + lock (_gate) + { + _timers.Remove(timer); + if (dueTime == Timeout.InfiniteTimeSpan) + { + timer.Cancelled = true; + return; + } + + timer.Cancelled = false; + timer.DueAt = dueTime <= TimeSpan.Zero ? _elapsed : _elapsed + dueTime; + if (timer.DueAt <= _elapsed) + { + timer.FireImmediately = true; + return; + } + + _timers.Add(timer); + } + } + + private void Cancel(PendingTimer timer) + { + lock (_gate) + { + timer.Cancelled = true; + _timers.Remove(timer); + } + } + + private sealed class PendingTimer : ITimer + { + private readonly ControllableTimeProvider _owner; + + internal PendingTimer(ControllableTimeProvider owner, TimerCallback callback, object? state) + { + _owner = owner; + Callback = callback; + State = state; + } + + internal TimerCallback Callback { get; } + + internal object? State { get; } + + internal TimeSpan DueAt { get; set; } + + internal bool Cancelled { get; set; } + + internal bool FireImmediately { get; set; } + + public bool Change(TimeSpan dueTime, TimeSpan period) + { + _ = period; + FireImmediately = false; + _owner.Schedule(this, dueTime); + if (FireImmediately && !Cancelled) + { + FireImmediately = false; + Callback(State); + } + + return true; + } + + public void Dispose() => _owner.Cancel(this); + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs index 73f1d484..6e11c7fe 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs @@ -160,6 +160,12 @@ internal FakeBackFillerRabbitMqConnection(string host, int port, string virtualH /// Raised by ; not wired to a real broker. public event EventHandler? ConnectionLost; + /// Raised by ; not wired to a real broker. + public event EventHandler? ConnectionBlocked; + + /// Raised by ; not wired to a real broker. + public event EventHandler? ConnectionUnblocked; + /// Manual-ack channels created through this connection in order. public List Channels { get; } = []; @@ -302,6 +308,19 @@ internal void SimulateLost( ConnectionLost?.Invoke(this, new RabbitMqConnectionLostEventArgs(replyCode, replyText, initiator)); } + /// Raises while leaving unchanged. + /// Block reason delivered to subscribers. + internal void SimulateBlocked(string reason = "low on disk") + { + ConnectionBlocked?.Invoke(this, new RabbitMqConnectionBlockedEventArgs(reason)); + } + + /// Raises while leaving unchanged. + internal void SimulateUnblocked() + { + ConnectionUnblocked?.Invoke(this, EventArgs.Empty); + } + /// When set, thrown after dispose gates complete and the dispose count is incremented. public Exception? DisposeException { get; set; } diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqClientTlsEndpointTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqClientTlsEndpointTests.cs index feee5bdb..dc08fb6f 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqClientTlsEndpointTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqClientTlsEndpointTests.cs @@ -126,6 +126,28 @@ public void CreateClientFactory_does_not_pin_tls_server_name_to_first_host() Assert.True(string.IsNullOrEmpty(factory.Ssl.ServerName)); } + /// + /// AMQP connection establishment uses SocketTimeoutSeconds, not the blocked-duration setting. + /// + [Fact] + public void CreateClientFactory_uses_socket_timeout_for_requested_connection_timeout() + { + var options = RabbitMqOptionsValidatorTests.CreateValidConnectivityOnly(); + options.SocketTimeoutSeconds = 17; + options.ConnectionBlockedTimeoutSeconds = 45; + options.RpcTimeoutSeconds = 30; + var factory = RabbitMqClientConnectionFactory.CreateClientFactory( + options.ToRuntimeOptions(), + "VectorNNTPD.Common.Tests:timeouts"); + + Assert.Equal(TimeSpan.FromSeconds(17), factory.RequestedConnectionTimeout); + Assert.Equal(TimeSpan.FromSeconds(17), factory.SocketReadTimeout); + Assert.Equal(TimeSpan.FromSeconds(17), factory.SocketWriteTimeout); + Assert.NotEqual( + TimeSpan.FromSeconds(45), + factory.RequestedConnectionTimeout); + } + /// /// TLS-disabled factory keeps SSL disabled with protocol defaults unchanged. /// diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs index 050c873a..d69c6497 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqLogEventSinkTests.cs @@ -860,6 +860,16 @@ internal void NotePublication() /// public event EventHandler? ConnectionLost; + /// + /// Raised by with the supplied broker block reason. + /// + public event EventHandler? ConnectionBlocked; + + /// + /// Raised by . + /// + public event EventHandler? ConnectionUnblocked; + /// /// Sets to and invokes when a handler is attached. /// @@ -869,6 +879,23 @@ public void SimulateLost() ConnectionLost?.Invoke(this, new RabbitMqConnectionLostEventArgs(320, "lost", "Peer")); } + /// + /// Invokes without changing . + /// + /// Block reason delivered to subscribers. + public void SimulateBlocked(string reason = "low on disk") + { + ConnectionBlocked?.Invoke(this, new RabbitMqConnectionBlockedEventArgs(reason)); + } + + /// + /// Invokes without changing . + /// + public void SimulateUnblocked() + { + ConnectionUnblocked?.Invoke(this, EventArgs.Empty); + } + /// /// Creates a for and appends it to . /// diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs index 4b96b71b..0e4e8878 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqOptionsValidatorTests.cs @@ -23,6 +23,8 @@ public void Defaults_MatchSharedContract() Assert.Equal(1024, options.WorkRequestMaxPayloadBytes); Assert.Equal(60, options.ChannelLeaseTimeoutSeconds); Assert.Equal(30, options.RpcTimeoutSeconds); + Assert.Equal(30, options.ConnectionBlockedTimeoutSeconds); + Assert.Equal(30, options.SocketTimeoutSeconds); Assert.Empty(options.Hosts!); Assert.Equal("/", options.VirtualHost); Assert.True(options.EnableSsl); @@ -31,6 +33,41 @@ public void Defaults_MatchSharedContract() Assert.Null(options.Management.BaseUrl); } + /// + /// ConnectionBlockedTimeoutSeconds below the validated minimum fails startup validation. + /// + [Fact] + public void Validate_Fails_WhenConnectionBlockedTimeoutBelowMinimum() + { + var options = CreateValidConnectivityOnly(); + options.ConnectionBlockedTimeoutSeconds = 4; + options.RpcTimeoutSeconds = 1; + var result = new RabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static failure => failure.Contains("ConnectionBlockedTimeoutSeconds", StringComparison.Ordinal) + && failure.Contains("between 5 and 3600", StringComparison.Ordinal)); + } + + /// + /// ConnectionBlockedTimeoutSeconds must remain at least RpcTimeoutSeconds. + /// + [Fact] + public void Validate_Fails_WhenConnectionBlockedTimeoutLessThanRpcTimeout() + { + var options = CreateValidConnectivityOnly(); + options.RpcTimeoutSeconds = 60; + options.ConnectionBlockedTimeoutSeconds = 30; + var result = new RabbitMqOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static failure => failure.Contains( + "ConnectionBlockedTimeoutSeconds must be greater than or equal to RpcTimeoutSeconds", + StringComparison.Ordinal)); + } + /// /// Asserts that connectivity-only options with a host and SSL disabled pass validation when management is unset. /// From a98b21372c85c93a99e450cb28bb281de4d3fc2d Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 21 Oct 2026 19:40:18 +0200 Subject: [PATCH 2/3] Log RabbitMQ blocked-timeout CTS disposal races --- .../Messaging/RabbitMq/RabbitMqLogMessages.cs | 16 ++++++++++++++++ .../Messaging/RabbitMq/RabbitMqService.cs | 9 ++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqLogMessages.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqLogMessages.cs index 3f86326b..5d8d4a4a 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqLogMessages.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqLogMessages.cs @@ -229,5 +229,21 @@ internal static partial void ConnectionReplacedHandlerFailed( Exception exception, long Generation, string Subscriber); + + /// + /// Logs event 2821 (debug): blocked-timeout CTS cancel or dispose raced with another cleanup path. + /// Cleanup continues; this is not a recovery failure. + /// + /// Logger that receives the event. + /// The from the raced CTS. + /// Cancel or Dispose. + [LoggerMessage( + EventId = 2821, + Level = LogLevel.Debug, + Message = "RabbitMQ blocked-timeout CTS was already disposed during {Operation}")] + internal static partial void BlockedTimeoutCtsAlreadyDisposed( + ILogger logger, + Exception exception, + string Operation); } } diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs index 7b3eb9f4..941138d5 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqService.cs @@ -754,8 +754,9 @@ await Task.Delay(TimeSpan.FromSeconds(timeoutSeconds), _timeProvider, cts.Token) { cts.Dispose(); } - catch (ObjectDisposedException) + catch (ObjectDisposedException ex) { + RabbitMqLogMessages.BlockedTimeoutCtsAlreadyDisposed(_logger, ex, "Dispose"); } if (!shouldRecover) @@ -783,16 +784,18 @@ private void CancelBlockedTimeoutLocked() { cts.Cancel(); } - catch (ObjectDisposedException) + catch (ObjectDisposedException ex) { + RabbitMqLogMessages.BlockedTimeoutCtsAlreadyDisposed(_logger, ex, "Cancel"); } try { cts.Dispose(); } - catch (ObjectDisposedException) + catch (ObjectDisposedException ex) { + RabbitMqLogMessages.BlockedTimeoutCtsAlreadyDisposed(_logger, ex, "Dispose"); } } From 5d137e2ee2430025421a5369fa7bce9a7f603465 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 21 Oct 2026 19:56:32 +0200 Subject: [PATCH 3/3] Stabilize NNTP shutdown deadline test on Linux --- .../Nntp/NntpShutdownDeadlineTests.cs | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs index c3092f84..e590d639 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs @@ -11,14 +11,29 @@ namespace VectorNNTPD.BackFillerService.Tests.Nntp /// Shared shutdown deadline across stalled sessions, providers, and one replacement drain. /// /// - /// The configured budget in the scale tests is 500 milliseconds. The accepted ceiling is 1200 milliseconds. - /// That is less than one standalone two-second QUIT allowance, and less than three sequential copies of the 500 millisecond budget. - /// A wall-clock assertion is only used to separate one shared budget from a budget that grows with session or provider count. + /// Scale pools use (500 ms) and keepAliveSeconds: 0, so replacement + /// spends essentially one shared + /// retirement deadline on stalled QUITs (keepalive drain is empty). A correct run finishes near that budget + /// plus scheduling. Regressing to per-session QuitAllowance (2 s) with + /// yields about two serial waves (~4 s) and also starts a QUIT on + /// every session. rejects the QuitAllowance stack; quitStarted bounds + /// reject per-session independent budgets that still finish under the ceiling. + /// is only hang detection for and does not replace + /// . /// public sealed class NntpShutdownDeadlineTests { - /// How long the scale tests allow one shared shutdown to run, including scheduling after the budget. - private static readonly TimeSpan SharedShutdownCeiling = TimeSpan.FromMilliseconds(1200); + /// + /// Acceptance bound for scale shutdown elapsed time: below one QuitAllowance (2 s), with room for + /// CancelAfter scheduling delay after a single shared . + /// + private static readonly TimeSpan SharedShutdownCeiling = TimeSpan.FromMilliseconds(1500); + + /// + /// Hang detection for . Wider than so a slow + /// but still shared-budget drain is judged by instead of . + /// + private static readonly TimeSpan MeasureHangTimeout = TimeSpan.FromSeconds(2); /// Budget captured once for the many-session and many-provider shutdown tests. private static readonly TimeSpan SharedBudget = TimeSpan.FromMilliseconds(500); @@ -61,7 +76,7 @@ public void Deadline_cap_is_local_to_the_value_that_was_captured() /// Later sessions do not each receive a fresh two-second QUIT allowance. /// /// - /// The budget is 500 milliseconds and the accepted ceiling is 1200 milliseconds. + /// The budget is 500 milliseconds and the accepted ceiling is 1500 milliseconds (below one QuitAllowance). /// Eight independent two-second QUIT attempts would exceed that ceiling. /// At most writes can be in progress. /// The shared deadline token cancels that wave, and a later session observes the same token before it can start QUIT. @@ -566,6 +581,11 @@ public async Task Independent_pool_dispose_does_not_reuse_another_pools_deadline /// /// A replacement drain of many stalled sessions uses one retirement grace, not a fresh two-second QUIT for each session. /// + /// + /// Primary shared-deadline proof: at most QUITs start, fewer than + /// sessions, and those writes cancel on the shared token. is the + /// secondary wall-clock bound against stacking per-session QuitAllowance (2 s) waves (~4 s for eight sessions). + /// [Fact] public async Task Replacement_of_many_stalled_sessions_shares_one_retirement_deadline() { @@ -583,11 +603,11 @@ public async Task Replacement_of_many_stalled_sessions_shares_one_retirement_dea var quitStarted = QuitStarts(servers); Report("replacement-sessions", SharedBudget, sessions, providers: 1, quitStarted, elapsed); - AssertWithinCeiling(elapsed); Assert.InRange(quitStarted, 1, NntpShutdownWork.Parallelism); Assert.True(quitStarted < sessions); Assert.Equal(quitStarted, CountQuit(logger)); AssertQuitCancelledByToken(logger.Messages, quitStarted); + AssertWithinCeiling(elapsed); Assert.All(servers, static server => Assert.Equal(1, server.ClientStreamDisposeCount)); Assert.Equal(0, pool.LiveSessionCount); } @@ -604,9 +624,9 @@ public async Task Replacement_of_many_stalled_sessions_shares_one_retirement_dea /// /// /// Wall-clock acceptance is one standalone two-second QUIT allowance, not . - /// Concurrent registry CancelAfter load can delay the replacement timer past 1200 ms without multiplying - /// session budgets or adopting the registry's ten-second deadline. Four serial two-second QUITs or a ten-second - /// shared registry budget would still fail this bound. + /// Concurrent registry CancelAfter load can delay the replacement timer past the shared scale ceiling without + /// multiplying session budgets or adopting the registry's ten-second deadline. Four serial two-second QUITs or a + /// ten-second shared registry budget would still fail this bound. /// [Fact] public async Task Replacement_deadline_stays_independent_of_registry_shutdown() @@ -897,18 +917,21 @@ private static void AssertWithinCeiling(TimeSpan elapsed) /// /// Starts and returns how long it ran. - /// Releases stalled QUIT gates when the call exceeds , then waits for the same task to finish. + /// Releases stalled QUIT gates when the call exceeds , then waits for the same task to finish. /// /// Starts the shutdown under test. Called once, after the clock starts. /// Servers whose QUIT gates must be released if shutdown does not finish. - /// Elapsed time of a shutdown that finished inside . + /// Elapsed time of a shutdown that finished inside . + /// + /// Callers that require the shared-budget proof still call on the returned elapsed time. + /// private static async Task MeasureAsync(Func shutdown, IReadOnlyList servers) { var started = Stopwatch.GetTimestamp(); var dispose = shutdown(); try { - await dispose.WaitAsync(SharedShutdownCeiling); + await dispose.WaitAsync(MeasureHangTimeout); return Stopwatch.GetElapsedTime(started); } catch