diff --git a/src/VectorNNTPD.BackFillerService/Listener/CacheListenerLogMessages.cs b/src/VectorNNTPD.BackFillerService/Listener/CacheListenerLogMessages.cs index 9c865689..f0ec6fde 100644 --- a/src/VectorNNTPD.BackFillerService/Listener/CacheListenerLogMessages.cs +++ b/src/VectorNNTPD.BackFillerService/Listener/CacheListenerLogMessages.cs @@ -188,5 +188,61 @@ internal static partial void Running( Level = LogLevel.Warning, Message = "Cache Listener accepted socket options failed remote={Remote} error={Error}")] internal static partial void AcceptedSocketOptionsFailed(ILogger logger, string Remote, string Error); + + /// + /// Written when fails while the listener is still running. + /// Rate-limited by the accept loop so a persistent fault cannot flood the log. + /// + /// Logger that receives the event. + /// name from the failure. + /// Consecutive accept failures on this listen socket since the last success. + /// Configured threshold that stops the accept loop. + [LoggerMessage( + EventId = 5416, + Level = LogLevel.Warning, + Message = "Cache Listener accept failed socketError={SocketError} consecutiveFailures={ConsecutiveFailures} maxConsecutiveFailures={MaxConsecutiveFailures}")] + internal static partial void AcceptFailed( + ILogger logger, + string SocketError, + int ConsecutiveFailures, + int MaxConsecutiveFailures); + + /// + /// Written when accept succeeds after one or more consecutive failures on the same listen socket. + /// + /// Logger that receives the event. + /// Consecutive failures cleared by this successful accept. + [LoggerMessage( + EventId = 5417, + Level = LogLevel.Information, + Message = "Cache Listener accept recovered afterFailures={RecoveredAfterFailures}")] + internal static partial void AcceptRecovered(ILogger logger, int RecoveredAfterFailures); + + /// + /// Written when consecutive accept failures reach the configured threshold and the accept loop faults. + /// + /// Logger that receives the event. + /// name from the last failure. + /// Consecutive failures that triggered the stop. + [LoggerMessage( + EventId = 5418, + Level = LogLevel.Error, + Message = "Cache Listener accept loop stopping after consecutiveFailures={ConsecutiveFailures} lastSocketError={SocketError}")] + internal static partial void AcceptLoopStopping( + ILogger logger, + int ConsecutiveFailures, + string SocketError); + + /// + /// Written when dispose observes a non-cancellation fault from the accept task. + /// Cleanup continues to the stopped state; the fault was already visible on Execution. + /// + /// Logger that receives the event. + /// Runtime exception type name from the accept-task fault. + [LoggerMessage( + EventId = 5419, + Level = LogLevel.Warning, + Message = "Cache Listener accept loop fault observed during dispose error={ErrorType}")] + internal static partial void AcceptLoopFaultDuringDispose(ILogger logger, string ErrorType); } } diff --git a/src/VectorNNTPD.BackFillerService/Listener/CacheListenerService.cs b/src/VectorNNTPD.BackFillerService/Listener/CacheListenerService.cs index e4189f6f..08abc322 100644 --- a/src/VectorNNTPD.BackFillerService/Listener/CacheListenerService.cs +++ b/src/VectorNNTPD.BackFillerService/Listener/CacheListenerService.cs @@ -102,6 +102,26 @@ internal sealed class CacheListenerService : IHostedService, IApplicationService /// Connections that passed and have not yet reached . private int _activeConnections; + /// + /// Delay before retrying after a transient + /// . Cancellation-aware via . + /// + internal static readonly TimeSpan AcceptFailureRetryDelay = TimeSpan.FromMilliseconds(250); + + /// + /// Consecutive accept failures on one listen socket before the accept loop faults. + /// A faulted is observed by + /// as unexpected termination + /// (host unhealthy / stop). + /// + internal const int MaxConsecutiveAcceptFailures = 5; + + /// + /// Minimum interval between Warning-level accept-failure logs for one accept loop. + /// The first failure, the threshold-crossing failure, and failures after this interval are logged. + /// + internal static readonly TimeSpan AcceptFailureLogInterval = TimeSpan.FromSeconds(5); + /// /// Invoked after admission and before on an accepted socket. /// Null in production. A or from this @@ -115,6 +135,18 @@ internal sealed class CacheListenerService : IHostedService, IApplicationService /// internal Action? AfterAcceptedSocketConfigured { get; set; } + /// + /// Replaces when non-null. Null in production. + /// Tests use this to inject deterministic accept failures without closing the listen socket. + /// + internal Func>? AcceptAsyncOverride { get; set; } + + /// + /// Replaces the post-failure accept backoff when non-null. Null in production uses + /// on . + /// + internal Func? AcceptFailureDelayAsync { get; set; } + /// Initializes the Listener service for isolated tests with a pre-ready certificate gate. /// Runtime snapshot. Must not be null. /// Certificate provider. Must not be null. @@ -338,8 +370,15 @@ public async Task StopAsync(CancellationToken cancellationToken) /// /// State moves to before cancel, then to /// after drain. from the - /// accept task is ignored. Any other accept-task exception propagates. Connection-task exceptions are - /// logged and ignored by so socket release and the stopped state still run. + /// accept task is ignored. Expected accept-task faults + /// ( after consecutive accept failures, + /// , , and + /// containing only those) are logged and ignored here so drain and the + /// stopped state still run; already exposed those faults to + /// . + /// Unexpected accept-task faults are retained, cleanup still reaches + /// , then the aggregate is rethrown so programming errors stay visible. + /// Connection-task exceptions are logged and ignored by . /// The run token source is disposed before the stopped state. /// public async ValueTask DisposeAsync() @@ -358,17 +397,11 @@ public async ValueTask DisposeAsync() await _runCts.CancelAsync().ConfigureAwait(false); CloseListenSockets(); + Exception? unexpectedAcceptFault = null; var accept = _acceptTask; if (accept is not null) { - try - { - await accept.ConfigureAwait(false); - } - catch (OperationCanceledException) - { - ShutdownObservations.Canceled(_logger, "cache listener accept loop"); - } + unexpectedAcceptFault = await ObserveAcceptTaskDuringDisposeAsync(accept).ConfigureAwait(false); } await DrainConnectionsAsync().ConfigureAwait(false); @@ -380,6 +413,131 @@ public async ValueTask DisposeAsync() } CacheListenerLogMessages.Stopped(_logger); + + if (unexpectedAcceptFault is not null) + { + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(unexpectedAcceptFault).Throw(); + } + } + + /// + /// Observes during dispose without a catch-all. Expected faults are logged; + /// unexpected aggregates are returned for rethrow after cleanup reaches . + /// + /// The accept-loop task published as . + /// An unexpected fault to rethrow after cleanup, or . + private async Task ObserveAcceptTaskDuringDisposeAsync(Task accept) + { + if (accept.IsCompleted) + { + return ClassifyCompletedAcceptTask(accept); + } + + try + { + await accept.ConfigureAwait(false); + return null; + } + catch (OperationCanceledException) + { + ShutdownObservations.Canceled(_logger, "cache listener accept loop"); + return null; + } + catch (ObjectDisposedException ex) + { + ObserveExpectedAcceptLoopFaultDuringDispose(ex); + return null; + } + catch (SocketException ex) + { + ObserveExpectedAcceptLoopFaultDuringDispose(ex); + return null; + } + catch (InvalidOperationException ex) + { + ObserveExpectedAcceptLoopFaultDuringDispose(ex); + return null; + } + catch (AggregateException ex) + { + if (IsExpectedAcceptLoopAggregate(ex)) + { + ObserveExpectedAcceptLoopFaultDuringDispose(ex); + return null; + } + + return ex; + } + } + + /// + /// Classifies an already-completed accept task using (always an aggregate when faulted). + /// + /// Completed accept-loop task. + /// Unexpected aggregate to rethrow after cleanup, or . + private Exception? ClassifyCompletedAcceptTask(Task accept) + { + if (accept.IsCanceled) + { + ShutdownObservations.Canceled(_logger, "cache listener accept loop"); + return null; + } + + if (!accept.IsFaulted) + { + return null; + } + + var fault = accept.Exception!; + if (IsExpectedAcceptLoopAggregate(fault)) + { + ObserveExpectedAcceptLoopFaultDuringDispose(fault); + return null; + } + + return fault; + } + + /// + /// Logs an expected accept-task fault observed while disposing. Terminal accept failures were already + /// logged by the accept loop; this records that dispose continued cleanup. + /// + /// Expected fault from . + private void ObserveExpectedAcceptLoopFaultDuringDispose(Exception exception) => + CacheListenerLogMessages.AcceptLoopFaultDuringDispose( + _logger, + exception.GetBaseException().GetType().Name); + + /// + /// Reports whether every flattened inner exception is an expected accept-loop fault type. + /// + /// Fault from via . + /// + /// when every inner is , + /// , , or + /// . + /// + private static bool IsExpectedAcceptLoopAggregate(AggregateException exception) + { + var inners = exception.Flatten().InnerExceptions; + if (inners.Count == 0) + { + return false; + } + + foreach (var inner in inners) + { + if (inner is not ( + InvalidOperationException + or ObjectDisposedException + or SocketException + or OperationCanceledException)) + { + return false; + } + } + + return true; } /// Binds each planned endpoint, skipping unsupported implicit-wildcard families. @@ -451,26 +609,41 @@ private async Task AcceptAllAsync(CancellationToken cancellationToken) await Task.WhenAll(sockets.Select(socket => AcceptLoopAsync(socket, cancellationToken))).ConfigureAwait(false); } - /// Accepts sockets until cancellation, disposal, or a non-transient socket error. + /// Accepts sockets until cancellation, disposal, or persistent accept failure. /// Bound socket sampled by . /// Service run token. - /// A task that completes when the loop breaks. + /// A task that completes when the loop breaks, or faults after consecutive accept failures. /// /// Cancellation, , and end the loop /// when is cancelled. A while the token - /// is still active is ignored and the loop continues. Over-capacity sockets are disposed and not processed. + /// is still active is logged (rate-limited), delayed by , and retried. + /// After consecutive failures the loop throws + /// so faults and the host health path observes it. + /// A successful accept resets the consecutive-failure count. Over-capacity sockets are disposed and not processed. /// An admitted socket is configured with before . /// Option failure disposes that socket, releases the admission slot, and continues the loop. /// Admitted work is tracked in until the task completes. /// private async Task AcceptLoopAsync(Socket listenSocket, CancellationToken cancellationToken) { + var consecutiveFailures = 0; + DateTimeOffset? lastFailureLogUtc = null; + while (!cancellationToken.IsCancellationRequested) { Socket accepted; try { - accepted = await listenSocket.AcceptAsync(cancellationToken).ConfigureAwait(false); + accepted = AcceptAsyncOverride is not null + ? await AcceptAsyncOverride(listenSocket, cancellationToken).ConfigureAwait(false) + : await listenSocket.AcceptAsync(cancellationToken).ConfigureAwait(false); + + if (consecutiveFailures > 0) + { + CacheListenerLogMessages.AcceptRecovered(_logger, consecutiveFailures); + consecutiveFailures = 0; + lastFailureLogUtc = null; + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -484,13 +657,52 @@ private async Task AcceptLoopAsync(Socket listenSocket, CancellationToken cancel { break; } - catch (SocketException) + catch (SocketException ex) { if (cancellationToken.IsCancellationRequested) { break; } + consecutiveFailures++; + var socketError = ex.SocketErrorCode.ToString(); + var now = _time.GetUtcNow(); + if (ShouldLogAcceptFailure(consecutiveFailures, lastFailureLogUtc, now)) + { + CacheListenerLogMessages.AcceptFailed( + _logger, + socketError, + consecutiveFailures, + MaxConsecutiveAcceptFailures); + lastFailureLogUtc = now; + } + + if (consecutiveFailures >= MaxConsecutiveAcceptFailures) + { + CacheListenerLogMessages.AcceptLoopStopping(_logger, consecutiveFailures, socketError); + throw new InvalidOperationException( + $"Cache Listener accept failed {consecutiveFailures} consecutive times (last error: {socketError}).", + ex); + } + + try + { + if (AcceptFailureDelayAsync is not null) + { + await AcceptFailureDelayAsync(AcceptFailureRetryDelay, cancellationToken) + .ConfigureAwait(false); + } + else + { + await Task.Delay(AcceptFailureRetryDelay, _time, cancellationToken) + .ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + continue; } @@ -518,6 +730,27 @@ private async Task AcceptLoopAsync(Socket listenSocket, CancellationToken cancel } } + /// + /// Reports whether an accept failure should emit a Warning log given consecutive count and prior log time. + /// + /// Failures since the last successful accept on this loop. + /// UTC of the last Warning for this loop, or null when none yet. + /// Current UTC from the service . + /// for the first failure, threshold crossing, or after . + private static bool ShouldLogAcceptFailure( + int consecutiveFailures, + DateTimeOffset? lastFailureLogUtc, + DateTimeOffset nowUtc) + { + if (consecutiveFailures == 1 || consecutiveFailures >= MaxConsecutiveAcceptFailures) + { + return true; + } + + return lastFailureLogUtc is null + || nowUtc - lastFailureLogUtc.Value >= AcceptFailureLogInterval; + } + /// Handshakes one accepted socket and runs a single NNTP session. /// Accepted socket. Owned by the created here. /// Remote endpoint text captured before processing, used only in logs. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Listener/CacheListenerServiceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Listener/CacheListenerServiceTests.cs index 22d4da6d..6ccf1b78 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Listener/CacheListenerServiceTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Listener/CacheListenerServiceTests.cs @@ -3,15 +3,18 @@ using System.Reflection; using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Xunit.Abstractions; using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Hosting; using VectorNNTPD.BackFillerService.Listener; using VectorNNTPD.BackFillerService.Networking; using VectorNNTPD.BackFillerService.Retention; using VectorNNTPD.BackFillerService.Tests.Fixtures; using VectorNNTPD.BackFillerService.Tests.Retention; using System.Net.Sockets; +using VectorNNTPD.Common.Acme; using VectorNNTPD.Common.Articles; using VectorNNTPD.Common.Articles.Parsing; using VectorNNTPD.Common.Networking.Certificates; @@ -833,6 +836,212 @@ await ssl.AuthenticateAsClientAsync(new SslClientAuthenticationOptions return new TlsClient(tcp, ssl); } + /// + /// Transient accept SocketException is followed by the configured backoff before the next accept attempt. + /// + [Fact] + public async Task Transient_accept_failure_delays_before_retry_and_resets_after_success() + { + var logger = new CollectingLogger(); + var port = GetFreePort(); + await using var certificates = TestListenerCertificates.CreatePublishedProvider(); + var service = CreateListenerService(CreateRuntime(port), certificates, logger); + var acceptAttempts = 0; + var delayCalls = new ConcurrentBag(); + service.AcceptFailureDelayAsync = (delay, _) => + { + delayCalls.Add(delay); + return Task.CompletedTask; + }; + service.AcceptAsyncOverride = (listenSocket, cancellationToken) => + { + var attempt = Interlocked.Increment(ref acceptAttempts); + if (attempt <= 2) + { + return ValueTask.FromException( + new SocketException((int)SocketError.ConnectionReset)); + } + + return listenSocket.AcceptAsync(cancellationToken); + }; + + await service.StartAsync(CancellationToken.None); + try + { + await using var client = await ConnectAsync(port); + await ReadGreetingAsync(client.Stream); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => delayCalls.Count >= 2 && acceptAttempts >= 3, + TimeSpan.FromSeconds(5)); + + Assert.Equal(2, delayCalls.Count); + Assert.All(delayCalls, static delay => Assert.Equal(CacheListenerService.AcceptFailureRetryDelay, delay)); + Assert.Contains(logger.Entries, static entry => entry.EventId.Id == 5416); + Assert.Contains(logger.Entries, static entry => entry.EventId.Id == 5417); + Assert.Equal(CacheListenerState.Running, service.State); + Assert.False(service.Execution!.IsFaulted); + } + finally + { + await service.DisposeAsync(); + } + } + + /// + /// Consecutive accept failures reach the threshold, fault Execution, and stop with rate-limited logging. + /// + [Fact] + public async Task Consecutive_accept_failures_reach_threshold_and_fault_execution() + { + var logger = new CollectingLogger(); + var port = GetFreePort(); + await using var certificates = TestListenerCertificates.CreatePublishedProvider(); + var service = CreateListenerService(CreateRuntime(port), certificates, logger); + var acceptAttempts = 0; + var delayCalls = 0; + service.AcceptFailureDelayAsync = (_, _) => + { + Interlocked.Increment(ref delayCalls); + return Task.CompletedTask; + }; + service.AcceptAsyncOverride = (_, _) => + { + Interlocked.Increment(ref acceptAttempts); + return ValueTask.FromException( + new SocketException((int)SocketError.NetworkDown)); + }; + + await service.StartAsync(CancellationToken.None); + var execution = service.Execution; + Assert.NotNull(execution); + var fault = await Assert.ThrowsAnyAsync(async () => await execution); + var invalid = Assert.IsType( + fault is AggregateException aggregate + ? Assert.Single(aggregate.Flatten().InnerExceptions.OfType()) + : fault); + Assert.Contains("consecutive", invalid.Message, StringComparison.OrdinalIgnoreCase); + Assert.IsType(invalid.InnerException); + Assert.Equal(CacheListenerService.MaxConsecutiveAcceptFailures, acceptAttempts); + Assert.Equal(CacheListenerService.MaxConsecutiveAcceptFailures - 1, delayCalls); + Assert.Equal( + 2, + logger.Entries.Count(static entry => entry.EventId.Id == 5416)); + Assert.Contains(logger.Entries, static entry => entry.EventId.Id == 5418); + Assert.DoesNotContain(logger.Entries, static entry => entry.EventId.Id == 5417); + + await service.DisposeAsync(); + Assert.Equal(CacheListenerState.Stopped, service.State); + Assert.Contains(logger.Entries, static entry => entry.EventId.Id == 5419); + } + + /// + /// An unexpected accept-task fault is not treated as an expected dispose observation; cleanup still stops, + /// then dispose rethrows so the programming error remains visible. + /// + [Fact] + public async Task Unexpected_accept_fault_is_not_swallowed_by_dispose() + { + var logger = new CollectingLogger(); + var port = GetFreePort(); + await using var certificates = TestListenerCertificates.CreatePublishedProvider(); + var service = CreateListenerService(CreateRuntime(port), certificates, logger); + service.AcceptAsyncOverride = (_, _) => + ValueTask.FromException(new DivideByZeroException("accept boom")); + + await service.StartAsync(CancellationToken.None); + var execution = service.Execution; + Assert.NotNull(execution); + await Assert.ThrowsAnyAsync(async () => await execution); + Assert.True(execution.IsFaulted); + + var disposeFault = await Record.ExceptionAsync(async () => await service.DisposeAsync()); + Assert.NotNull(disposeFault); + Assert.Contains( + disposeFault is AggregateException aggregate + ? aggregate.Flatten().InnerExceptions + : [disposeFault], + static ex => ex is DivideByZeroException || ex.GetBaseException() is DivideByZeroException); + Assert.DoesNotContain(logger.Entries, static entry => entry.EventId.Id == 5419); + Assert.Equal(CacheListenerState.Stopped, service.State); + } + + /// + /// Cancellation during accept backoff exits the loop without treating shutdown as an unexpected failure. + /// + [Fact] + public async Task Cancellation_during_accept_backoff_exits_without_faulting_execution() + { + var logger = new CollectingLogger(); + var port = GetFreePort(); + await using var certificates = TestListenerCertificates.CreatePublishedProvider(); + var service = CreateListenerService(CreateRuntime(port), certificates, logger); + var enteredDelay = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + service.AcceptAsyncOverride = (_, _) => ValueTask.FromException( + new SocketException((int)SocketError.Interrupted)); + service.AcceptFailureDelayAsync = async (_, cancellationToken) => + { + enteredDelay.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + }; + + await service.StartAsync(CancellationToken.None); + var execution = service.Execution; + Assert.NotNull(execution); + await enteredDelay.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.DisposeAsync(); + + Assert.True(execution.IsCompletedSuccessfully); + Assert.Equal(CacheListenerState.Stopped, service.State); + Assert.DoesNotContain(logger.Entries, static entry => entry.EventId.Id == 5418); + } + + /// + /// Shutdown cancellation while AcceptAsync is waiting exits cleanly without an unexpected failure log. + /// + [Fact] + public async Task Shutdown_cancellation_during_accept_exits_cleanly() + { + var logger = new CollectingLogger(); + var port = GetFreePort(); + await using var certificates = TestListenerCertificates.CreatePublishedProvider(); + var service = CreateListenerService(CreateRuntime(port), certificates, logger); + var enteredAccept = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + service.AcceptAsyncOverride = async (_, cancellationToken) => + { + enteredAccept.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + throw new InvalidOperationException("Accept override should not resume after cancellation."); + }; + + await service.StartAsync(CancellationToken.None); + var execution = service.Execution; + Assert.NotNull(execution); + await enteredAccept.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await service.DisposeAsync(); + + Assert.True(execution.IsCompletedSuccessfully); + Assert.Equal(CacheListenerState.Stopped, service.State); + Assert.DoesNotContain(logger.Entries, static entry => entry.EventId.Id == 5416); + Assert.DoesNotContain(logger.Entries, static entry => entry.EventId.Id == 5418); + } + + /// Builds a production-shaped listener with injectable logger for accept-loop tests. + private static CacheListenerService CreateListenerService( + BackFillerRuntimeOptions runtime, + ITlsCertificateContextProvider certificates, + ILogger logger) + { + var readiness = new AcmeCertificateReadiness(); + readiness.MarkReady(); + return new CacheListenerService( + runtime, + certificates, + ArticleRetentionAuthorityTests.Create(TimeProvider.System, 1024), + readiness, + new BackFillerStartupJournal(), + logger); + } + /// Builds valid runtime options bound to loopback with a connection cap. /// TLS bind port for the listener under test. /// Maximum concurrent active connections.