From edadcbcdd14fac16a99d288caeb94e65825e32a7 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Sun, 25 Oct 2026 20:14:20 +0200 Subject: [PATCH 1/3] fix(backfiller): harden long-running service recovery Detect stalled article processing and recover through the existing reconciliation lifecycle. Ensure recovery cancels hung work even when normal retirement has already started, and prevent queued admissions from masking stalled handlers. Improve quorum and topology failure observability, and disable blocking on full asynchronous file logging queues to prevent logging backpressure from stalling article processing. Add regression tests for retirement races, cancellation failures, delayed cancellation, and concurrent recovery. Preserve existing daily log rotation, compression, and retention policies. Release x64 build and all applicable regression suites pass. --- .../ArticleWork/ArticleWorkConsumerService.cs | 94 +++- .../ArticleWork/ArticleWorkConsumerSession.cs | 112 ++++- .../ArticleWork/ArticleWorkLogMessages.cs | 53 ++ .../Logging/BackFillerFileLogging.cs | 8 +- .../ArticleWorkChannelFailureRecoveryTests.cs | 457 +++++++++++++++++- ...leWorkQuorumHealthAndDeliveryLimitTests.cs | 7 +- .../Logging/BackFillerSerilogLoggingTests.cs | 2 +- 7 files changed, 717 insertions(+), 16 deletions(-) diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs index fca20c5c..c947fb1a 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs @@ -32,6 +32,19 @@ internal sealed class ArticleWorkConsumerService : IHostedService, IAsyncDisposa /// Upper bound for the reconcile-loop backoff after repeated unexpected faults or topology-not-ready passes. private static readonly TimeSpan ReconcileBackoffCap = TimeSpan.FromMinutes(2); + /// + /// Age of the last active-pipeline progress mark before a healthy-looking session is retired and rebuilt. + /// + /// + /// Exceeds default NNTP connect, command, and ARTICLE receive budgets so a legitimate long retrieval is not + /// treated as a stall. Idle sessions (no active work) are never retired by this threshold. + /// + internal static readonly TimeSpan ProcessingStallThreshold = + NntpSessionOptions.Default.ConnectTimeout + + NntpSessionOptions.Default.CommandTimeout + + NntpSessionOptions.Default.ReceiveTimeout + + TimeSpan.FromMinutes(1); + /// Process RabbitMQ connection owner. This service opens channels on it and does not open another connection. private readonly IRabbitMqService _connections; @@ -73,6 +86,12 @@ internal sealed class ArticleWorkConsumerService : IHostedService, IAsyncDisposa /// How many times the loop has armed its delay. Used by tests to observe backoff without a second loop. private int _reconcileWaitsStarted; + /// + /// Consecutive reconcile passes that returned . + /// Reset when a pass succeeds. Used for sustained topology/quorum unavailability observability. + /// + private int _consecutiveInfrastructureUnavailable; + /// /// Guards the session dictionary, , and publication of . /// Non-recursive. No caller enters it again on the same call stack. @@ -194,6 +213,12 @@ public bool IsRunning /// How many reconcile delays the loop has armed, including the delay that follows a fault. internal int ReconcileWaitsStarted => Volatile.Read(ref _reconcileWaitsStarted); + /// + /// Consecutive infrastructure-unavailable reconcile passes. Zero after a successful membership pass. + /// + internal int ConsecutiveInfrastructureUnavailable => + Volatile.Read(ref _consecutiveInfrastructureUnavailable); + /// Delay the loop will use for the wait it has armed or will arm next. internal TimeSpan CurrentReconcileDelay => TimeSpan.FromTicks(Volatile.Read(ref _reconcileDelayTicks)); @@ -571,6 +596,48 @@ private static bool IsHealthyRunningSession(ArticleWorkConsumerSession session) session.State == ArticleWorkConsumerState.Running && session.Channel is { IsOpen: true }; + /// + /// Records a topology/quorum unavailability pass and logs desired capacity versus healthy consumers. + /// Does not change fail-closed membership policy. + /// + /// Slots desired at the start of this reconcile pass. + private void ObserveInfrastructureUnavailable(int desiredSlotCount) + { + var streak = Interlocked.Increment(ref _consecutiveInfrastructureUnavailable); + var healthy = 0; + lock (_gate) + { + foreach (var session in _sessions.Values) + { + if (IsHealthyRunningSession(session)) + { + healthy++; + } + } + } + + ArticleWorkLogMessages.ConsumerInfrastructureUnavailable( + _logger, + desiredSlotCount, + healthy, + streak); + } + + /// + /// Clears a prior infrastructure-unavailable streak after a successful membership pass. + /// + /// Slots desired at the start of the recovering pass. + private void ObserveInfrastructureRecovered(int desiredSlotCount) + { + var prior = Interlocked.Exchange(ref _consecutiveInfrastructureUnavailable, 0); + if (prior <= 0) + { + return; + } + + ArticleWorkLogMessages.ConsumerInfrastructureRecovered(_logger, desiredSlotCount, prior); + } + /// Reconciles after a capacity snapshot. /// /// A task that completes when finishes or its exception has been logged. @@ -621,14 +688,30 @@ private async Task ReconcileSessionsAsync(CancellationToken cance : (ushort)1; var desired = BuildDesiredSessions(); + var desiredSlotCount = desired.Count; var started = new List(); List stale = []; + HashSet stalled = []; lock (_gate) { foreach (var pair in _sessions.ToArray()) { if (desired.ContainsKey(pair.Key) && IsHealthyRunningSession(pair.Value)) { + if (pair.Value.IsActiveProcessingStalled(ProcessingStallThreshold)) + { + ArticleWorkLogMessages.ConsumerProcessingStalled( + _logger, + pair.Value.Backbone, + pair.Value.Generation, + pair.Value.ActiveCount, + ProcessingStallThreshold); + _sessions.Remove(pair.Key); + stale.Add(pair.Value); + stalled.Add(pair.Value); + continue; + } + desired.Remove(pair.Key); continue; } @@ -640,7 +723,10 @@ private async Task ReconcileSessionsAsync(CancellationToken cance foreach (var session in stale) { - await session.RetireAsync(cancellationToken).ConfigureAwait(false); + var retire = stalled.Contains(session) + ? session.RetireForRecoveryAsync(cancellationToken) + : session.RetireAsync(cancellationToken); + await retire.ConfigureAwait(false); await session.DisposeAsync().ConfigureAwait(false); } @@ -657,6 +743,7 @@ private async Task ReconcileSessionsAsync(CancellationToken cance infrastructureUnavailable = requestedBackbones.Any(backbone => !readyBackbones.Contains(backbone)); if (readyBackbones.Count == 0) { + ObserveInfrastructureUnavailable(desiredSlotCount); return ReconcilePass.InfrastructureUnavailable; } @@ -680,7 +767,8 @@ private async Task ReconcileSessionsAsync(CancellationToken cance _logger, _runtime.Shutdown, identity.ConnectionNumber, - identity.ConnectionLimit) + identity.ConnectionLimit, + _time) { UnexpectedChannelFault = OnSessionChannelFaulted, }; @@ -722,9 +810,11 @@ private async Task ReconcileSessionsAsync(CancellationToken cance if (infrastructureUnavailable) { + ObserveInfrastructureUnavailable(desiredSlotCount); return ReconcilePass.InfrastructureUnavailable; } + ObserveInfrastructureRecovered(desiredSlotCount); ArticleWorkLogMessages.ConsumerReconcileCompleted(_logger, Sessions.Count); return ReconcilePass.Succeeded; } diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs index ccab9478..6166f82a 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs @@ -57,6 +57,14 @@ internal sealed class ArticleWorkConsumerSession : IAsyncDisposable /// Shutdown policy captured at construction. Not re-read from options later. private readonly BackFillerShutdownRuntimeOptions _shutdown; + /// Clock for processing-progress stamps. Production uses . + private readonly TimeProvider _time; + + /// + /// UTC ticks of the last admit or active-pipeline progress mark. Zero until the first admit. + /// + private long _lastProgressUtcTicks; + /// /// Guards lifecycle state, the channel, the consumer tag, and the admitted-delivery count. /// Non-recursive. No caller enters it again on the same call stack. @@ -99,6 +107,12 @@ internal sealed class ArticleWorkConsumerSession : IAsyncDisposable /// One after has entered disposal. private int _disposed; + /// + /// When non-zero, retirement cancels queued and active work even if the captured shutdown policy would finish them. + /// Used for processing-stall recovery so a hung handler cannot block membership rebuild. + /// + private int _forceCancelWorkOnRetire; + /// /// Optional callback invoked when an unexpected channel fault retires this session while it was running. /// The consumer service uses it to schedule an observed reconcile without reconnecting. @@ -113,6 +127,9 @@ internal sealed class ArticleWorkConsumerSession : IAsyncDisposable /// Parse/classify/settle pipeline. /// Process connection owner. Must not be used to open a second connection. /// Session logger. + /// + /// Clock for processing-progress stamps. Null uses . + /// /// Thrown when is null or whitespace. /// /// Thrown when , , or is null. @@ -123,8 +140,9 @@ internal ArticleWorkConsumerSession( ushort prefetch, ArticleWorkDeliveryPipeline pipeline, IRabbitMqService connections, - ILogger logger) - : this(backbone, prefetch, pipeline, connections, logger, DefaultShutdown, connectionNumber: 1, connectionLimit: 1) + ILogger logger, + TimeProvider? time = null) + : this(backbone, prefetch, pipeline, connections, logger, DefaultShutdown, connectionNumber: 1, connectionLimit: 1, time) { } @@ -139,6 +157,10 @@ internal ArticleWorkConsumerSession( /// Immutable runtime shutdown policy. Must not be re-read from options. /// One-based consume slot (old NNTP connection number). /// Desired NNTP slot count for this backbone. + /// + /// Clock for processing-progress stamps. Null uses . + /// Should match the consumer service clock so stall detection and progress marks share one timeline. + /// /// Thrown when is null or whitespace. /// /// Thrown when , , , or is null. @@ -155,7 +177,8 @@ internal ArticleWorkConsumerSession( ILogger logger, BackFillerShutdownRuntimeOptions shutdown, int connectionNumber = 1, - int connectionLimit = 1) + int connectionLimit = 1, + TimeProvider? time = null) { ArgumentException.ThrowIfNullOrWhiteSpace(backbone); ArgumentNullException.ThrowIfNull(pipeline); @@ -180,6 +203,7 @@ internal ArticleWorkConsumerSession( _connections = connections; _logger = logger; _shutdown = shutdown; + _time = time ?? TimeProvider.System; } /// Provider backbone label for this consume session. Not case-folded. @@ -222,6 +246,53 @@ internal int QueuedCount /// Deliveries that hold and are inside the pipeline handler. internal int ActiveCount => Volatile.Read(ref _active); + /// + /// UTC instant of the last active-pipeline enter or exit progress mark. + /// until a delivery becomes active pipeline work. + /// + /// + /// Admission alone does not stamp progress. A queued delivery must not reset the stall clock while + /// another delivery remains hung in the active pipeline. + /// + internal DateTimeOffset? LastProgressUtc + { + get + { + var ticks = Volatile.Read(ref _lastProgressUtcTicks); + return ticks == 0 ? null : new DateTimeOffset(ticks, TimeSpan.Zero); + } + } + + /// + /// True when an admitted delivery still holds the dispatch path (active pipeline work). + /// Queued-only waiters are not treated as processing stalls; the active holder is. + /// + internal bool HasActiveProcessing => Volatile.Read(ref _active) > 0; + + /// + /// True when active pipeline work has made no progress for at least . + /// Idle sessions (no active work) never stall. Legitimate ARTICLE receive budgets are covered by + /// choosing a threshold above command and receive timeouts. + /// + /// Minimum age of the last progress mark before stall is declared. + /// Whether the session should be retired and rebuilt by membership reconcile. + internal bool IsActiveProcessingStalled(TimeSpan threshold) + { + if (!HasActiveProcessing) + { + return false; + } + + var ticks = Volatile.Read(ref _lastProgressUtcTicks); + if (ticks == 0) + { + return false; + } + + var age = _time.GetUtcNow() - new DateTimeOffset(ticks, TimeSpan.Zero); + return age >= threshold; + } + /// Cancelled when retirement or dispose must not admit new queued pipeline work. private CancellationToken QueueCancellationToken => _queueCts.Token; @@ -377,6 +448,26 @@ internal async Task StartAsync(CancellationToken cancellationToken) /// internal Task RetireAsync() => RetireAsync(CancellationToken.None); + /// + /// Retires the session for recovery, cancelling queued and active work even when the shutdown policy would finish them. + /// + /// Forwarded to . + /// The session retirement task. + /// + /// Used when active pipeline work is stalled past . + /// Without forced cancel, a hung handler would block drain forever and prevent consumer rebuild. + /// + internal Task RetireForRecoveryAsync(CancellationToken shutdownToken) + { + Interlocked.Exchange(ref _forceCancelWorkOnRetire, 1); + var retireTask = RetireAsync(shutdownToken); + // If a non-recovery RetireAsync already won publication, RetireCore may still be waiting + // under FinishActiveArticles. Cancel both tokens so a hung handler cannot deadlock rebuild. + _queueCts.Cancel(); + _workCts.Cancel(); + return retireTask; + } + /// /// Cancels the consumer, applies the captured shutdown policy, and waits until admitted work drains or is cancelled. /// @@ -420,7 +511,7 @@ internal Task RetireAsync(CancellationToken shutdownToken) _retireTask = retireTask; } - if (!_shutdown.DrainQueuedWork) + if (!_shutdown.DrainQueuedWork || Volatile.Read(ref _forceCancelWorkOnRetire) == 1) { _queueCts.Cancel(); } @@ -499,12 +590,13 @@ private async Task RetireCoreAsync( _ = await TryBasicCancelAsync(channel, consumerTag).ConfigureAwait(false); } - if (!_shutdown.DrainQueuedWork) + var forceCancel = Volatile.Read(ref _forceCancelWorkOnRetire) == 1; + if (!_shutdown.DrainQueuedWork || forceCancel) { await _queueCts.CancelAsync().ConfigureAwait(false); } - if (!_shutdown.FinishActiveArticles) + if (!_shutdown.FinishActiveArticles || forceCancel) { await _workCts.CancelAsync().ConfigureAwait(false); } @@ -651,12 +743,14 @@ private async Task OnDeliveryAsync(RabbitMqManualAckDelivery delivery) } Interlocked.Increment(ref _active); + MarkProgress(); try { await ProcessCurrentAsync(delivery, _workCts.Token).ConfigureAwait(false); } finally { + MarkProgress(); Interlocked.Decrement(ref _active); } } @@ -806,6 +900,12 @@ private bool TryAdmit() } } + /// + /// Records an active-pipeline progress stamp on for stall detection during membership reconcile. + /// + private void MarkProgress() => + Volatile.Write(ref _lastProgressUtcTicks, _time.GetUtcNow().UtcTicks); + /// /// Decrements the admitted count and completes drain when retirement is waiting and nothing admitted remains. /// diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs index 498e57d7..5a7802e1 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs @@ -518,5 +518,58 @@ internal static partial void DeliveryLimitExhausted( int deliveryCount, int deliveryLimit, string outcome); + + /// + /// Writes a Warning when a running consumer with active pipeline work has made no progress beyond the stall threshold. + /// Membership reconcile retires and replaces the session. + /// + /// Logger that receives the event. + /// Provider backbone of the stalled session. + /// Connection generation of the consumer channel. + /// Active pipeline deliveries observed at detection. + /// Configured stall age that was exceeded. + [LoggerMessage( + EventId = 5328, + Level = LogLevel.Warning, + Message = "Article Work consumer processing stalled backbone={Backbone} generation={Generation} activeCount={ActiveCount} stallThreshold={StallThreshold}; retiring session for rebuild")] + internal static partial void ConsumerProcessingStalled( + ILogger logger, + string backbone, + long generation, + int activeCount, + TimeSpan stallThreshold); + + /// + /// Writes a Warning when desired consume slots cannot start because provider topology or quorum membership is not ready. + /// Fail-closed policy is unchanged; the reconcile loop continues with backoff. + /// + /// Logger that receives the event. + /// Consume slots desired at the start of the pass. + /// Tracked sessions that still have an open consume channel. + /// How many consecutive infrastructure-unavailable passes have occurred. + [LoggerMessage( + EventId = 5329, + Level = LogLevel.Warning, + Message = "Article Work infrastructure unavailable desiredSlots={DesiredSlots} healthySessions={HealthySessions} consecutiveUnavailablePasses={ConsecutiveUnavailablePasses}")] + internal static partial void ConsumerInfrastructureUnavailable( + ILogger logger, + int desiredSlots, + int healthySessions, + int consecutiveUnavailablePasses); + + /// + /// Writes an Information event when membership reconcile succeeds after one or more infrastructure-unavailable passes. + /// + /// Logger that receives the event. + /// Consume slots desired at the start of the recovering pass. + /// Consecutive unavailable passes cleared by this success. + [LoggerMessage( + EventId = 5330, + Level = LogLevel.Information, + Message = "Article Work infrastructure recovered desiredSlots={DesiredSlots} previousUnavailablePasses={PreviousUnavailablePasses}")] + internal static partial void ConsumerInfrastructureRecovered( + ILogger logger, + int desiredSlots, + int previousUnavailablePasses); } } diff --git a/src/VectorNNTPD.BackFillerService/Logging/BackFillerFileLogging.cs b/src/VectorNNTPD.BackFillerService/Logging/BackFillerFileLogging.cs index 93d68a37..e81b791c 100644 --- a/src/VectorNNTPD.BackFillerService/Logging/BackFillerFileLogging.cs +++ b/src/VectorNNTPD.BackFillerService/Logging/BackFillerFileLogging.cs @@ -56,7 +56,13 @@ internal static class BackFillerFileLogging /// /// When true, the async file and RabbitMQ wrappers block once events are queued. /// - private const bool AsyncBlockWhenFull = true; + /// + /// False so a stalled file sink (disk full, day-roll gzip under lock) drops excess events instead of + /// blocking article-path loggers and delaying settlement. Daily rotation, compression, and retention + /// are unchanged. Critical diagnostics may be lost under sustained backpressure; Serilog SelfLog may + /// record drops. Prefer ops alerts on log-volume disk pressure over blocking the data plane. + /// + private const bool AsyncBlockWhenFull = false; /// Passed as the File sink buffered flag. True leaves that sink buffered. private const bool FileBuffered = true; diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs index 18b3154b..c9cac550 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs @@ -220,6 +220,386 @@ await ArticleWorkTestDeliveries.WaitUntilAsync( Assert.Equal(1, context.Factory.ConnectCount); } + /// + /// Active pipeline work with no progress beyond the stall threshold is retired and replaced without reconnecting. + /// Recovery cancels the hung handler; the gate is never released (proves no FinishActiveArticles deadlock). + /// + [Fact] + public async Task Active_processing_stall_retires_session_and_starts_replacement_without_reconnect() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "held", + Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + await using var context = await StartConsumerAsync(handler, clock); + var first = Assert.Single(context.Consumer.Sessions); + var firstChannel = Assert.IsType(first.Channel); + var processing = firstChannel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + await handler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.True(first.HasActiveProcessing); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + await context.Consumer.ReconcileAsync(CancellationToken.None); + + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => context.Consumer.Sessions.Count == 1 + && !ReferenceEquals(context.Consumer.Sessions[0], first) + && context.Consumer.Sessions[0].Channel is { IsOpen: true }, + TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.Equal(1, context.Factory.ConnectCount); + Assert.False(handler.Gate!.Task.IsCompleted); + + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(1, firstChannel.DisposeCount); + var interrupted = Assert.Single(firstChannel.Settlements); + Assert.False(interrupted.Acknowledge); + Assert.True(interrupted.Requeue); + + // Replacement must accept new work without depending on the hung first gate. + handler.Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handler.Gate.TrySetResult(); + var replacement = Assert.IsType(context.Consumer.Sessions[0].Channel); + await replacement.DeliverAsync(ArticleWorkTestDeliveries.Canonical(deliveryTag: 2)); + Assert.NotEmpty(replacement.Settlements); + } + + /// + /// An idle running consumer is not retired merely because wall time advances past the stall threshold. + /// + [Fact] + public async Task Idle_consumer_is_not_retired_when_stall_threshold_elapses() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + await using var context = await StartConsumerAsync(time: clock); + var first = Assert.Single(context.Consumer.Sessions); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold + TimeSpan.FromMinutes(1)); + await context.Consumer.ReconcileAsync(CancellationToken.None); + + Assert.Same(first, Assert.Single(context.Consumer.Sessions)); + Assert.Equal(ArticleWorkConsumerState.Running, first.State); + Assert.True(first.Channel!.IsOpen); + } + + /// + /// Active work younger than the stall threshold is left running; reconcile does not retire it. + /// + [Fact] + public async Task Active_work_below_stall_threshold_is_not_retired() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "held", + Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + await using var context = await StartConsumerAsync(handler, clock); + var first = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(first.Channel); + var processing = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + await handler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold - TimeSpan.FromSeconds(1)); + await context.Consumer.ReconcileAsync(CancellationToken.None); + + Assert.Same(first, Assert.Single(context.Consumer.Sessions)); + Assert.Equal(ArticleWorkConsumerState.Running, first.State); + Assert.True(first.HasActiveProcessing); + + handler.Gate!.TrySetResult(); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.NotEmpty(channel.Settlements); + } + + /// + /// When a normal retire wins the race, recovery still force-cancels hung work so rebuild cannot deadlock. + /// + [Fact] + public async Task RetireForRecovery_force_cancels_after_non_recovery_retire_already_started() + { + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "held", + Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + await using var context = await StartConsumerAsync(handler); + var first = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(first.Channel); + var processing = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + await handler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + var normalRetire = first.RetireAsync(CancellationToken.None); + var recoveryRetire = first.RetireForRecoveryAsync(CancellationToken.None); + Assert.Same(normalRetire, recoveryRetire); + + await recoveryRetire.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.False(handler.Gate!.Task.IsCompleted); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + } + + /// + /// Dispose overlapping an active stall recovery completes without waiting on the hung gate. + /// + [Fact] + public async Task Dispose_during_stalled_active_work_completes_without_gate_release() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "held", + Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + var consumer = new ArticleWorkConsumerService( + connections, + RabbitMqServiceTests.CreateFastRuntime(), + handler, + new RecordingArticleWorkResponsePublisher(), + NullLogger.Instance, + catalog: new StaticBackFillerProviderCatalog( + [ + new BackFillerProviderDefinition( + "Giganews", + "127.0.0.1", + 119, + false, + "nntp-user", + "p", + 0, + 1), + ]), + capacity: CreateCapacity("Giganews", 1), + time: clock); + await consumer.StartAsync(CancellationToken.None); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => consumer.Sessions.Count == 1 && consumer.Sessions[0].State == ArticleWorkConsumerState.Running, + TimeSpan.FromSeconds(2)); + + var first = Assert.Single(consumer.Sessions); + var channel = Assert.IsType(first.Channel); + var processing = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + await handler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + var reconcile = consumer.ReconcileAsync(CancellationToken.None); + await consumer.DisposeAsync(); + await reconcile.WaitAsync(TimeSpan.FromSeconds(2)); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.False(handler.Gate!.Task.IsCompleted); + await connections.DisposeAsync(); + } + + /// + /// A handler that completes normally under the stall window settles and is not recovery-retired. + /// + [Fact] + public async Task Normal_handler_completion_settles_without_stall_recovery() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "missing", + }; + await using var context = await StartConsumerAsync(handler, clock); + var session = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(session.Channel); + + await channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + Assert.NotEmpty(channel.Settlements); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + await context.Consumer.ReconcileAsync(CancellationToken.None); + + Assert.Same(session, Assert.Single(context.Consumer.Sessions)); + Assert.Equal(ArticleWorkConsumerState.Running, session.State); + } + + /// + /// Completing work then starting new work refreshes progress so wall time from the prior article + /// does not falsely stall the next active delivery. + /// + [Fact] + public async Task Progress_refresh_on_new_active_work_avoids_false_stall() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "missing", + }; + await using var context = await StartConsumerAsync(handler, clock); + var session = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(session.Channel); + + await channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical(deliveryTag: 1)); + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold - TimeSpan.FromSeconds(5)); + + handler.Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handler.Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var second = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical(deliveryTag: 2)); + await handler.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + clock.Advance(TimeSpan.FromSeconds(10)); + await context.Consumer.ReconcileAsync(CancellationToken.None); + + Assert.Same(session, Assert.Single(context.Consumer.Sessions)); + Assert.True(session.HasActiveProcessing); + + handler.Gate.TrySetResult(); + await second.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(2, channel.Settlements.Count); + } + + /// + /// A second reconcile while stall recovery is in flight shares serialization and still finishes + /// without releasing the hung gate. + /// + [Fact] + public async Task Concurrent_reconcile_during_stall_recovery_completes_once() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "held", + Started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + await using var context = await StartConsumerAsync(handler, clock); + var first = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(first.Channel); + var processing = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + await handler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + var firstReconcile = context.Consumer.ReconcileAsync(CancellationToken.None); + var secondReconcile = context.Consumer.ReconcileAsync(CancellationToken.None); + await Task.WhenAll(firstReconcile, secondReconcile).WaitAsync(TimeSpan.FromSeconds(2)); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.False(handler.Gate!.Task.IsCompleted); + Assert.Single(context.Consumer.Sessions); + Assert.Equal(1, context.Factory.ConnectCount); + } + + /// + /// A handler that observes cancel then delays exit still drains under recovery without waiting on the gate. + /// + [Fact] + public async Task Stall_recovery_waits_for_delayed_cancel_cleanup_without_gate_release() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var cancelSeen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var allowExit = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var handler = new DelayedCancelArticleWorkHandler(cancelSeen, allowExit); + await using var context = await StartConsumerAsync(handler, clock); + var first = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(first.Channel); + var processing = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + await handler.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + var reconcile = context.Consumer.ReconcileAsync(CancellationToken.None); + await cancelSeen.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.False(reconcile.IsCompleted); + + allowExit.TrySetResult(); + await reconcile.WaitAsync(TimeSpan.FromSeconds(2)); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.Single(context.Consumer.Sessions); + } + + /// + /// A handler that throws after observing cancel still allows recovery to complete and replace the session. + /// + [Fact] + public async Task Stall_recovery_survives_handler_throw_during_cancel_cleanup() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var handler = new ThrowingOnCancelArticleWorkHandler(); + await using var context = await StartConsumerAsync(handler, clock); + var first = Assert.Single(context.Consumer.Sessions); + var channel = Assert.IsType(first.Channel); + var processing = channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical()); + await handler.Started.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + await context.Consumer.ReconcileAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2)); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.Single(context.Consumer.Sessions); + Assert.NotSame(first, context.Consumer.Sessions[0]); + } + + /// + /// Two capacity slots with active hung work are both recovery-retired and replaced. + /// + [Fact] + public async Task Multiple_stalled_sessions_are_retired_together() + { + var clock = new ManualTimeProvider(DateTimeOffset.Parse("2026-10-25T12:00:00Z")); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "held", + Gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + }; + var started1 = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started2 = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + handler.Stages.Enqueue(new ArticleWorkControlStage( + started1, + handler.Gate, + ArticleWorkOutcome.ArticleNotFound, + Error: "held-1")); + handler.Stages.Enqueue(new ArticleWorkControlStage( + started2, + handler.Gate, + ArticleWorkOutcome.ArticleNotFound, + Error: "held-2")); + + await using var context = await StartConsumerAsync(handler, clock, capacitySlots: 2); + Assert.Equal(2, context.Consumer.Sessions.Count); + var first = context.Consumer.Sessions[0]; + var second = context.Consumer.Sessions[1]; + var channel1 = Assert.IsType(first.Channel); + var channel2 = Assert.IsType(second.Channel); + var processing1 = channel1.DeliverAsync(ArticleWorkTestDeliveries.Canonical(deliveryTag: 1)); + var processing2 = channel2.DeliverAsync(ArticleWorkTestDeliveries.Canonical(deliveryTag: 2)); + await Task.WhenAll(started1.Task, started2.Task).WaitAsync(TimeSpan.FromSeconds(2)); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + await context.Consumer.ReconcileAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2)); + await Task.WhenAll(processing1, processing2).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkConsumerState.Stopped, first.State); + Assert.Equal(ArticleWorkConsumerState.Stopped, second.State); + Assert.Equal(2, context.Consumer.Sessions.Count); + Assert.All(context.Consumer.Sessions, s => Assert.NotEqual(ArticleWorkConsumerState.Stopped, s.State)); + Assert.False(handler.Gate.Task.IsCompleted); + } + /// /// Publish-channel shutdown installs a replacement on the same connection generation. /// @@ -446,10 +826,15 @@ private static ArticleWorkResponseIntent SuccessIntent() => ArticleWorkTestDeliveries.CanonicalListenerPort, ArticleWorkTestDeliveries.CanonicalArticleIdHex); - /// Starts a single-session Article Work consumer against a fake RabbitMQ factory. + /// Starts an Article Work consumer against a fake RabbitMQ factory. /// Handler wired into the consumer; defaults to . + /// Optional clock for reconcile delay and processing-stall detection. + /// Usable NNTP capacity slots published for Giganews (session count). /// Harness holding factory, connections, and running consumer. - private static async Task StartConsumerAsync(IArticleWorkHandler? handler = null) + private static async Task StartConsumerAsync( + IArticleWorkHandler? handler = null, + TimeProvider? time = null, + int capacitySlots = 1) { var factory = new FakeBackFillerRabbitMqConnectionFactory(); var connections = RabbitMqServiceTests.CreateService(factory); @@ -470,16 +855,78 @@ private static async Task StartConsumerAsync(IArticleWorkHandle "nntp-user", "p", 0, - 1), + capacitySlots), ]), - capacity: CreateCapacity("Giganews", 1)); + capacity: CreateCapacity("Giganews", capacitySlots), + time: time); await consumer.StartAsync(CancellationToken.None); await ArticleWorkTestDeliveries.WaitUntilAsync( - () => consumer.Sessions.Count == 1 && consumer.Sessions[0].State == ArticleWorkConsumerState.Running, + () => consumer.Sessions.Count == capacitySlots + && consumer.Sessions.All(static s => s.State == ArticleWorkConsumerState.Running), TimeSpan.FromSeconds(2)); return new ConsumerHarness(factory, connections, consumer); } + /// + /// Handler that waits until cancelled, signals cancel observation, then blocks until the test releases exit. + /// + private sealed class DelayedCancelArticleWorkHandler( + TaskCompletionSource cancelSeen, + TaskCompletionSource allowExit) : IArticleWorkHandler + { + /// Signalled when begins waiting. + public TaskCompletionSource Started { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + public async ValueTask HandleAsync( + ArticleWorkItem item, + CancellationToken cancellationToken) + { + Started.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + cancelSeen.TrySetResult(); + await allowExit.Task.ConfigureAwait(false); + throw; + } + + return new ArticleWorkHandlerResult(ArticleWorkOutcome.ArticleNotFound, "unreachable"); + } + } + + /// + /// Handler that waits until cancelled, then throws a non-cancel exception during cleanup. + /// + private sealed class ThrowingOnCancelArticleWorkHandler : IArticleWorkHandler + { + /// Signalled when begins waiting. + public TaskCompletionSource Started { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + public async ValueTask HandleAsync( + ArticleWorkItem item, + CancellationToken cancellationToken) + { + Started.TrySetResult(); + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw new InvalidOperationException("cleanup failed after cancel"); + } + + return new ArticleWorkHandlerResult(ArticleWorkOutcome.ArticleNotFound, "unreachable"); + } + } + /// Starts a running with publish confirms enabled. private static async Task StartPublisherAsync() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs index 5f39bb50..b3a5f9e4 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs @@ -344,13 +344,18 @@ public async Task Later_healthy_membership_allows_readiness_via_reconcile() await consumer.StartAsync(CancellationToken.None); Assert.Empty(consumer.Sessions); + Assert.True(consumer.ConsecutiveInfrastructureUnavailable >= 1); + + await consumer.ReconcileAsync(CancellationToken.None); + Assert.True(consumer.ConsecutiveInfrastructureUnavailable >= 2); probe.Next = Healthy(required: 3, observed: 3, online: 3); await consumer.ReconcileAsync(CancellationToken.None); var session = Assert.Single(consumer.Sessions); Assert.Equal(ArticleWorkConsumerState.Running, session.State); - Assert.True(probe.InspectCount >= 2); + Assert.Equal(0, consumer.ConsecutiveInfrastructureUnavailable); + Assert.True(probe.InspectCount >= 3); await consumer.DisposeAsync(); await connections.DisposeAsync(); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Logging/BackFillerSerilogLoggingTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Logging/BackFillerSerilogLoggingTests.cs index 008d64c9..ed868721 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Logging/BackFillerSerilogLoggingTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Logging/BackFillerSerilogLoggingTests.cs @@ -376,7 +376,7 @@ public void ExplicitConfiguration_BuildsUnlimitedDailyAsyncGzipFileSink() var asyncSink = Assert.Single( sinks, static n => n.GetType().Name.Equals("BackgroundWorkerSink", StringComparison.Ordinal)); - Assert.True(Assert.IsType(ReadInstanceField(asyncSink, "_blockWhenFull")!)); + Assert.False(Assert.IsType(ReadInstanceField(asyncSink, "_blockWhenFull")!)); var queue = ReadInstanceField(asyncSink, "_queue") ?? throw new InvalidOperationException("BackgroundWorkerSink._queue was not found."); var boundedCapacity = queue.GetType().GetProperty("BoundedCapacity")?.GetValue(queue) From 89c1d28a4485a75b0ef2ca9a6f15dafccc34ace8 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Mon, 26 Oct 2026 06:50:05 +0200 Subject: [PATCH 2/3] feat(backfiller): add SilentConsumerMonitor with comprehensive tests and documentation --- VectorNNTPD.slnx | 1 - docs/architecture.md | 86 +++ docs/configuration.md | 1 + .../ManagementQueueClient.cs | 303 ++++++++++ .../ManagementQueueSnapshot.cs | 79 +++ .../SilentConsumerMonitor/ObservationScope.cs | 79 +++ .../ObservationStateMachine.cs | 164 ++++++ .../QueueSampleHistory.cs | 128 +++++ .../SilentConsumerEvaluators.cs | 366 ++++++++++++ .../SilentConsumerMonitorDefaults.cs | 72 +++ .../SilentConsumerMonitorEngine.cs | 481 ++++++++++++++++ .../SilentConsumerMonitorHostedService.cs | 316 ++++++++++ .../SilentConsumerMonitorLogMessages.cs | 166 ++++++ .../SilentConsumerMonitorOptions.cs | 23 + .../SilentConsumerMonitorOptionsValidator.cs | 25 + .../SilentConsumerMonitorTargets.cs | 63 ++ .../SilentConsumerSessionObservation.cs | 58 ++ .../BackFillerServiceCollectionExtensions.cs | 29 +- .../ManagementQueueClientTests.cs | 390 +++++++++++++ .../ObservationStateMachineTests.cs | 45 ++ .../QueueSampleHistoryTests.cs | 150 +++++ ...lentConsumerDispatchStallLifecycleTests.cs | 520 +++++++++++++++++ .../SilentConsumerEvaluatorAndEngineTests.cs | 340 +++++++++++ ...entConsumerMonitorOptionsValidatorTests.cs | 22 + .../SilentConsumerMonitorTargetsTests.cs | 76 +++ .../SilentConsumerMultiInstanceTests.cs | 540 ++++++++++++++++++ .../Hosting/BackFillerHostCompositionTests.cs | 10 +- ...VectorNNTPD.BackFillerService.Tests.csproj | 1 + 28 files changed, 4530 insertions(+), 4 deletions(-) create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ManagementQueueClient.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ManagementQueueSnapshot.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationScope.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationStateMachine.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/QueueSampleHistory.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluators.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorDefaults.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorHostedService.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorLogMessages.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptions.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptionsValidator.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorTargets.cs create mode 100644 src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerSessionObservation.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ManagementQueueClientTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ObservationStateMachineTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/QueueSampleHistoryTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerDispatchStallLifecycleTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluatorAndEngineTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptionsValidatorTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorTargetsTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMultiInstanceTests.cs diff --git a/VectorNNTPD.slnx b/VectorNNTPD.slnx index ae1f5980..a10628c1 100644 --- a/VectorNNTPD.slnx +++ b/VectorNNTPD.slnx @@ -25,4 +25,3 @@ - diff --git a/docs/architecture.md b/docs/architecture.md index 21174eca..4473c344 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -24,6 +24,92 @@ Article Work shutdown uses `BackFillerShutdownOptions`: The consumer membership loop waits 15 seconds after a pass that started every desired consumer, including a pass with nothing to start. A pass that cannot declare topology because RabbitMQ is not ready, because channel creation or declaration failed for a desired backbone, or because Management-plane quorum membership is deficient or unverifiable, does not reset that delay. The next wait doubles, up to 2 minutes. An unexpected exception uses the same delay and does not end the loop. Shutdown cancellation ends the loop and is not a failure. The host stays healthy while RabbitMQ is temporarily unavailable. Capacity snapshots and connection replacement reconcile once on the shared gate and do not change the delay. If replacement cannot restore a session, the same periodic loop retries it. A later successful pass resets the delay to 15 seconds. +Active-processing stalls (`IsActiveProcessingStalled` / Event 5328) are remediated only by that reconcile path (`RetireForRecoveryAsync`). Silent-consumer monitoring may observe the same predicate but does not introduce a second recovery action. + +## Silent-consumer monitoring (in-process) + +Monitoring runs inside each `BackFillerService` process as `SilentConsumerMonitorHostedService`. There is no separate monitor executable. Multiple concurrent BackFiller instances consuming the same provider queues are the intended operating model. The monitor is observational only: it does not call `RetireForRecoveryAsync` or otherwise compete with Article Work reconcile recovery (Event 5328). + +### RabbitMQ configuration sources (traced) + +| Setting | Authoritative source | Runtime update | +|---|---|---| +| AMQP `Hosts`, `Port`, `VirtualHost`, TLS | Bound `RabbitMQ` section (`VectorNNTPD.RabbitMQ.json` / env). **Not** columns in `nntpsharedconfig`. | Captured into `RabbitMqService`'s start-time `RabbitMqRuntimeOptions` for logging, reconnect backoff, and related fields. Each `ConnectAndInstallAsync` dials via `ConnectAsync(_options.Value, …)` so the factory builds from the **live** `RabbitMqOptions` at that attempt. Hosts/vhost/TLS are not database-managed; changing the bound JSON/env still requires a process restart to pick up new endpoints in practice (they are not projected from MySQL). | +| Management `BaseUrl`, `RequestTimeoutSeconds` | `RabbitMQ:Management` in the same section. Distinct from AMQP port 5672. | Read live from `IOptions` on each Management HTTP call (monitor and quorum probe). | +| Username / password | MySQL `nntpsharedconfig.rabbitmqusername` / `rabbitmqpassword` via `ProviderAccountConfigurationService.Project` → `NntpSharedConfiguration.CopyRabbitMqCredentialsTo`. Bound JSON/env credentials are discarded at options PostConfigure. | Written onto the live `RabbitMqOptions` instance each account refresh. Management clients see new credentials on the next HTTP request. AMQP uses those live credentials on the **next** connection attempt (`ConnectAndInstallAsync` → factory `ToRuntimeOptions()` from `_options.Value`); an already-open connection is not re-authenticated in place. | +| Article Work queues | MySQL provider catalog (`nntpbackfilleraccounts` → `ProviderConfigurationCatalog`). Queue name = `BackFillerRabbitMqTopology.ComposeProviderEntity(backbone)` → `backfiller.{normalized}`. | Catalog `Publish` replaces the list; monitor resolves targets every cycle. | +| Expected capacity for shared ConsumerLoss | `MaxSessions > 0` and `IBackboneUsableCapacityProvider.HasUsableCapacityForBackbone` (same gate as `BuildDesiredSessions`). | Capacity snapshots publish from `NntpProviderRegistry`; monitor re-reads each cycle. | + +There is no database column for Management URL or AMQP hosts in this repository. The monitor does not invent Management from the AMQP host/port. Missing `RabbitMQ:Management:BaseUrl` or credentials yields a clear unavailable diagnostic; instance-local monitoring continues. + +Optional public switch only: `BackFiller:SilentConsumerMonitor:Enabled` (default true). Detection windows live in `SilentConsumerMonitorDefaults` (`ProcessingStall` / `DispatchStall` / `ConsumerLoss`). + +### Observation pipeline + +Each monitor cycle: project live `ArticleWorkConsumerSession` counters via `SilentConsumerSessionObservation.FromSession` → `SilentConsumerMonitorEngine.ObserveInstance` → (when Management is configured) `ManagementQueueClient.GetQueueAsync` → `ObserveSharedQueue`. Instance evaluation always runs before the Management gate, so missing BaseUrl, missing credentials, or Management HTTP failures do not disable local Detection. Opened / Continued / Resolved transitions emit structured events 9110–9112 with `scope`, `condition`, `target` (session key or queue name), and `instanceId`. + +| Condition | Observation source | Open window / resolve | Alert identity | Opens when | Resolves when | +|---|---|---|---|---|---| +| Instance ProcessingStall | Session `IsActiveProcessingStalled` (progress clock; same predicate as Event 5328) | 90s / 20s | `Instance` + session key | Running and actively stalled | Predicate clears for resolve window, or session removed | +| Instance DispatchStall | Session `QueuedCount` / `ActiveCount` from admit → `_dispatch` → `_active++` | 90s / 20s | `Instance` + session key | Running, queued > 0, active == 0 | Queued clears / active advances for resolve window, or session removed | +| Instance ConsumerLoss | Desired slot count vs Running+open-channel sessions on this process | 45s / 20s | `Instance` + `local` | Desired > 0 and zero usable local consumers | A local consumer returns, or desired becomes 0 | +| Shared ProcessingStall | Management unacked + near-zero ack rate with consumers ≥ 1 | 90s / 20s | `SharedQueue` + queue name | Aggregate looks stalled | Aggregate progress returns, or target pruned | +| Shared DispatchStall | Management ready backlog + near-zero deliver/ack | 90s / 20s | `SharedQueue` + queue name | Aggregate looks stalled | Aggregate progress returns, or target pruned | +| Shared ConsumerLoss | Management `consumers == 0` + ready floor while expected-active | 45s / 20s | `SharedQueue` + queue name | Fleet-wide empty consumers with backlog | A consumer reappears, capacity not expected, or target pruned | + +### Dispatch-acquisition failure mode + +Admission increments `_inFlight` before the dispatch lock. `QueuedCount` is `_inFlight - _active`. The session waits on `_dispatch`, optionally on the test seam `DispatchAcquireHold`, then increments `_active` and enters the pipeline. A hold (or any block) after the lock and before `_active++` leaves the consumer registered, the channel open, local work queued, and `IsActiveProcessingStalled` false because active processing never started. Instance DispatchStall covers that gap; active-processing recovery (Event 5328) remains authoritative once `_active > 0`. + +Legitimate idleness is `QueuedCount == 0` (and typically `ActiveCount == 0`): no DispatchStall. Broker ready count is not used for instance DispatchStall. + +### Per-instance health + +Local session projections only (no Management call). Events include `instanceId` = runtime FQDN. ProcessingStall and DispatchStall are keyed by the canonical session identity `ArticleWorkConsumerSession.ComposeSessionKey` (`backbone/connectionNumber`) so multiple `MaxSessions` slots on the same backbone — and distinct backbones — keep independent observation state. Shared-queue observations remain keyed by queue name only. Removing one session applies Inactive `"session removed"` only to that session key. + +| Condition | Investigation label | Predicate | Idle vs fault | +|---|---|---|---| +| ProcessingStall | D | Running session with `IsActiveProcessingStalled` (same as reconcile) | Idle sessions never stall (`HasActiveProcessing` false). | +| DispatchStall | D2 | Running, `QueuedCount > 0`, `ActiveCount == 0` (dispatch hold before `_active++`, including `DispatchAcquireHold`) | `QueuedCount == 0` is legitimate idle, not a stall. | +| ConsumerLoss | E | Desired slots (usable capacity × MaxSessions) > 0 and zero Running sessions with an open channel on **this** process | Desired=0 → inactive. | + +A healthy sibling session on the same process, or a sibling instance advancing aggregate ACK/deliver rates, cannot clear these local observations. + +### Shared queue health + +Management polling of catalog-derived queues when BaseUrl and projected credentials exist. Aggregate gauges/rates describe the queue, not a consumer identity. A healthy sibling keeps shared ProcessingStall/DispatchStall/ConsumerLoss inactive while still leaving local stalls visible on the broken instance. Shared ConsumerLoss requires `consumers == 0`, ready backlog, and expected-active capacity on this instance. Idle or capacity-empty queues are not faults. + +Every polling instance may emit the same shared observation independently (**no cross-process deduplication**). Removing a provider from the catalog emits an explicit Resolved event for any open shared observation on that queue (target gone — not broker recovery), then drops history; reintroducing the provider recreates fresh history and state machines (next Open is a new Opened, not Continued). In-memory alert state is per process and is lost on restart. + +### Management outages and `Unknown` + +Two distinct unavailability paths: + +1. **Management not ready to poll** (missing `BaseUrl` or credentials): HostedService skips `ObserveSharedQueue` and emits Event 9123 once per transition into unavailable. No `Unknown` signal is applied; any already-open shared alerts simply remain as last evaluated. +2. **Management polled but sample unusable** (HTTP auth/timeout/transport/incomplete gauges): `ObserveSharedQueue` records a failure sample; shared evaluators return `ConditionSignal.Unknown`. The state machine neither opens nor resolves on `Unknown`. Event 9113 may log (rate-limited per observation key = scope + condition + queue/session target, interval = ProcessingStall open window) that an already-open alert received Unavailable/`Unknown` and was retained — not that Management was never configured. + +Consequences: + +- An already-`Open` shared alert **remains Open** for the duration of a Management outage or while polls return non-success. That freeze is intentional: lack of observation must not be treated as recovery or as proof the underlying queue condition cleared. +- An `Open` shared alert during an outage does **not** establish that the broker condition is still present; it only means the last successful evaluation left the alert open and no conclusive Inactive signal has been observed since. +- `Unknown` is not healthy and is not recovery. +- Instance-local monitoring continues on every cycle before the Management gate (`ObserveInstance` always runs). Missing BaseUrl, missing credentials, or HTTP failures do not disable local ProcessingStall / DispatchStall / ConsumerLoss. +- When valid Management samples resume, shared conditions are reevaluated: Active continues an open alert, Inactive starts the resolve window, and catalog prune still emits Resolved with the target-removed reason (monitoring-scope change, not broker recovery). +- Event 9113 / Continued rate-limit state is keyed per observation identity (not global across queues or conditions). Shared prune removes that rate-limit entry with the machine. + +### Known limitations + +- Shared metrics cannot identify which consumer is unhealthy; they only describe the queue aggregate. +- Duplicate shared Opens across BackFiller instances are expected. +- Shared alerts may remain `Open` while Management observations are unavailable (`Unknown` or no poll); operators must not read that freeze as confirmed ongoing fault or as recovery. +- Instance DispatchStall sees only this process’s admitted queue, not another instance’s waiting work on the same broker queue. +- `MaxSessions > 0` with usable capacity means desired consume slots exist; it does not prove a broker consumer tag is present until the session reaches Running with an open channel. +- Monitor observations do not recover sessions; reconcile remains the only remediation for Event 5328. + +### Isolation + +Monitor HTTP never enters Article Work dispatch, settlement, NNTP, or ACK/NACK paths. Failed/missing metrics are `Unknown` (never invented as zero). The monitor does not change reconcile, topology, or logging retention. + Provider queue type is derived from the configured unique `RabbitMQ:Hosts` count (never from reachable hosts or live cluster members). Exactly one host declares a durable classic queue (`x-queue-type=classic`) without quorum arguments or Management quorum health. Two or more hosts declare a durable quorum queue with `x-quorum-initial-group-size` and readiness membership floor equal to that host count, plus configured `x-delivery-limit`. All configured hosts must belong to the same RabbitMQ cluster; the client host list does not establish membership. Successful AMQP quorum declaration alone does not mark a backbone ready: readiness also requires Management HTTP verification of members, online members, and leader. Pre-existing queues with inequivalent type or immutable arguments are not deleted or recreated; operators must migrate or grow membership explicitly. Changing `Hosts` from one entry to multiple (or the reverse) changes the desired queue type and is not a safe in-place conversion. Management outages and timeouts leave multi-host backbones unready and follow the same reconcile backoff. ## Article Work settlement diff --git a/docs/configuration.md b/docs/configuration.md index 253065ad..95109e24 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -58,6 +58,7 @@ Nested groups: - `BackFiller:Nntp` — `MaxConcurrentSessionEstablishments` default 8, range 1–1024. Zero is invalid. This does not replace per-provider `MaxSessions` from MySQL. `SendBufferBytes` default 1048576 (1 MiB) and `ReceiveBufferBytes` default 2097152 (2 MiB) are the upstream TCP buffer limits. Each is validated from 16384 through 16777216. They are independent of each other and of the 64 KiB managed reader buffer. Nagle is disabled in code. One upstream session at the defaults requests 3 MiB of kernel buffering. Steady-state session count remains the sum of per-provider `MaxSessions`. These buffers do not raise the establishment gate. The effective window still depends on the operating system, autotuning, TLS, and the path. - `BackFiller:TcpKeepAlive` — TCP `SO_KEEPALIVE` for every upstream NNTP socket and every accepted cache-listener socket. `TimeSeconds` default 10 (range 1–7200) is idle time before the first probe. `IntervalSeconds` default 5 (range 1–300) is the gap between probes. `RetryCount` default 3 (range 1–20) is unanswered probes before the stack drops the connection. At the defaults, dead-path detection completes in about 25 s, before the default NNTP command timeout (30 s) and well before the listener I/O-progress timeout (60 s). Independent of per-provider MySQL `keepalive` (NNTP `DATE`). Nagle and buffer settings are unchanged. Platforms that cannot set a probe option still enable `SO_KEEPALIVE` and keep OS defaults for that option; enabling keepalive itself must succeed or the socket is rejected. - `BackFiller:Systemd` — notify and watchdog enablement +- `BackFiller:SilentConsumerMonitor` — optional. Only `Enabled` (default true) is public. Do not place VirtualHost, Hosts, Management URL, or queue lists here. AMQP hosts/vhost/port and Management BaseUrl bind from the `RabbitMQ` section; username/password are projected from `nntpsharedconfig` onto live `RabbitMqOptions` on account refresh (Management uses them on the next HTTP call; AMQP uses them on the next reconnect dial). Article Work queues come from the MySQL provider catalog as `backfiller.{backbone}`. Detection thresholds are internal code defaults (`ProcessingStall` / `DispatchStall` / `ConsumerLoss`). Management poll failures yield `Unknown` and do not resolve open shared alerts; instance-local monitoring continues. See `docs/architecture.md` § Silent-consumer monitoring (including Management outages and `Unknown`) for the traced data flow and multi-instance semantics. Monitor failures never disable consumption. ## RabbitMQ section diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ManagementQueueClient.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ManagementQueueClient.cs new file mode 100644 index 00000000..08bdc42d --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ManagementQueueClient.cs @@ -0,0 +1,303 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Options; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Read-only RabbitMQ Management HTTP client for + /// GET /api/queues/{vhost}/{queue}. Classifies success and failures into + /// without throwing for transport/auth/parse outcomes. + /// Never logs username or password. + /// + internal sealed class ManagementQueueClient + { + /// + /// Named client registered for this monitor + /// (infinite HttpClient timeout; per-request cancel enforces Management timeout). + /// + internal const string HttpClientName = "VectorNNTPD.BackFiller.SilentConsumerMonitor"; + + /// Factory used to create the named Management HTTP client per request. + private readonly IHttpClientFactory _httpClientFactory; + + /// + /// Live RabbitMQ options; is read on each poll so + /// Management BaseUrl, RequestTimeoutSeconds, Username, and Password reflect account-refresh projection. + /// + private readonly IOptions _rabbitOptions; + + /// Clock used to stamp on successful parses. + private readonly TimeProvider _time; + + /// + /// Creates a Management queue client that polls via the named HTTP client and live RabbitMQ options. + /// + /// + /// Factory used to create for each call. + /// + /// + /// Options monitor whose current supply Management BaseUrl, + /// RequestTimeoutSeconds (clamped 1–60, default + /// ), Username, and Password. + /// Values are read at call time, not cached in the constructor. + /// + /// + /// Time provider used only to set when parsing + /// a successful JSON document; not used for HTTP timeout (timeout uses a linked CTS). + /// + /// + /// Thrown when , , or + /// is . + /// + public ManagementQueueClient( + IHttpClientFactory httpClientFactory, + IOptions rabbitOptions, + TimeProvider time) + { + ArgumentNullException.ThrowIfNull(httpClientFactory); + ArgumentNullException.ThrowIfNull(rabbitOptions); + ArgumentNullException.ThrowIfNull(time); + _httpClientFactory = httpClientFactory; + _rabbitOptions = rabbitOptions; + _time = time; + } + + /// + /// Fetches one queue document using live (BaseUrl, timeout, username/password). + /// + /// RabbitMQ virtual host (URI-escaped in the request path). + /// Queue entity name (URI-escaped in the request path). + /// + /// Caller cancellation. Linked with a per-request timeout CTS; timeout alone yields + /// ; caller cancel yields + /// . + /// + /// + /// Classified poll result. Success carries a snapshot (gauges may still be incomplete). + /// Missing BaseUrl/credentials, HTTP errors, timeouts, and parse failures return failure kinds + /// without throwing. + /// + /// + /// Thrown when or is null, empty, or whitespace. + /// + /// + /// Builds Basic auth from the options instance at call time so account-refresh credential projection + /// applies on the next request. Does not log username or password. Missing BaseUrl or credentials + /// yield without throwing. + /// + public async Task GetQueueAsync( + string virtualHost, + string queueName, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(virtualHost); + ArgumentException.ThrowIfNullOrWhiteSpace(queueName); + + if (!TryResolve(out var baseUrl, out var username, out var password, out var failure)) + { + return Fail(ManagementPollKind.TransientFailure, failure); + } + + var configuredTimeout = _rabbitOptions.Value.Management?.RequestTimeoutSeconds + ?? SilentConsumerMonitorDefaults.ManagementHttpTimeoutSeconds; + var timeoutSeconds = Math.Clamp(configuredTimeout, 1, 60); + try + { + var client = _httpClientFactory.CreateClient(HttpClientName); + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(TimeSpan.FromSeconds(timeoutSeconds)); + var uri = BuildQueueUri(baseUrl, virtualHost, queueName); + using var request = new HttpRequestMessage(HttpMethod.Get, uri); + request.Headers.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.ASCII.GetBytes(username + ":" + password))); + + using var response = await client + .SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutCts.Token) + .ConfigureAwait(false); + + if (response.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden) + { + return Fail(ManagementPollKind.AuthenticationFailure, $"HTTP {(int)response.StatusCode}"); + } + + if (response.StatusCode == HttpStatusCode.NotFound) + { + return Fail(ManagementPollKind.QueueMissing, "Queue missing."); + } + + if (!response.IsSuccessStatusCode) + { + return Fail(ManagementPollKind.TransientFailure, $"HTTP {(int)response.StatusCode}"); + } + + await using var stream = await response.Content.ReadAsStreamAsync(timeoutCts.Token).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: timeoutCts.Token) + .ConfigureAwait(false); + var snapshot = Parse(document.RootElement, virtualHost, queueName, _time.GetUtcNow()); + return new ManagementPollResult + { + Kind = ManagementPollKind.Success, + Reason = snapshot.HasRequiredGauges ? "ok" : "incomplete gauges", + Snapshot = snapshot, + }; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + return Fail(ManagementPollKind.TransientFailure, "timeout"); + } + catch (OperationCanceledException) + { + return Fail(ManagementPollKind.Cancelled, "cancelled"); + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or IOException or InvalidOperationException) + { + return Fail(ManagementPollKind.TransientFailure, ex.GetType().Name); + } + } + + /// Builds the Management queue URI without embedding credentials. + /// Management HTTP base URL (trailing slash trimmed). + /// Virtual host segment (URI-escaped). + /// Queue name segment (URI-escaped). + /// {base}/api/queues/{vhost}/{queue} as an absolute . + internal static Uri BuildQueueUri(string baseUrl, string vhost, string queueName) => + new($"{baseUrl.TrimEnd('/')}/api/queues/{Uri.EscapeDataString(vhost)}/{Uri.EscapeDataString(queueName)}"); + + /// + /// Parses Management queue JSON gauges and rates without inventing zeros for missing fields. + /// + /// Root object of the queue document. + /// Virtual host stamped onto the snapshot. + /// Queue name stamped onto the snapshot. + /// Sample timestamp stamped onto the snapshot. + /// + /// Snapshot with nullable gauges/rates; missing or invalid JSON properties remain . + /// + internal static ManagementQueueSnapshot Parse( + JsonElement root, + string virtualHost, + string queueName, + DateTimeOffset atUtc) + { + long? ackTotal = null; + long? deliverGetTotal = null; + double? ackRate = null; + double? deliverGetRate = null; + if (root.TryGetProperty("message_stats", out var stats) && stats.ValueKind == JsonValueKind.Object) + { + ackTotal = TryInt64(stats, "ack"); + deliverGetTotal = TryInt64(stats, "deliver_get"); + ackRate = TryRate(stats, "ack_details"); + deliverGetRate = TryRate(stats, "deliver_get_details") ?? TryRate(stats, "deliver_details"); + } + + return new ManagementQueueSnapshot + { + QueueName = queueName, + VirtualHost = virtualHost, + SampledAtUtc = atUtc, + MessagesReady = TryInt64(root, "messages_ready"), + MessagesUnacknowledged = TryInt64(root, "messages_unacknowledged"), + Consumers = TryInt64(root, "consumers"), + AckRate = ackRate, + DeliverGetRate = deliverGetRate, + AckTotal = ackTotal, + DeliverGetTotal = deliverGetTotal, + }; + } + + /// + /// Reads current for Management BaseUrl and AMQP Username/Password. + /// + /// Configured Management BaseUrl when resolution succeeds; otherwise empty. + /// Configured Username when resolution succeeds; otherwise empty. + /// Configured Password when resolution succeeds; otherwise empty. + /// Human-readable reason when resolution fails; empty on success. + /// + /// when BaseUrl is non-whitespace and Username/Password are present; + /// otherwise (caller maps this to ). + /// + private bool TryResolve(out string baseUrl, out string username, out string password, out string failure) + { + var rabbit = _rabbitOptions.Value; + var baseCandidate = rabbit.Management?.BaseUrl; + if (string.IsNullOrWhiteSpace(baseCandidate)) + { + baseUrl = string.Empty; + username = string.Empty; + password = string.Empty; + failure = "Management BaseUrl is not configured."; + return false; + } + + if (string.IsNullOrWhiteSpace(rabbit.Username) || string.IsNullOrEmpty(rabbit.Password)) + { + baseUrl = string.Empty; + username = string.Empty; + password = string.Empty; + failure = "Management credentials are not configured."; + return false; + } + + baseUrl = baseCandidate; + username = rabbit.Username; + password = rabbit.Password; + failure = string.Empty; + return true; + } + + /// Builds a failure with no snapshot. + /// Failure classification. + /// Non-secret diagnostic reason. + /// Poll result with unset. + private static ManagementPollResult Fail(ManagementPollKind kind, string reason) => + new() { Kind = kind, Reason = reason }; + + /// Reads a JSON number property as , or null if absent/unconvertible. + /// Object that may contain . + /// Property name. + /// Parsed integer, truncated double in long range, or . + private static long? TryInt64(JsonElement parent, string name) + { + if (!parent.TryGetProperty(name, out var value)) + { + return null; + } + + return value.ValueKind switch + { + JsonValueKind.Number when value.TryGetInt64(out var i) => i, + JsonValueKind.Number when value.TryGetDouble(out var d) + && d is >= long.MinValue and <= long.MaxValue => (long)d, + _ => null, + }; + } + + /// + /// Reads {detailsName}.rate from message_stats when present and finite; otherwise null. + /// + /// message_stats object. + /// Details object name (for example ack_details). + /// Finite rate value, or when missing/NaN/Infinity. + private static double? TryRate(JsonElement stats, string detailsName) + { + if (!stats.TryGetProperty(detailsName, out var details) + || details.ValueKind != JsonValueKind.Object + || !details.TryGetProperty("rate", out var rate) + || rate.ValueKind != JsonValueKind.Number + || !rate.TryGetDouble(out var value) + || double.IsNaN(value) + || double.IsInfinity(value)) + { + return null; + } + + return value; + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ManagementQueueSnapshot.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ManagementQueueSnapshot.cs new file mode 100644 index 00000000..d82c5ded --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ManagementQueueSnapshot.cs @@ -0,0 +1,79 @@ +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// Queue-level Management metrics. Missing fields stay null (never invent zeros). + internal sealed class ManagementQueueSnapshot + { + /// Queue entity name from the Management request. + public required string QueueName { get; init; } + + /// Virtual host from the Management request. + public required string VirtualHost { get; init; } + + /// UTC sample time from the engine at parse time. + public required DateTimeOffset SampledAtUtc { get; init; } + + /// Broker messages_ready gauge when present; otherwise null (never invented as zero). + public long? MessagesReady { get; init; } + + /// Broker messages_unacknowledged gauge when present; otherwise null. + public long? MessagesUnacknowledged { get; init; } + + /// Broker consumers gauge when present; otherwise null. + public long? Consumers { get; init; } + + /// Broker message_stats.ack_details.rate when present and finite; otherwise null. + public double? AckRate { get; init; } + + /// + /// Broker deliver/get rate from deliver_get_details.rate or deliver_details.rate when present; otherwise null. + /// + public double? DeliverGetRate { get; init; } + + /// Cumulative message_stats.ack counter when present; used to derive rates across samples. + public long? AckTotal { get; init; } + + /// Cumulative message_stats.deliver_get counter when present; used to derive rates across samples. + public long? DeliverGetTotal { get; init; } + + /// + /// True when , , and are all non-null. + /// + public bool HasRequiredGauges => + Consumers is not null && MessagesReady is not null && MessagesUnacknowledged is not null; + } + + /// Management poll classification. + internal enum ManagementPollKind + { + /// Parsed document. + Success = 0, + + /// HTTP 404. + QueueMissing = 1, + + /// 401/403. + AuthenticationFailure = 2, + + /// Timeout, transport, HTTP, or parse failure. + TransientFailure = 3, + + /// Caller cancel. + Cancelled = 4, + } + + /// One Management poll result (success snapshot or classified failure). + internal sealed class ManagementPollResult + { + /// Success vs missing queue, auth failure, transient, or cancel. + public required ManagementPollKind Kind { get; init; } + + /// Non-secret reason suitable for diagnostics (never credentials). + public required string Reason { get; init; } + + /// Parsed gauges when is Success; otherwise null. + public ManagementQueueSnapshot? Snapshot { get; init; } + + /// True when Kind is Success and a snapshot is present. + public bool IsSuccess => Kind == ManagementPollKind.Success && Snapshot is not null; + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationScope.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationScope.cs new file mode 100644 index 00000000..13ca3135 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationScope.cs @@ -0,0 +1,79 @@ +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// Whether an observation describes this process or a shared queue aggregate. + internal enum ObservationScope + { + /// Derived from this instance's consumer sessions / stall clock. + Instance = 1, + + /// Derived from Management queue gauges/rates (multi-consumer aggregate). + SharedQueue = 2, + } + + /// + /// Operational silent-consumer condition. + /// Investigation labels D / D2 / E map to ProcessingStall / DispatchStall / ConsumerLoss. + /// + internal enum SilentConsumerCondition + { + /// In-flight active processing stall (investigation Scenario D). + ProcessingStall = 1, + + /// Dispatch / queued-before-active stall (investigation Scenario D2). + DispatchStall = 2, + + /// Consumer loss with backlog (investigation Scenario E). + ConsumerLoss = 3, + } + + /// Alert / observation lifecycle phase. + internal enum ObservationPhase + { + /// Condition clear. + Clear = 0, + + /// Condition open. + Open = 1, + } + + /// Condition signal for one evaluation tick. + internal enum ConditionSignal + { + /// Cannot evaluate; must not open or resolve. + Unknown = 0, + + /// Condition present. + Active = 1, + + /// Condition absent. + Inactive = 2, + } + + /// State-machine transition. + internal enum ObservationTransition + { + /// No change. + None = 0, + + /// Newly opened. + Opened = 1, + + /// Still open. + Continued = 2, + + /// Resolved. + Resolved = 3, + } + + /// Stable observation identity within one process. + /// Instance vs shared queue. + /// ProcessingStall, DispatchStall, or ConsumerLoss. + /// + /// Shared: queue name. Instance ProcessingStall/DispatchStall: canonical session key + /// (backbone/connectionNumber). Instance ConsumerLoss: the process-scoped token local. + /// + internal readonly record struct ObservationKey( + ObservationScope Scope, + SilentConsumerCondition Condition, + string QueueOrBackbone); +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationStateMachine.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationStateMachine.cs new file mode 100644 index 00000000..f2bcc527 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/ObservationStateMachine.cs @@ -0,0 +1,164 @@ +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Open/continue/resolve machine for one . + /// + /// + /// neither opens nor resolves: an already-open alert stays open + /// so a Management outage is not treated as recovery. Sustained + /// past the configured open window opens; sustained past the + /// resolve window resolves. Persistence clocks reset when the opposite conclusive signal arrives. + /// + internal sealed class ObservationStateMachine + { + /// Stable observation identity owned by this machine. + private readonly ObservationKey _key; + + /// Minimum sustained Active duration before opening. + private readonly TimeSpan _openWindow; + + /// Minimum sustained Inactive duration before resolving while open. + private readonly TimeSpan _resolveWindow; + + /// Clear until opened; Open until resolved. + private ObservationPhase _phase; + + /// UTC start of the current Active persistence clock; null when not accumulating Active. + private DateTimeOffset? _activeSinceUtc; + + /// UTC start of the current Inactive persistence clock while open; null otherwise. + private DateTimeOffset? _inactiveSinceUtc; + + /// Creates a machine for one observation identity. + /// Stable identity (instance session key, local, or shared queue name). + /// Minimum sustained Active duration before . + /// Minimum sustained Inactive duration before . + public ObservationStateMachine(ObservationKey key, TimeSpan openWindow, TimeSpan resolveWindow) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(openWindow, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(resolveWindow, TimeSpan.Zero); + _key = key; + _openWindow = openWindow; + _resolveWindow = resolveWindow; + } + + /// Current phase after the last conclusive apply. + public ObservationPhase Phase => _phase; + + /// + /// Applies one evaluation signal at . + /// + /// Active, Inactive, or Unknown. + /// Evaluation instant from the engine . + /// Non-secret diagnostic text stamped on the result. + /// Transition for this tick; Unknown always yields . + public ObservationResult Apply(ConditionSignal signal, DateTimeOffset atUtc, string? detail = null) + { + switch (signal) + { + case ConditionSignal.Unknown: + return Result(signal, ObservationTransition.None, CurrentPersistence(atUtc), detail); + + case ConditionSignal.Active: + _inactiveSinceUtc = null; + _activeSinceUtc ??= atUtc; + var activeFor = atUtc - _activeSinceUtc.Value; + if (_phase == ObservationPhase.Clear) + { + if (activeFor >= _openWindow) + { + _phase = ObservationPhase.Open; + return Result(signal, ObservationTransition.Opened, activeFor, detail); + } + + return Result(signal, ObservationTransition.None, activeFor, detail); + } + + return Result(signal, ObservationTransition.Continued, activeFor, detail); + + case ConditionSignal.Inactive: + _activeSinceUtc = null; + if (_phase == ObservationPhase.Clear) + { + _inactiveSinceUtc = null; + return Result(signal, ObservationTransition.None, TimeSpan.Zero, detail); + } + + _inactiveSinceUtc ??= atUtc; + var inactiveFor = atUtc - _inactiveSinceUtc.Value; + if (inactiveFor >= _resolveWindow) + { + _phase = ObservationPhase.Clear; + _inactiveSinceUtc = null; + return Result(signal, ObservationTransition.Resolved, inactiveFor, detail); + } + + return Result(signal, ObservationTransition.Continued, inactiveFor, detail); + + default: + return Result(ConditionSignal.Unknown, ObservationTransition.None, TimeSpan.Zero, "unhandled"); + } + } + + /// Active or Inactive persistence relative to , or zero when neither clock is set. + /// Evaluation instant. + /// Elapsed persistence for the active clock, inactive clock, or . + private TimeSpan CurrentPersistence(DateTimeOffset atUtc) + { + if (_activeSinceUtc is { } a) + { + return atUtc - a; + } + + if (_inactiveSinceUtc is { } i) + { + return atUtc - i; + } + + return TimeSpan.Zero; + } + + /// Builds an stamped with this machine's key and current phase. + /// Signal applied this tick. + /// Transition produced this tick. + /// Persistence duration for diagnostics. + /// Optional non-secret detail. + /// Immutable result for emit/tests. + private ObservationResult Result( + ConditionSignal signal, + ObservationTransition transition, + TimeSpan persistence, + string? detail) => + new() + { + Key = _key, + Signal = signal, + Transition = transition, + Phase = _phase, + Persistence = persistence, + Detail = detail, + }; + } + + /// One evaluation result emitted to diagnostics and tests. + internal sealed class ObservationResult + { + /// Observation identity that owns the state machine. + public required ObservationKey Key { get; init; } + + /// Signal applied this tick. + public required ConditionSignal Signal { get; init; } + + /// Opened, Continued, Resolved, or None. + public required ObservationTransition Transition { get; init; } + + /// Phase after this apply (Clear or Open). + public required ObservationPhase Phase { get; init; } + + /// How long the current Active or Inactive persistence clock has run. + public required TimeSpan Persistence { get; init; } + + /// Non-secret evaluator or prune detail for logs. + public string? Detail { get; init; } + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/QueueSampleHistory.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/QueueSampleHistory.cs new file mode 100644 index 00000000..6613ac9d --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/QueueSampleHistory.cs @@ -0,0 +1,128 @@ +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// One sample history entry (success snapshot or classified failure). + /// Sample timestamp (snapshot time on success; caller-supplied on failure). + /// True when a usable snapshot with required gauges is present. + /// Parsed snapshot on success attempts; null on explicit failures. + /// Failure kind when invalid; null on valid success. + /// Non-secret failure or incomplete-gauges reason when invalid. + internal readonly record struct QueueSampleEntry( + DateTimeOffset AtUtc, + bool IsValid, + ManagementQueueSnapshot? Snapshot, + ManagementPollKind? FailureKind, + string? FailureReason); + + /// Bounded ring of Management samples for one queue. + internal sealed class QueueSampleHistory + { + /// Fixed-capacity ring buffer storage. + private readonly QueueSampleEntry[] _entries; + + /// Number of retained entries (≤ capacity). + private int _count; + + /// Next write index in the ring. + private int _next; + + /// Creates history with fixed capacity (minimum 2). + /// Ring capacity; must be ≥ 2. + /// Thrown when is less than 2. + public QueueSampleHistory(int capacity) + { + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 2); + _entries = new QueueSampleEntry[capacity]; + } + + /// Number of retained samples in the ring. + public int Count => _count; + + /// Appends a success sample; incomplete gauges are stored as invalid with reason. + /// Parsed Management snapshot (required). + /// Thrown when is null. + public void AddSuccess(ManagementQueueSnapshot snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + Push(new QueueSampleEntry( + snapshot.SampledAtUtc, + snapshot.HasRequiredGauges, + snapshot, + null, + snapshot.HasRequiredGauges ? null : "incomplete gauges")); + } + + /// Appends an explicit failure (no synthetic zeros). + /// Failure timestamp. + /// Management poll failure kind. + /// Non-secret reason text. + public void AddFailure(DateTimeOffset atUtc, ManagementPollKind kind, string reason) => + Push(new QueueSampleEntry(atUtc, false, null, kind, reason)); + + /// Returns an oldest-to-newest copy of retained samples. + /// Empty array when empty; otherwise a new array of length . + public IReadOnlyList Snapshot() + { + if (_count == 0) + { + return Array.Empty(); + } + + var result = new QueueSampleEntry[_count]; + var start = (_next - _count + _entries.Length) % _entries.Length; + for (var i = 0; i < _count; i++) + { + result[i] = _entries[(start + i) % _entries.Length]; + } + + return result; + } + + /// + /// Derives a per-second counter rate between two valid samples; null on reset/gap/missing. + /// + /// Earlier sample. + /// Later sample. + /// Selects the cumulative counter from each snapshot. + /// + /// (newer - older) / elapsedSeconds when both samples are valid, counters present, + /// elapsed > 0, and the counter did not decrease; otherwise . + /// + public static double? TryComputeCounterRate( + QueueSampleEntry older, + QueueSampleEntry newer, + Func select) + { + if (!older.IsValid || !newer.IsValid || older.Snapshot is null || newer.Snapshot is null) + { + return null; + } + + var a = select(older.Snapshot); + var b = select(newer.Snapshot); + if (a is null || b is null) + { + return null; + } + + var elapsed = (newer.AtUtc - older.AtUtc).TotalSeconds; + if (elapsed <= 0 || b.Value < a.Value) + { + return null; + } + + return (b.Value - a.Value) / elapsed; + } + + /// Writes into the ring, overwriting the oldest when full. + /// Sample to retain. + private void Push(QueueSampleEntry entry) + { + _entries[_next] = entry; + _next = (_next + 1) % _entries.Length; + if (_count < _entries.Length) + { + _count++; + } + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluators.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluators.cs new file mode 100644 index 00000000..459e298a --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluators.cs @@ -0,0 +1,366 @@ +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Condition predicates. Instance ProcessingStall defers remediation to existing stall recovery; + /// shared metrics never claim a specific consumer identity and must not mask a local stall. + /// + internal static class SilentConsumerEvaluators + { + /// + /// Instance ProcessingStall for one session: active pipeline stalled past threshold + /// (same predicate as reconcile). Observation only — does not call RetireForRecoveryAsync. + /// + /// Projected session for this tick. + /// Non-secret diagnostic text for logs/tests. + /// when Running and stalled; otherwise Inactive. + public static ConditionSignal EvaluateInstanceProcessingStall( + SilentConsumerSessionObservation session, + out string detail) + { + if (session.State != ArticleWorkConsumerState.Running || !session.IsActiveProcessingStalled) + { + detail = $"session={session.SessionKey} no active-processing stall"; + return ConditionSignal.Inactive; + } + + detail = + $"session={session.SessionKey} active={session.ActiveCount}; " + + "remediation owned by ArticleWork reconcile (Event 5328); " + + "sibling instance aggregate ACK progress does not clear this"; + return ConditionSignal.Active; + } + + /// + /// Instance DispatchStall for one session: admitted/queued work holding dispatch before + /// _active increments. Legitimate idle sessions have QueuedCount == 0 and are inactive. + /// + /// Projected session for this tick. + /// Non-secret diagnostic text for logs/tests. + /// + /// Active when Running with QueuedCount > 0 and ActiveCount == 0; otherwise Inactive. + /// + public static ConditionSignal EvaluateInstanceDispatchStall( + SilentConsumerSessionObservation session, + out string detail) + { + if (session.State != ArticleWorkConsumerState.Running) + { + detail = $"session={session.SessionKey} not running"; + return ConditionSignal.Inactive; + } + + if (session.QueuedCount <= 0 || session.ActiveCount != 0) + { + detail = + $"session={session.SessionKey} queued={session.QueuedCount} active={session.ActiveCount}; " + + "idle (queued=0) is not a dispatch stall"; + return ConditionSignal.Inactive; + } + + detail = + $"session={session.SessionKey} queued={session.QueuedCount} active=0; " + + "dispatch hold before active pipeline (DispatchAcquireHold class)"; + return ConditionSignal.Active; + } + + /// + /// Instance ConsumerLoss: this process has zero running sessions while usable capacity desires consumers. + /// Process-scoped (not per-backbone): desired slots vs open Running channels on this instance. + /// + /// All projected sessions for this process. + /// Desired Running slot count for this instance. + /// Non-secret diagnostic text. + /// + /// Inactive when desired is ≤ 0 or any Running session has an open channel; Active when desired > 0 and none are Running with open channels. + /// + public static ConditionSignal EvaluateInstanceConsumerLoss( + IReadOnlyList sessions, + int desiredRunningSessions, + out string detail) + { + if (desiredRunningSessions <= 0) + { + detail = "no desired sessions for this instance"; + return ConditionSignal.Inactive; + } + + var running = 0; + foreach (var session in sessions) + { + if (session.State == ArticleWorkConsumerState.Running && session.HasOpenChannel) + { + running++; + } + } + + if (running > 0) + { + detail = $"runningSessions={running} desired={desiredRunningSessions}"; + return ConditionSignal.Inactive; + } + + detail = + $"runningSessions=0 desired={desiredRunningSessions}; " + + "local connection/consumer loss — not inferred from sibling aggregate rates"; + return ConditionSignal.Active; + } + + /// + /// Shared ProcessingStall: consumers≥1, unacked floor, near-zero ack (aggregate). + /// Active only when the fleet-wide ack path looks stalled; a healthy sibling suppresses this signal. + /// + /// Oldest-to-newest Management samples for the queue. + /// Floors and ack-rate epsilon for this condition. + /// Non-secret diagnostic text. + /// Active/Inactive when gauges and ack rate are known; Unknown when the latest sample is invalid or ack rate is unavailable. + public static ConditionSignal EvaluateSharedProcessingStall( + IReadOnlyList history, + SilentConsumerConditionThresholds options, + out string detail) + { + if (!TryLatest(history, out var snap, out detail)) + { + return ConditionSignal.Unknown; + } + + if (snap.Consumers < 1) + { + detail = "consumers < 1"; + return ConditionSignal.Inactive; + } + + if (snap.MessagesUnacknowledged < options.UnackedFloor) + { + detail = $"unacked {snap.MessagesUnacknowledged} < {options.UnackedFloor}"; + return ConditionSignal.Inactive; + } + + if (!TryAckRate(history, snap, out var ackRate, out detail)) + { + return ConditionSignal.Unknown; + } + + if (ackRate > options.AckRateEpsilon) + { + detail = + $"ackRate={ackRate:0.###}; aggregate progress present — " + + "does not prove every instance is healthy"; + return ConditionSignal.Inactive; + } + + detail = + $"SHARED aggregate consumers={snap.Consumers} unacked={snap.MessagesUnacknowledged} ackRate={ackRate:0.###}; " + + "does not identify a specific instance/consumer"; + return ConditionSignal.Active; + } + + /// + /// Shared DispatchStall: ready backlog + near-zero deliver/ack (aggregate; may be masked by siblings). + /// + /// Oldest-to-newest Management samples for the queue. + /// Floors and rate epsilons for this condition. + /// Non-secret diagnostic text. + /// Active/Inactive when metrics are known; Unknown when samples or rates are unavailable. + public static ConditionSignal EvaluateSharedDispatchStall( + IReadOnlyList history, + SilentConsumerConditionThresholds options, + out string detail) + { + if (!TryLatest(history, out var snap, out detail)) + { + return ConditionSignal.Unknown; + } + + if (snap.Consumers < 1) + { + detail = "consumers < 1"; + return ConditionSignal.Inactive; + } + + if (snap.MessagesReady < options.ReadyFloor) + { + detail = $"ready {snap.MessagesReady} < {options.ReadyFloor}"; + return ConditionSignal.Inactive; + } + + if (options.UnackedFloor > 0 && snap.MessagesUnacknowledged < options.UnackedFloor) + { + detail = $"unacked {snap.MessagesUnacknowledged} < {options.UnackedFloor}"; + return ConditionSignal.Inactive; + } + + if (!TryDeliverRate(history, snap, out var deliverRate, out detail) + || !TryAckRate(history, snap, out var ackRate, out detail)) + { + return ConditionSignal.Unknown; + } + + if (deliverRate > options.DeliverRateEpsilon || ackRate > options.AckRateEpsilon) + { + detail = + $"deliver={deliverRate:0.###} ack={ackRate:0.###}; " + + "aggregate progress — cannot clear a local DispatchStall on another instance"; + return ConditionSignal.Inactive; + } + + detail = + $"SHARED aggregate ready={snap.MessagesReady} unacked={snap.MessagesUnacknowledged}; " + + "healthy siblings may mask a single stalled consumer"; + return ConditionSignal.Active; + } + + /// + /// Shared ConsumerLoss: consumers==0 and ready floor while this instance expects the queue to be active. + /// One remaining consumer on another instance keeps consumers≥1 and keeps this inactive. + /// + /// Oldest-to-newest Management samples for the queue. + /// Ready floor for this condition. + /// False suppresses the condition (no usable capacity / desired slots). + /// Non-secret diagnostic text. + /// Inactive when not expected-active or consumers≠0 or ready below floor; Active on fleet-wide zero consumers with backlog; Unknown without a valid latest sample. + public static ConditionSignal EvaluateSharedConsumerLoss( + IReadOnlyList history, + SilentConsumerConditionThresholds options, + bool expectedActive, + out string detail) + { + if (!expectedActive) + { + detail = "queue not expected-active for this instance (no usable capacity / desired slots)"; + return ConditionSignal.Inactive; + } + + if (!TryLatest(history, out var snap, out detail)) + { + return ConditionSignal.Unknown; + } + + if (snap.Consumers != 0) + { + detail = + $"consumers={snap.Consumers}; not fleet-wide consumer loss " + + "(one healthy consumer keeps this inactive)"; + return ConditionSignal.Inactive; + } + + if (snap.MessagesReady < options.ReadyFloor) + { + detail = $"ready {snap.MessagesReady} < {options.ReadyFloor}"; + return ConditionSignal.Inactive; + } + + detail = + $"SHARED consumers=0 ready={snap.MessagesReady}; every polling instance may log this independently"; + return ConditionSignal.Active; + } + + /// Reads the newest valid snapshot from . + /// Sample ring snapshot. + /// Latest valid snapshot when returning true. + /// Failure reason when returning false. + /// when the latest entry is valid and has a snapshot. + private static bool TryLatest( + IReadOnlyList history, + out ManagementQueueSnapshot snap, + out string detail) + { + snap = null!; + if (history.Count == 0) + { + detail = "no samples"; + return false; + } + + var latest = history[^1]; + if (!latest.IsValid || latest.Snapshot is null) + { + detail = latest.FailureReason ?? "invalid sample"; + return false; + } + + snap = latest.Snapshot; + detail = string.Empty; + return true; + } + + /// + /// Resolves ack rate from broker or derived counters. + /// + /// Samples used for counter derivation when broker rate is absent. + /// Latest snapshot. + /// Resolved rate when returning true; 0 when false. + /// Failure reason when returning false. + /// when a usable ack rate is available. + private static bool TryAckRate( + IReadOnlyList history, + ManagementQueueSnapshot snap, + out double rate, + out string detail) + { + if (snap.AckRate is { } broker) + { + rate = broker; + detail = string.Empty; + return true; + } + + if (history.Count >= 2) + { + var derived = QueueSampleHistory.TryComputeCounterRate( + history[^2], + history[^1], + static s => s.AckTotal); + if (derived is { } value) + { + rate = value; + detail = string.Empty; + return true; + } + } + + rate = 0; + detail = "ack rate unavailable"; + return false; + } + + /// + /// Resolves deliver/get rate from broker or derived counters. + /// + /// Samples used for counter derivation when broker rate is absent. + /// Latest snapshot. + /// Resolved rate when returning true; 0 when false. + /// Failure reason when returning false. + /// when a usable deliver rate is available. + private static bool TryDeliverRate( + IReadOnlyList history, + ManagementQueueSnapshot snap, + out double rate, + out string detail) + { + if (snap.DeliverGetRate is { } broker) + { + rate = broker; + detail = string.Empty; + return true; + } + + if (history.Count >= 2) + { + var derived = QueueSampleHistory.TryComputeCounterRate( + history[^2], + history[^1], + static s => s.DeliverGetTotal); + if (derived is { } value) + { + rate = value; + detail = string.Empty; + return true; + } + } + + rate = 0; + detail = "deliver rate unavailable"; + return false; + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorDefaults.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorDefaults.cs new file mode 100644 index 00000000..b07dcc23 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorDefaults.cs @@ -0,0 +1,72 @@ +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Internal candidate defaults for silent-consumer detection. + /// Not production-validated paging thresholds; not exposed as JSON hierarchy. + /// + internal static class SilentConsumerMonitorDefaults + { + /// Seconds between monitor cycles. + internal const int PollIntervalSeconds = 10; + + /// Consecutive Management failures before monitor-health is degraded. + internal const int DegradedAfterConsecutiveFailures = 3; + + /// Fallback Management HTTP timeout when RabbitMQ:Management:RequestTimeoutSeconds is unset. + internal const int ManagementHttpTimeoutSeconds = 5; + + /// Processing-stall (in-flight) shared-queue windows and floors. + internal static SilentConsumerConditionThresholds ProcessingStall { get; } = new() + { + OpenWindow = TimeSpan.FromSeconds(90), + ResolveWindow = TimeSpan.FromSeconds(20), + UnackedFloor = 1, + AckRateEpsilon = 0.05, + }; + + /// Dispatch-stall (queued-before-active) shared-queue windows and floors. + internal static SilentConsumerConditionThresholds DispatchStall { get; } = new() + { + OpenWindow = TimeSpan.FromSeconds(90), + ResolveWindow = TimeSpan.FromSeconds(20), + ReadyFloor = 1, + UnackedFloor = 1, + AckRateEpsilon = 0.05, + DeliverRateEpsilon = 0.05, + }; + + /// Consumer-loss shared-queue windows and floors. + internal static SilentConsumerConditionThresholds ConsumerLoss { get; } = new() + { + OpenWindow = TimeSpan.FromSeconds(45), + ResolveWindow = TimeSpan.FromSeconds(20), + ReadyFloor = 1, + }; + + /// Bounded sample history capacity from the longest open window. + internal static int SampleHistoryCapacity => + Math.Clamp((int)(ProcessingStall.OpenWindow.TotalSeconds / PollIntervalSeconds) + 8, 8, 512); + } + + /// Internal thresholds for one silent-consumer condition. + internal sealed class SilentConsumerConditionThresholds + { + /// Sustained-active duration before opening. + public TimeSpan OpenWindow { get; init; } = TimeSpan.FromSeconds(90); + + /// Sustained-clear duration before resolving. + public TimeSpan ResolveWindow { get; init; } = TimeSpan.FromSeconds(20); + + /// Minimum unacknowledged messages. + public long UnackedFloor { get; init; } + + /// Minimum ready messages. + public long ReadyFloor { get; init; } + + /// Ack rate epsilon (msg/s). + public double AckRateEpsilon { get; init; } = 0.05; + + /// Deliver-get rate epsilon (msg/s). + public double DeliverRateEpsilon { get; init; } = 0.05; + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs new file mode 100644 index 00000000..853fc82d --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs @@ -0,0 +1,481 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Per-process silent-consumer evaluation. In-memory state is not shared across BackFiller instances; + /// shared-queue observations may therefore be duplicated in logs. Queue and backbone identities are + /// registered lazily from the live catalog/sessions so database-driven changes apply without restart. + /// + internal sealed class SilentConsumerMonitorEngine + { + /// Supplier for the instance id stamped on diagnostic events (typically runtime FQDN). + private readonly Func _instanceId; + + /// Logger for structured silent-consumer events (9100+). + private readonly ILogger _logger; + + /// Clock for evaluation timestamps, persistence, and rate-limit windows. + private readonly TimeProvider _time; + + /// Per-queue Management sample rings (shared scope only). + private readonly Dictionary _histories = new(StringComparer.Ordinal); + + /// One state machine per observation identity (instance session key, local, or queue). + private readonly Dictionary _machines = new(); + + /// + /// Per-key last emit time for Continued (9111) and Unavailable-while-open (9113). + /// Interval is open window. + /// + private readonly Dictionary _lastContinuationUtc = new(); + + /// Consecutive non-success Management poll samples since the last success. + private int _consecutiveManagementFailures; + + /// True after consecutive failures reach until a success clears it. + private bool _managementDegraded; + + /// + /// Creates the in-process evaluation engine and seeds the process-scoped instance ConsumerLoss machine. + /// + /// + /// Delegate invoked when stamping diagnostics (and ); typically + /// () => runtime.Fqdn so refreshes are visible without rebuilding the engine. + /// + /// Structured logger for Events 9100+. + /// Time provider for evaluation and rate limiting. + /// + /// Thrown when , , or is null. + /// + public SilentConsumerMonitorEngine( + Func instanceId, + ILogger logger, + TimeProvider time) + { + ArgumentNullException.ThrowIfNull(instanceId); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(time); + _instanceId = instanceId; + _logger = logger; + _time = time; + EnsureInstanceTarget( + "local", + SilentConsumerCondition.ConsumerLoss, + SilentConsumerMonitorDefaults.ConsumerLoss); + } + + /// Instance identifier stamped on diagnostic events (typically current runtime FQDN). + public string InstanceId => _instanceId(); + + /// True when Management polling is degraded. + public bool IsManagementDegraded => _managementDegraded; + + /// + /// Evaluates instance-local session health (no Management call). + /// ProcessingStall and DispatchStall are keyed by + /// so multiple MaxSessions + /// slots on the same backbone (and distinct backbones) keep independent state machines. + /// + /// Projected live sessions for this process (null not allowed). + /// + /// Count of desired Running consume slots for this instance (usable capacity × MaxSessions). + /// Used only by process-scoped ConsumerLoss (local). + /// + /// Per-condition results for seen sessions, removed session keys, and ConsumerLoss; also emits diagnostics. + /// Thrown when is null. + public IReadOnlyList ObserveInstance( + IReadOnlyList sessions, + int desiredRunningSessions) + { + ArgumentNullException.ThrowIfNull(sessions); + var now = _time.GetUtcNow(); + var results = new List(); + var seenSessionKeys = new HashSet(StringComparer.Ordinal); + + foreach (var session in sessions) + { + if (string.IsNullOrWhiteSpace(session.Backbone) || session.ConnectionNumber < 1) + { + continue; + } + + var sessionKey = session.SessionKey; + seenSessionKeys.Add(sessionKey); + EnsureInstanceTarget( + sessionKey, + SilentConsumerCondition.ProcessingStall, + SilentConsumerMonitorDefaults.ProcessingStall); + EnsureInstanceTarget( + sessionKey, + SilentConsumerCondition.DispatchStall, + SilentConsumerMonitorDefaults.DispatchStall); + + results.Add( + Apply( + new ObservationKey( + ObservationScope.Instance, + SilentConsumerCondition.ProcessingStall, + sessionKey), + SilentConsumerEvaluators.EvaluateInstanceProcessingStall(session, out var stallDetail), + stallDetail, + now)); + results.Add( + Apply( + new ObservationKey( + ObservationScope.Instance, + SilentConsumerCondition.DispatchStall, + sessionKey), + SilentConsumerEvaluators.EvaluateInstanceDispatchStall(session, out var dispatchDetail), + dispatchDetail, + now)); + } + + // Drive Inactive for session keys that left the live session set (sibling keys untouched). + foreach (var key in _machines.Keys.ToArray()) + { + if (key.Scope != ObservationScope.Instance + || key.Condition == SilentConsumerCondition.ConsumerLoss + || string.Equals(key.QueueOrBackbone, "local", StringComparison.Ordinal)) + { + continue; + } + + if (seenSessionKeys.Contains(key.QueueOrBackbone)) + { + continue; + } + + results.Add(Apply(key, ConditionSignal.Inactive, "session removed", now)); + } + + results.Add( + Apply( + new ObservationKey(ObservationScope.Instance, SilentConsumerCondition.ConsumerLoss, "local"), + SilentConsumerEvaluators.EvaluateInstanceConsumerLoss( + sessions, + desiredRunningSessions, + out var lossDetail), + lossDetail, + now)); + + Emit(results); + return results; + } + + /// + /// Ingests one Management poll and evaluates shared conditions for that queue. + /// + /// Article Work queue entity name (shared observation identity). + /// Classified Management poll for this cycle. + /// + /// True when this instance currently desires consume slots for the queue's backbone + /// (usable capacity); gates shared ConsumerLoss. + /// + /// Three shared-condition results (ProcessingStall, DispatchStall, ConsumerLoss); also emits diagnostics. + /// Thrown when is null/empty/whitespace. + /// Thrown when is null. + /// + /// Non-success polls append an invalid sample so shared evaluators return . + /// Unknown neither opens nor resolves; an already-open shared alert remains open until a conclusive + /// Inactive signal, catalog prune (target gone — not broker recovery), or process restart. + /// + public IReadOnlyList ObserveSharedQueue( + string queueName, + ManagementPollResult poll, + bool expectedActive) + { + ArgumentException.ThrowIfNullOrWhiteSpace(queueName); + ArgumentNullException.ThrowIfNull(poll); + EnsureShared(queueName); + + var history = _histories[queueName]; + var now = _time.GetUtcNow(); + if (poll.IsSuccess && poll.Snapshot is { } snap) + { + history.AddSuccess(snap); + NoteManagementSuccess(); + } + else + { + history.AddFailure(now, poll.Kind, poll.Reason); + SilentConsumerMonitorLogMessages.ManagementPollFailed( + _logger, + InstanceId, + queueName, + poll.Kind.ToString(), + poll.Reason); + NoteManagementFailure(poll.Reason); + } + + var samples = history.Snapshot(); + var results = new List(3) + { + Apply( + new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.ProcessingStall, queueName), + SilentConsumerEvaluators.EvaluateSharedProcessingStall( + samples, + SilentConsumerMonitorDefaults.ProcessingStall, + out var dDetail), + dDetail, + now), + Apply( + new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.DispatchStall, queueName), + SilentConsumerEvaluators.EvaluateSharedDispatchStall( + samples, + SilentConsumerMonitorDefaults.DispatchStall, + out var d2Detail), + d2Detail, + now), + Apply( + new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.ConsumerLoss, queueName), + SilentConsumerEvaluators.EvaluateSharedConsumerLoss( + samples, + SilentConsumerMonitorDefaults.ConsumerLoss, + expectedActive, + out var eDetail), + eDetail, + now), + }; + Emit(results); + return results; + } + + /// + /// Removes sample history for queues no longer in the live catalog. + /// Open shared observations emit an explicit Resolved event (target removed), then machines are dropped. + /// This is not a claim that the underlying queue condition cleared on the broker. + /// + /// Queue names still present in the resolved catalog targets this cycle. + /// Resolved results for pruned open shared observations (may be empty). + /// Thrown when is null. + public IReadOnlyList PruneQueues(IReadOnlyCollection activeQueues) + { + ArgumentNullException.ThrowIfNull(activeQueues); + var active = activeQueues as HashSet + ?? new HashSet(activeQueues, StringComparer.Ordinal); + var stale = _histories.Keys.Where(q => !active.Contains(q)).ToArray(); + if (stale.Length == 0) + { + return Array.Empty(); + } + + var results = new List(); + foreach (var queue in stale) + { + foreach (var condition in new[] + { + SilentConsumerCondition.ProcessingStall, + SilentConsumerCondition.DispatchStall, + SilentConsumerCondition.ConsumerLoss, + }) + { + var key = new ObservationKey(ObservationScope.SharedQueue, condition, queue); + if (!_machines.TryGetValue(key, out var machine)) + { + continue; + } + + if (machine.Phase == ObservationPhase.Open) + { + results.Add(new ObservationResult + { + Key = key, + Signal = ConditionSignal.Inactive, + Transition = ObservationTransition.Resolved, + Phase = ObservationPhase.Clear, + Persistence = TimeSpan.Zero, + Detail = "provider/queue removed from catalog (observation target gone; not broker recovery)", + }); + } + } + + _histories.Remove(queue); + RemoveSharedMachines(queue); + } + + Emit(results); + return results; + } + + /// Applies to the machine for (must already exist). + /// Observation identity whose machine receives the signal. + /// Evaluator signal for this tick. + /// Non-secret detail text. + /// Evaluation instant. + /// State-machine result for this tick. + private ObservationResult Apply( + ObservationKey key, + ConditionSignal signal, + string detail, + DateTimeOffset atUtc) => + _machines[key].Apply(signal, atUtc, detail); + + /// + /// Emits structured logs for Opened/Resolved immediately and rate-limits Continued / Unavailable-while-open. + /// + /// Results produced by the current evaluation pass. + private void Emit(IReadOnlyList results) + { + foreach (var result in results) + { + switch (result.Transition) + { + case ObservationTransition.Opened: + SilentConsumerMonitorLogMessages.ObservationOpened( + _logger, + result.Key.Scope, + result.Key.Condition, + result.Key.QueueOrBackbone, + InstanceId, + result.Persistence.TotalSeconds, + result.Detail); + break; + case ObservationTransition.Resolved: + SilentConsumerMonitorLogMessages.ObservationResolved( + _logger, + result.Key.Scope, + result.Key.Condition, + result.Key.QueueOrBackbone, + InstanceId, + result.Detail); + break; + case ObservationTransition.Continued when result.Phase == ObservationPhase.Open: + EmitRateLimited( + result.Key, + () => SilentConsumerMonitorLogMessages.ObservationContinued( + _logger, + result.Key.Scope, + result.Key.Condition, + result.Key.QueueOrBackbone, + InstanceId, + result.Persistence.TotalSeconds, + result.Detail)); + break; + case ObservationTransition.None + when result.Signal == ConditionSignal.Unknown && result.Phase == ObservationPhase.Open: + EmitRateLimited( + result.Key, + () => SilentConsumerMonitorLogMessages.ObservationUnavailableWhileOpen( + _logger, + result.Key.Scope, + result.Key.Condition, + result.Key.QueueOrBackbone, + InstanceId, + result.Detail)); + break; + } + } + } + + /// + /// Emits at most once per observation key per ProcessingStall open window. + /// Continued and Unavailable-while-open share the same timestamp for a given key. + /// + /// Observation identity used as the rate-limit key. + /// Log emission to run when the interval has elapsed. + private void EmitRateLimited(ObservationKey key, Action emit) + { + var now = _time.GetUtcNow(); + if (!_lastContinuationUtc.TryGetValue(key, out var last) + || now - last >= SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow) + { + _lastContinuationUtc[key] = now; + emit(); + } + } + + /// Clears Management-degraded latch after a successful poll sample. + private void NoteManagementSuccess() + { + if (_managementDegraded) + { + _managementDegraded = false; + SilentConsumerMonitorLogMessages.ManagementRecovered(_logger, InstanceId); + } + + _consecutiveManagementFailures = 0; + } + + /// Counts consecutive poll failures and emits Event 9120 when the degraded threshold is crossed. + /// Non-secret failure reason stamped on the degraded event. + private void NoteManagementFailure(string reason) + { + _consecutiveManagementFailures++; + if (!_managementDegraded + && _consecutiveManagementFailures >= SilentConsumerMonitorDefaults.DegradedAfterConsecutiveFailures) + { + _managementDegraded = true; + SilentConsumerMonitorLogMessages.ManagementDegraded( + _logger, + InstanceId, + _consecutiveManagementFailures, + reason); + } + } + + /// Lazily creates shared history and state machines for a catalog queue name. + /// Queue entity name. + private void EnsureShared(string queue) + { + if (_histories.ContainsKey(queue)) + { + return; + } + + _histories[queue] = new QueueSampleHistory(SilentConsumerMonitorDefaults.SampleHistoryCapacity); + EnsureInstanceTarget( + queue, + SilentConsumerCondition.ProcessingStall, + SilentConsumerMonitorDefaults.ProcessingStall, + ObservationScope.SharedQueue); + EnsureInstanceTarget( + queue, + SilentConsumerCondition.DispatchStall, + SilentConsumerMonitorDefaults.DispatchStall, + ObservationScope.SharedQueue); + EnsureInstanceTarget( + queue, + SilentConsumerCondition.ConsumerLoss, + SilentConsumerMonitorDefaults.ConsumerLoss, + ObservationScope.SharedQueue); + } + + /// Creates a state machine for when absent (instance session key or shared queue). + /// Session key, local, or shared queue name. + /// Condition the machine tracks. + /// Open/resolve windows for the machine. + /// Instance (default) or SharedQueue. + private void EnsureInstanceTarget( + string target, + SilentConsumerCondition condition, + SilentConsumerConditionThresholds thresholds, + ObservationScope scope = ObservationScope.Instance) + { + var key = new ObservationKey(scope, condition, target); + if (_machines.ContainsKey(key)) + { + return; + } + + _machines[key] = new ObservationStateMachine(key, thresholds.OpenWindow, thresholds.ResolveWindow); + } + + /// Drops all shared machines for a pruned queue name. + /// Queue entity name being removed from catalog targets. + private void RemoveSharedMachines(string queue) + { + RemoveMachine(new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.ProcessingStall, queue)); + RemoveMachine(new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.DispatchStall, queue)); + RemoveMachine(new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.ConsumerLoss, queue)); + } + + /// Removes the machine and its rate-limit timestamp so a reintroduced target starts fresh. + /// Observation identity to drop. + private void RemoveMachine(ObservationKey key) + { + _machines.Remove(key); + _lastContinuationUtc.Remove(key); + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorHostedService.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorHostedService.cs new file mode 100644 index 00000000..3203647f --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorHostedService.cs @@ -0,0 +1,316 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.Common.Core; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Background silent-consumer monitor hosted inside BackFillerService. + /// Failures are logged and never propagate into article processing. + /// + /// + /// + /// Observational only: never calls RetireForRecoveryAsync or changes ACK/NACK. + /// Event 5328 recovery remains owned by Article Work reconcile. + /// + /// + /// Each cycle projects live sessions, runs , + /// prunes shared targets removed from the catalog, then (when Management is ready) polls queues. + /// Instance evaluation always runs before the Management gate so missing BaseUrl/credentials or + /// HTTP failures do not disable local monitoring. + /// + /// + /// Virtual host, Management base URL, and credentials come from live + /// (credentials refreshed from nntpsharedconfig). Queue targets come from the MySQL-backed + /// provider catalog each cycle. + /// + /// + internal sealed class SilentConsumerMonitorHostedService : BackgroundService + { + /// In-process observation engine (instance + shared evaluation and diagnostics). + private readonly SilentConsumerMonitorEngine _engine; + + /// HTTP Management client used for shared-queue polls. + private readonly ManagementQueueClient _management; + + /// Live Article Work consumer sessions projected each cycle. + private readonly ArticleWorkConsumerService _consumers; + + /// Usable NNTP capacity used for expected-active targets and desired session counts. + private readonly IBackboneUsableCapacityProvider _capacity; + + /// MySQL-backed provider catalog used to resolve queue targets each cycle. + private readonly IBackFillerProviderCatalog _catalog; + + /// Public enable switch under BackFiller:SilentConsumerMonitor. + private readonly IOptions _options; + + /// Live RabbitMQ options (vhost, Management BaseUrl, credentials). + private readonly IOptions _rabbitMq; + + /// Runtime options; stamps diagnostic instance id. + private readonly BackFillerRuntimeOptions _runtime; + + /// Hosted-service logger for Events 9100+. + private readonly ILogger _logger; + + /// Clock used for poll-interval . + private readonly TimeProvider _time; + + /// True after Event 9123 until Management becomes ready again (emit once per outage). + private bool _loggedManagementUnavailable; + + /// + /// Creates the background silent-consumer monitor hosted inside BackFillerService. + /// + /// Observation engine that owns in-memory state machines and sample history. + /// Management HTTP client for shared-queue polls. + /// Article Work consumer service whose sessions are projected each cycle. + /// Usable backbone capacity for expected-active and desired-session counting. + /// Provider catalog supplying backbone/MaxSessions for queue targets. + /// Enable switch; when disabled the loop logs start and returns. + /// Live RabbitMQ options for vhost/Management readiness and credentials. + /// Runtime options providing the FQDN stamped on diagnostics. + /// Structured logger. + /// Time provider for cycle delay. + /// Thrown when any constructor argument is null. + public SilentConsumerMonitorHostedService( + SilentConsumerMonitorEngine engine, + ManagementQueueClient management, + ArticleWorkConsumerService consumers, + IBackboneUsableCapacityProvider capacity, + IBackFillerProviderCatalog catalog, + IOptions options, + IOptions rabbitMq, + BackFillerRuntimeOptions runtime, + ILogger logger, + TimeProvider time) + { + ArgumentNullException.ThrowIfNull(engine); + ArgumentNullException.ThrowIfNull(management); + ArgumentNullException.ThrowIfNull(consumers); + ArgumentNullException.ThrowIfNull(capacity); + ArgumentNullException.ThrowIfNull(catalog); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(rabbitMq); + ArgumentNullException.ThrowIfNull(runtime); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(time); + _engine = engine; + _management = management; + _consumers = consumers; + _capacity = capacity; + _catalog = catalog; + _options = options; + _rabbitMq = rabbitMq; + _runtime = runtime; + _logger = logger; + _time = time; + } + + /// + /// Runs the monitor loop until cancelled or disabled: project sessions, observe instance, + /// prune catalog targets, optionally poll Management, then delay one poll interval. + /// + /// Host shutdown token; cancels delay and in-flight polls. + /// A task that completes when the loop exits (disabled, cancelled, or host stop). + /// + /// Non-fatal cycle exceptions are logged as Management-degraded and do not stop the host. + /// Fatal exceptions are not caught here. + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (!_options.Value.Enabled) + { + SilentConsumerMonitorLogMessages.Starting(_logger, _runtime.Fqdn, enabled: false, queueCount: 0); + return; + } + + var startedLogged = false; + try + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + var targets = SilentConsumerMonitorTargets.Resolve(_catalog, _capacity); + if (!startedLogged) + { + SilentConsumerMonitorLogMessages.Starting( + _logger, + _runtime.Fqdn, + enabled: true, + queueCount: targets.Count); + startedLogged = true; + } + + var observations = ProjectSessions(); + var desired = CountDesiredSessions(); + _ = _engine.ObserveInstance(observations, desired); + _engine.PruneQueues(targets.Select(static t => t.QueueName).ToArray()); + + if (!TryGetManagementReady(out var virtualHost, out var unavailableReason)) + { + if (!_loggedManagementUnavailable) + { + SilentConsumerMonitorLogMessages.ManagementUnavailable( + _logger, + _runtime.Fqdn, + unavailableReason); + _loggedManagementUnavailable = true; + } + } + else + { + if (_loggedManagementUnavailable) + { + _loggedManagementUnavailable = false; + SilentConsumerMonitorLogMessages.ManagementRecovered(_logger, _runtime.Fqdn); + } + + foreach (var target in targets) + { + stoppingToken.ThrowIfCancellationRequested(); + try + { + var poll = await _management + .GetQueueAsync(virtualHost, target.QueueName, stoppingToken) + .ConfigureAwait(false); + _ = _engine.ObserveSharedQueue( + target.QueueName, + poll, + target.ExpectedActive); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + SilentConsumerMonitorLogMessages.ManagementPollFailed( + _logger, + _runtime.Fqdn, + target.QueueName, + "Exception", + ex.GetType().Name); + } + } + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + SilentConsumerMonitorLogMessages.ManagementDegraded( + _logger, + _runtime.Fqdn, + consecutiveFailures: 1, + reason: "cycle:" + ex.GetType().Name); + } + + try + { + await Task + .Delay( + TimeSpan.FromSeconds(SilentConsumerMonitorDefaults.PollIntervalSeconds), + _time, + stoppingToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + } + finally + { + SilentConsumerMonitorLogMessages.Stopped(_logger, _runtime.Fqdn); + } + } + + /// + /// Returns whether live can support a Management poll this cycle. + /// Does not perform HTTP; a false result skips shared evaluation (instance monitoring already ran). + /// + /// + /// Virtual host to poll (trimmed; defaults to / when options VirtualHost is blank). + /// + /// Non-secret unavailability reason when returning false; empty on success. + /// + /// when Management BaseUrl is configured and Username/Password are present + /// (Password may be empty string once projected; null Password is treated as not ready). + /// + private bool TryGetManagementReady(out string virtualHost, out string reason) + { + var rabbit = _rabbitMq.Value; + virtualHost = string.IsNullOrWhiteSpace(rabbit.VirtualHost) ? "/" : rabbit.VirtualHost.Trim(); + var baseUrl = rabbit.Management?.BaseUrl; + if (string.IsNullOrWhiteSpace(baseUrl)) + { + reason = "RabbitMQ:Management:BaseUrl is not configured."; + return false; + } + + if (string.IsNullOrWhiteSpace(rabbit.Username) || rabbit.Password is null) + { + reason = "RabbitMQ credentials are not available yet (awaiting nntpsharedconfig projection)."; + return false; + } + + reason = string.Empty; + return true; + } + + /// + /// Projects tracked consumer sessions with the same stall threshold reconcile uses (Event 5328). + /// + /// Immutable session projections for the current cycle. + private List ProjectSessions() + { + var sessions = _consumers.Sessions; + var projected = new List(sessions.Count); + foreach (var session in sessions) + { + projected.Add( + SilentConsumerSessionObservation.FromSession( + session, + ArticleWorkConsumerService.ProcessingStallThreshold)); + } + + return projected; + } + + /// + /// Mirrors ArticleWorkConsumerService.BuildDesiredSessions slot counting: + /// MaxSessions for each provider with usable NNTP capacity. + /// + /// Sum of MaxSessions across providers that currently have usable backbone capacity. + private int CountDesiredSessions() + { + var total = 0; + foreach (var provider in _catalog.Providers) + { + if (provider.MaxSessions <= 0) + { + continue; + } + + if (!_capacity.HasUsableCapacityForBackbone(provider.Backbone)) + { + continue; + } + + total += provider.MaxSessions; + } + + return total; + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorLogMessages.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorLogMessages.cs new file mode 100644 index 00000000..72a702e3 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorLogMessages.cs @@ -0,0 +1,166 @@ +using Microsoft.Extensions.Logging; + +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// Structured silent-consumer monitor events (9100+). + internal static partial class SilentConsumerMonitorLogMessages + { + /// Event 9100: monitor loop starting (or exiting immediately when disabled). + /// Target logger. + /// Instance identity stamped on the event (typically FQDN). + /// Whether the monitor will run cycles. + /// Shared queue target count at first start log (0 when disabled). + [LoggerMessage( + EventId = 9100, + Level = LogLevel.Information, + Message = "Silent-consumer monitor starting instanceId={InstanceId} enabled={Enabled} sharedQueueTargets={QueueCount}")] + public static partial void Starting(ILogger logger, string instanceId, bool enabled, int queueCount); + + /// Event 9101: monitor loop stopped (finally after cancel/disable/exit). + /// Target logger. + /// Instance identity stamped on the event. + [LoggerMessage( + EventId = 9101, + Level = LogLevel.Information, + Message = "Silent-consumer monitor stopped instanceId={InstanceId}")] + public static partial void Stopped(ILogger logger, string instanceId); + + /// Event 9110: observation transitioned to Open. + /// Target logger. + /// Instance vs shared queue. + /// Condition that opened. + /// Session key, local, or queue name. + /// Instance identity. + /// Sustained Active duration at open, in seconds. + /// Non-secret evaluator detail. + [LoggerMessage( + EventId = 9110, + Level = LogLevel.Warning, + Message = "Silent-consumer observation opened scope={Scope} condition={Condition} target={Target} instanceId={InstanceId} persistenceSeconds={PersistenceSeconds} detail={Detail}")] + public static partial void ObservationOpened( + ILogger logger, + ObservationScope scope, + SilentConsumerCondition condition, + string target, + string instanceId, + double persistenceSeconds, + string? detail); + + /// Event 9111: open observation continued (rate-limited per key). + /// Target logger. + /// Instance vs shared queue. + /// Condition still open. + /// Session key, local, or queue name. + /// Instance identity. + /// Current persistence seconds. + /// Non-secret evaluator detail. + [LoggerMessage( + EventId = 9111, + Level = LogLevel.Information, + Message = "Silent-consumer observation continued scope={Scope} condition={Condition} target={Target} instanceId={InstanceId} persistenceSeconds={PersistenceSeconds} detail={Detail}")] + public static partial void ObservationContinued( + ILogger logger, + ObservationScope scope, + SilentConsumerCondition condition, + string target, + string instanceId, + double persistenceSeconds, + string? detail); + + /// Event 9112: observation resolved (Inactive window or catalog prune). + /// Target logger. + /// Instance vs shared queue. + /// Condition that resolved. + /// Session key, local, or queue name. + /// Instance identity. + /// Non-secret resolve detail. + [LoggerMessage( + EventId = 9112, + Level = LogLevel.Information, + Message = "Silent-consumer observation resolved scope={Scope} condition={Condition} target={Target} instanceId={InstanceId} detail={Detail}")] + public static partial void ObservationResolved( + ILogger logger, + ObservationScope scope, + SilentConsumerCondition condition, + string target, + string instanceId, + string? detail); + + /// + /// Event 9113: rate-limited when an already-open observation receives . + /// Unknown must not be read as recovery; the open phase is intentionally retained. + /// + /// Target logger. + /// Instance vs shared queue. + /// Condition that remains open. + /// Session key, local, or queue name. + /// Instance identity. + /// Non-secret unavailability detail. + [LoggerMessage( + EventId = 9113, + Level = LogLevel.Information, + Message = "Silent-consumer observation unavailable scope={Scope} condition={Condition} target={Target} instanceId={InstanceId} detail={Detail}; open alert retained (Unknown is not recovery)")] + public static partial void ObservationUnavailableWhileOpen( + ILogger logger, + ObservationScope scope, + SilentConsumerCondition condition, + string target, + string instanceId, + string? detail); + + /// Event 9120: Management polling crossed the consecutive-failure degraded threshold. + /// Target logger. + /// Instance identity. + /// Failure count at the moment degraded latched. + /// Non-secret failure reason. + [LoggerMessage( + EventId = 9120, + Level = LogLevel.Warning, + Message = "Silent-consumer monitor Management degraded instanceId={InstanceId} consecutiveFailures={ConsecutiveFailures} reason={Reason}")] + public static partial void ManagementDegraded( + ILogger logger, + string instanceId, + int consecutiveFailures, + string reason); + + /// Event 9121: Management recovered after a degraded latch or after Event 9123 readiness returned. + /// Target logger. + /// Instance identity. + [LoggerMessage( + EventId = 9121, + Level = LogLevel.Information, + Message = "Silent-consumer monitor Management recovered instanceId={InstanceId}")] + public static partial void ManagementRecovered(ILogger logger, string instanceId); + + /// Event 9122: one Management poll failed (debug); does not by itself mean degraded. + /// Target logger. + /// Instance identity. + /// Queue that was polled. + /// Failure kind or exception type name. + /// Non-secret reason. + [LoggerMessage( + EventId = 9122, + Level = LogLevel.Debug, + Message = "Silent-consumer Management poll failed instanceId={InstanceId} queue={QueueName} kind={Kind} reason={Reason}")] + public static partial void ManagementPollFailed( + ILogger logger, + string instanceId, + string queueName, + string kind, + string reason); + + /// + /// Event 9123: emitted once when Management is not ready to poll (missing BaseUrl/credentials). + /// Does not mean a shared alert received ; polling did not run. + /// Any already-open shared alerts are left unchanged until polling resumes or the catalog target is removed. + /// + /// Target logger. + /// Instance identity. + /// Non-secret readiness failure reason. + [LoggerMessage( + EventId = 9123, + Level = LogLevel.Warning, + Message = "Silent-consumer Management unavailable instanceId={InstanceId} reason={Reason}; instance-local monitoring continues; Management is not being polled, so shared alerts are neither refreshed nor auto-resolved (unavailable is not recovery)")] + public static partial void ManagementUnavailable(ILogger logger, string instanceId, string reason); + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptions.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptions.cs new file mode 100644 index 00000000..e33f3f35 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptions.cs @@ -0,0 +1,23 @@ +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Optional public switch for in-process silent-consumer monitoring under + /// BackFiller:SilentConsumerMonitor. + /// + /// + /// Virtual host, Management base URL, credentials, and Article Work queues are not configured here. + /// Those come from RabbitMQ options (hosts/vhost/Management) and the MySQL-backed provider catalog. + /// Detection thresholds are code defaults, not operator JSON. + /// + internal sealed class SilentConsumerMonitorOptions + { + /// Nested configuration key under BackFiller. + internal const string SectionRelativeName = "SilentConsumerMonitor"; + + /// + /// When , the hosted monitor loop exits after a start log. + /// Default . + /// + public bool Enabled { get; set; } = true; + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptionsValidator.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptionsValidator.cs new file mode 100644 index 00000000..207b2308 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptionsValidator.cs @@ -0,0 +1,25 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Validates . Infrastructure (vhost, queues, Management) + /// is not part of this options type. + /// + internal sealed class SilentConsumerMonitorOptionsValidator : IValidateOptions + { + /// + /// Validates . Currently accepts any non-null instance + /// (Enabled is unconstrained; infrastructure lives on RabbitMQ/catalog). + /// + /// Options name (unused). + /// Bound silent-consumer options. + /// when is non-null. + /// Thrown when is null. + public ValidateOptionsResult Validate(string? name, SilentConsumerMonitorOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorTargets.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorTargets.cs new file mode 100644 index 00000000..fc52b425 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorTargets.cs @@ -0,0 +1,63 @@ +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.RabbitMq; + +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// One Article Work queue this process may observe via Management, derived from the live provider catalog. + /// + /// Provider backbone label. + /// backfiller.{backbone} entity name. + /// + /// True when this instance currently desires consume slots for the backbone + /// ( > 0 and usable NNTP capacity). + /// Idle or capacity-empty queues must not raise shared consumer-loss. + /// + internal readonly record struct SilentConsumerMonitorQueueTarget( + string Backbone, + string QueueName, + bool ExpectedActive); + + /// Resolves monitor queue targets from database-backed catalog + capacity. + internal static class SilentConsumerMonitorTargets + { + /// + /// Builds the set of Article Work queues this instance is responsible for observing. + /// Providers with MaxSessions <= 0 are omitted (not BackFiller consume targets). + /// + /// Live provider catalog (backbone + MaxSessions). + /// Usable capacity used to set . + /// Deduplicated queue targets for this cycle. + /// + /// Thrown when or is null. + /// + public static IReadOnlyList Resolve( + IBackFillerProviderCatalog catalog, + IBackboneUsableCapacityProvider capacity) + { + ArgumentNullException.ThrowIfNull(catalog); + ArgumentNullException.ThrowIfNull(capacity); + + var targets = new List(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var provider in catalog.Providers) + { + if (provider.MaxSessions <= 0 || string.IsNullOrWhiteSpace(provider.Backbone)) + { + continue; + } + + var queue = BackFillerRabbitMqTopology.ComposeProviderEntity(provider.Backbone); + if (!seen.Add(queue)) + { + continue; + } + + var expected = capacity.HasUsableCapacityForBackbone(provider.Backbone); + targets.Add(new SilentConsumerMonitorQueueTarget(provider.Backbone, queue, expected)); + } + + return targets; + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerSessionObservation.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerSessionObservation.cs new file mode 100644 index 00000000..62011684 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerSessionObservation.cs @@ -0,0 +1,58 @@ +namespace VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor +{ + /// + /// Immutable projection of one consumer session for silent-consumer evaluation. + /// Avoids coupling evaluators to session mutation and keeps unit tests broker-free. + /// + /// Provider backbone label. + /// One-based consume slot for this backbone (same as reconcile). + /// Local session lifecycle state. + /// + /// Instance-local _inFlight - _active: admitted work not yet inside the pipeline handler. + /// Includes deliveries that hold the dispatch lock but have not incremented _active + /// (for example a DispatchAcquireHold). Not the broker ready count. + /// + /// Deliveries that have entered the pipeline handler (_active). + /// Whether the consume channel is present and open. + /// + /// Same predicate reconcile uses for Event 5328. Idle sessions (no active work) never stall. + /// + internal readonly record struct SilentConsumerSessionObservation( + string Backbone, + int ConnectionNumber, + ArticleWorkConsumerState State, + int QueuedCount, + int ActiveCount, + bool HasOpenChannel, + bool IsActiveProcessingStalled) + { + /// + /// Canonical session identity backbone/connectionNumber + /// (). + /// + public string SessionKey => + ArticleWorkConsumerSession.ComposeSessionKey(Backbone, ConnectionNumber); + + /// Projects a live session using the same stall threshold as reconcile. + /// Live consumer session. + /// + /// Active-processing stall threshold (same as ArticleWorkConsumerService.ProcessingStallThreshold). + /// + /// Immutable observation for evaluators/tests. + /// Thrown when is null. + public static SilentConsumerSessionObservation FromSession( + ArticleWorkConsumerSession session, + TimeSpan stallThreshold) + { + ArgumentNullException.ThrowIfNull(session); + return new SilentConsumerSessionObservation( + session.Backbone, + session.ConnectionNumber, + session.State, + session.QueuedCount, + session.ActiveCount, + session.Channel is { IsOpen: true }, + session.IsActiveProcessingStalled(stallThreshold)); + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs index bc56eb27..f9bcc811 100644 --- a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs +++ b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs @@ -3,6 +3,7 @@ using VectorNNTPD.BackFillerService.Accounts; using VectorNNTPD.BackFillerService.Acme; using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; using VectorNNTPD.BackFillerService.Configuration; using VectorNNTPD.Common.Core; using VectorNNTPD.BackFillerService.Hosting.Systemd; @@ -60,7 +61,11 @@ internal static class BackFillerServiceCollectionExtensions /// , , /// , /// , , - /// then . + /// , then . + /// Silent-consumer registration also binds , + /// registers with named HTTP client + /// , and a singleton + /// keyed by runtime FQDN. /// The notifier is first so it is constructed before later hosted services and can observe /// and /// . @@ -301,6 +306,28 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication builder.Services.AddSingleton(static provider => provider.GetRequiredService()); + builder.Services + .AddOptions() + .BindConfiguration( + BackFillerOptions.SectionName + ":" + SilentConsumerMonitorOptions.SectionRelativeName) + .ValidateOnStart(); + builder.Services.AddSingleton, SilentConsumerMonitorOptionsValidator>(); + builder.Services.AddHttpClient(ManagementQueueClient.HttpClientName, static client => + { + client.Timeout = Timeout.InfiniteTimeSpan; + client.DefaultRequestHeaders.ExpectContinue = false; + }); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(static provider => + { + var runtime = provider.GetRequiredService(); + return new SilentConsumerMonitorEngine( + () => runtime.Fqdn, + provider.GetRequiredService>(), + provider.GetRequiredService()); + }); + builder.Services.AddHostedService(); + return builder; } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ManagementQueueClientTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ManagementQueueClientTests.cs new file mode 100644 index 00000000..b08a15ef --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ManagementQueueClientTests.cs @@ -0,0 +1,390 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// Stub-HTTP coverage for URI construction, Basic auth, JSON parsing, + /// poll classification, cancellation, and live credential refresh. + /// Does not contact a real RabbitMQ cluster. + /// + public sealed class ManagementQueueClientTests + { + /// + /// percent-encodes virtual host and queue name in the management API path. + /// + [Fact] + public void BuildQueueUri_encodes_vhost_and_queue() + { + var uri = ManagementQueueClient.BuildQueueUri( + "http://198.18.0.3:15672", + "/", + "backfiller.giganews"); + Assert.Equal( + "http://198.18.0.3:15672/api/queues/%2F/backfiller.giganews", + uri.AbsoluteUri); + } + + /// + /// A well-formed management JSON document maps to with parsed gauges and rates. + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_parses_successful_document() + { + var json = """ + { + "messages_ready": 4, + "messages_unacknowledged": 1, + "messages": 5, + "consumers": 1, + "state": "running", + "type": "classic", + "message_stats": { + "ack": 10, + "ack_details": { "rate": 0.0 }, + "deliver_get": 11, + "deliver_get_details": { "rate": 0.0 } + } + } + """; + var client = CreateClient(new StubHandler(HttpStatusCode.OK, json)); + var result = await client.GetQueueAsync("/", "backfiller.giganews", CancellationToken.None); + Assert.Equal(ManagementPollKind.Success, result.Kind); + Assert.NotNull(result.Snapshot); + Assert.Equal(4, result.Snapshot.MessagesReady); + Assert.Equal(1, result.Snapshot.MessagesUnacknowledged); + Assert.Equal(1, result.Snapshot.Consumers); + Assert.Equal(0.0, result.Snapshot.AckRate); + Assert.True(result.Snapshot.HasRequiredGauges); + } + + /// + /// Missing optional JSON fields remain null on the snapshot and is false. + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_missing_gauges_are_null_not_zero() + { + var client = CreateClient(new StubHandler(HttpStatusCode.OK, """{"state":"running"}""")); + var result = await client.GetQueueAsync("/", "q", CancellationToken.None); + Assert.True(result.IsSuccess); + Assert.Null(result.Snapshot!.MessagesReady); + Assert.Null(result.Snapshot.Consumers); + Assert.False(result.Snapshot.HasRequiredGauges); + } + + /// + /// HTTP 401 maps to with no snapshot. + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_auth_failure() + { + var client = CreateClient(new StubHandler(HttpStatusCode.Unauthorized, "{}")); + var result = await client.GetQueueAsync("/", "q", CancellationToken.None); + Assert.Equal(ManagementPollKind.AuthenticationFailure, result.Kind); + Assert.Null(result.Snapshot); + } + + /// + /// Non-success HTTP status codes (other than 401/404) classify as . + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_http_error() + { + var client = CreateClient(new StubHandler(HttpStatusCode.InternalServerError, "err")); + var result = await client.GetQueueAsync("/", "q", CancellationToken.None); + Assert.Equal(ManagementPollKind.TransientFailure, result.Kind); + } + + /// + /// HTTP 404 maps to . + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_queue_missing() + { + var client = CreateClient(new StubHandler(HttpStatusCode.NotFound, "{}")); + var result = await client.GetQueueAsync("/", "q", CancellationToken.None); + Assert.Equal(ManagementPollKind.QueueMissing, result.Kind); + } + + /// + /// Invalid JSON on HTTP 200 yields transient failure with a reason mentioning JSON parsing. + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_malformed_json() + { + var client = CreateClient(new StubHandler(HttpStatusCode.OK, "{not-json")); + var result = await client.GetQueueAsync("/", "q", CancellationToken.None); + Assert.Equal(ManagementPollKind.TransientFailure, result.Kind); + Assert.Contains("Json", result.Reason, StringComparison.Ordinal); + } + + /// + /// An in-flight request cancelled via returns . + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_respects_cancellation() + { + var handler = new StubHandler(HttpStatusCode.OK, "{}") { Delay = TimeSpan.FromSeconds(30) }; + var client = CreateClient(handler); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); + var result = await client.GetQueueAsync("/", "q", cts.Token); + Assert.Equal(ManagementPollKind.Cancelled, result.Kind); + } + + /// + /// Null management base URL fails before HTTP with transient failure and a reason referencing BaseUrl. + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_missing_management_base_url_is_transient_failure() + { + var factory = new StubHttpClientFactory(new StubHandler(HttpStatusCode.OK, "{}")); + var rabbit = Options.Create(new RabbitMqOptions + { + Username = "user", + Password = "pass", + Management = new RabbitMqManagementOptions { BaseUrl = null }, + }); + var client = new ManagementQueueClient( + factory, + rabbit, + new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z"))); + var result = await client.GetQueueAsync("/", "q", CancellationToken.None); + Assert.Equal(ManagementPollKind.TransientFailure, result.Kind); + Assert.Contains("BaseUrl", result.Reason, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Each poll reads current username/password from the live instance, + /// including after . + /// + /// A task that completes when assertions pass. + [Fact] + public async Task GetQueueAsync_uses_live_options_credentials_and_picks_up_account_refresh() + { + var rabbit = new RabbitMqOptions + { + Username = "before-user", + Password = "before-pass", + Management = new RabbitMqManagementOptions + { + BaseUrl = "http://127.0.0.1:15672", + RequestTimeoutSeconds = 2, + }, + }; + var capturing = new CapturingHandler(HttpStatusCode.OK, """{"messages_ready":0,"messages_unacknowledged":0,"consumers":0}"""); + var client = new ManagementQueueClient( + new StubHttpClientFactory(capturing), + Options.Create(rabbit), + new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z"))); + + _ = await client.GetQueueAsync("/", "backfiller.giganews", CancellationToken.None); + Assert.Equal( + Convert.ToBase64String(Encoding.ASCII.GetBytes("before-user:before-pass")), + Assert.Single(capturing.AuthorizationParameters)); + + // Same path as ProviderAccountConfigurationService → CopyRabbitMqCredentialsTo on the live options instance. + new NntpSharedConfiguration( + MaxArticleBytes: 1, + SiteName: "site", + PrometheusUrl: "http://p", + AcmeDirectoryUrl: "https://acme", + AcmeRenewalThresholdDays: 30, + CloudFlareZoneId: "zone", + DnsSuffix: "example.net", + AcmeAccount: "a@example.net", + AcmeCertificatePassword: "x", + CloudFlareApiKey: "k", + RabbitMqPassword: "after-pass", + RabbitMqUsername: "after-user").CopyRabbitMqCredentialsTo(rabbit); + + _ = await client.GetQueueAsync("/", "backfiller.giganews", CancellationToken.None); + Assert.Equal(2, capturing.AuthorizationParameters.Count); + Assert.Equal( + Convert.ToBase64String(Encoding.ASCII.GetBytes("after-user:after-pass")), + capturing.AuthorizationParameters[1]); + Assert.Equal("after-user", rabbit.Username); + Assert.Equal("after-pass", rabbit.Password); + Assert.Equal("http://127.0.0.1:15672", rabbit.Management.BaseUrl); + } + + /// + /// Shared-queue management auth failure does not block instance-level dispatch-stall observation on the engine. + /// + /// A task that completes when assertions pass. + [Fact] + public async Task Management_auth_failure_does_not_prevent_subsequent_instance_observation() + { + var client = CreateClient(new StubHandler(HttpStatusCode.Unauthorized, "{}")); + var poll = await client.GetQueueAsync("/", "backfiller.giganews", CancellationToken.None); + Assert.Equal(ManagementPollKind.AuthenticationFailure, poll.Kind); + + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + _ = engine.ObserveSharedQueue("backfiller.giganews", poll, expectedActive: true); + _ = engine.ObserveInstance( + [ + new SilentConsumerSessionObservation( + "Giganews", + 1, + ArticleWorkConsumerState.Running, + QueuedCount: 2, + ActiveCount: 0, + HasOpenChannel: true, + IsActiveProcessingStalled: false), + ], + desiredRunningSessions: 1); + time.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + var opened = engine.ObserveInstance( + [ + new SilentConsumerSessionObservation( + "Giganews", + 1, + ArticleWorkConsumerState.Running, + QueuedCount: 2, + ActiveCount: 0, + HasOpenChannel: true, + IsActiveProcessingStalled: false), + ], + 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall); + Assert.Equal(ObservationTransition.Opened, opened.Transition); + Assert.Equal(ObservationScope.Instance, opened.Key.Scope); + } + + /// + /// Non-numeric ack_details.rate in JSON leaves null without failing the parse. + /// + [Fact] + public void Parse_ignores_invalid_rate() + { + using var doc = System.Text.Json.JsonDocument.Parse( + """{"messages_ready":1,"messages_unacknowledged":1,"consumers":1,"message_stats":{"ack_details":{"rate":"x"}}}"""); + var snap = ManagementQueueClient.Parse( + doc.RootElement, + "/", + "q", + DateTimeOffset.UtcNow); + Assert.Null(snap.AckRate); + Assert.Equal(1, snap.MessagesReady); + } + + /// + /// Builds a with stub HTTP, fixed RabbitMQ options, and a fake clock. + /// + /// Handler that simulates management HTTP responses. + /// Configured client under test. + private static ManagementQueueClient CreateClient(HttpMessageHandler handler) + { + var factory = new StubHttpClientFactory(handler); + var rabbit = Options.Create(new RabbitMqOptions + { + Username = "user", + Password = "pass", + Management = new RabbitMqManagementOptions + { + BaseUrl = "http://127.0.0.1:15672", + RequestTimeoutSeconds = 2, + }, + }); + return new ManagementQueueClient( + factory, + rabbit, + new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z"))); + } + + /// + /// Returns a new per factory call wrapping the same test handler (not production pooling). + /// + /// Shared handler instance returned on every call. + private sealed class StubHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory + { + /// Creates a client with infinite timeout so test handlers control delay/cancellation. + /// Ignored; production uses named clients. + /// HttpClient that does not dispose the shared handler. + public HttpClient CreateClient(string name) => new(handler, disposeHandler: false) + { + Timeout = Timeout.InfiniteTimeSpan, + }; + } + + /// Deterministic HTTP handler returning fixed status and JSON body; optionally delays to exercise cancellation. + private sealed class StubHandler : HttpMessageHandler + { + /// Fixed HTTP status returned by every . + private readonly HttpStatusCode _status; + + /// Fixed response body (JSON text) returned by every . + private readonly string _body; + + /// Fixes response status and body for every request. + /// HTTP status code to return. + /// Response entity body (UTF-8 JSON). + public StubHandler(HttpStatusCode status, string body) + { + _status = status; + _body = body; + } + + /// When positive, awaits this delay inside before responding. + public TimeSpan Delay { get; init; } + + /// + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Assert.NotNull(request.Headers.Authorization); + if (Delay > TimeSpan.Zero) + { + await Task.Delay(Delay, cancellationToken); + } + + return new HttpResponseMessage(_status) + { + Content = new StringContent(_body, Encoding.UTF8, "application/json"), + }; + } + } + + /// + /// Like but records each Basic auth parameter for credential refresh assertions. + /// + /// Fixed HTTP status for every response. + /// Fixed JSON response body. + private sealed class CapturingHandler(HttpStatusCode status, string body) : HttpMessageHandler + { + /// Base64 Basic credentials parameter from each captured request, in order. + public List AuthorizationParameters { get; } = []; + + /// + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Assert.NotNull(request.Headers.Authorization); + Assert.Equal("Basic", request.Headers.Authorization.Scheme); + AuthorizationParameters.Add(request.Headers.Authorization.Parameter ?? string.Empty); + return Task.FromResult( + new HttpResponseMessage(status) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }); + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ObservationStateMachineTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ObservationStateMachineTests.cs new file mode 100644 index 00000000..2ef07f63 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ObservationStateMachineTests.cs @@ -0,0 +1,45 @@ +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// Regression for a single : open-window timing, continued-open ticks, resolve-window recovery, + /// and Unknown signals that freeze an open alert without opening or resolving prematurely. + /// + public sealed class ObservationStateMachineTests + { + /// + /// Active condition must persist for the full open window before ; + /// subsequent Active ticks while open yield . + /// + [Fact] + public void Opens_after_window_once() + { + var key = new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.ProcessingStall, "q"); + var machine = new ObservationStateMachine(key, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); + var t0 = DateTimeOffset.Parse("2026-10-25T20:00:00Z"); + Assert.Equal(ObservationTransition.None, machine.Apply(ConditionSignal.Active, t0).Transition); + Assert.Equal(ObservationTransition.Opened, machine.Apply(ConditionSignal.Active, t0.AddSeconds(30)).Transition); + Assert.Equal(ObservationTransition.Continued, machine.Apply(ConditionSignal.Active, t0.AddSeconds(40)).Transition); + } + + /// + /// After open, produces no transition and leaves phase Open; + /// Inactive must run through the resolve window before . + /// + [Fact] + public void Unknown_does_not_open_or_resolve() + { + // Contract: Unknown freezes an open alert; it is neither recovery nor a new open. + var key = new ObservationKey(ObservationScope.SharedQueue, SilentConsumerCondition.ProcessingStall, "q"); + var machine = new ObservationStateMachine(key, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); + var t0 = DateTimeOffset.Parse("2026-10-25T20:00:00Z"); + Assert.Equal(ObservationTransition.None, machine.Apply(ConditionSignal.Active, t0).Transition); + Assert.Equal(ObservationTransition.Opened, machine.Apply(ConditionSignal.Active, t0.AddSeconds(1)).Transition); + Assert.Equal(ObservationTransition.None, machine.Apply(ConditionSignal.Unknown, t0.AddSeconds(2)).Transition); + Assert.Equal(ObservationPhase.Open, machine.Phase); + Assert.Equal(ObservationTransition.Continued, machine.Apply(ConditionSignal.Inactive, t0.AddSeconds(2)).Transition); + Assert.Equal(ObservationTransition.Resolved, machine.Apply(ConditionSignal.Inactive, t0.AddSeconds(3)).Transition); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/QueueSampleHistoryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/QueueSampleHistoryTests.cs new file mode 100644 index 00000000..b218bc64 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/QueueSampleHistoryTests.cs @@ -0,0 +1,150 @@ +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// retention, success/failure sample semantics, and + /// edge cases (no invented zeros on failed samples). + /// + public sealed class QueueSampleHistoryTests + { + /// + /// Counter rate is the delta of the selected total divided by elapsed seconds between two valid samples. + /// + [Fact] + public void Counter_rate_increases_normally() + { + var older = Valid(0, ackTotal: 10); + var newer = Valid(10, ackTotal: 20); + var rate = QueueSampleHistory.TryComputeCounterRate(older, newer, static s => s.AckTotal); + Assert.Equal(1.0, rate); + } + + /// + /// When the counter does not increase, the computed rate is zero rather than null. + /// + [Fact] + public void Counter_rate_no_progress_is_zero() + { + var older = Valid(0, ackTotal: 10); + var newer = Valid(10, ackTotal: 10); + var rate = QueueSampleHistory.TryComputeCounterRate(older, newer, static s => s.AckTotal); + Assert.Equal(0.0, rate); + } + + /// + /// A decreasing counter total (reset) yields null so callers do not treat the drop as negative progress. + /// + [Fact] + public void Counter_reset_returns_null() + { + var older = Valid(0, ackTotal: 50); + var newer = Valid(10, ackTotal: 1); + Assert.Null(QueueSampleHistory.TryComputeCounterRate(older, newer, static s => s.AckTotal)); + } + + /// + /// Either sample marked invalid or missing snapshot prevents rate computation. + /// + [Fact] + public void Missing_or_invalid_samples_return_null() + { + var valid = Valid(0, ackTotal: 1); + var invalid = new QueueSampleEntry( + DateTimeOffset.UtcNow, + IsValid: false, + Snapshot: null, + ManagementPollKind.TransientFailure, + "fail"); + Assert.Null(QueueSampleHistory.TryComputeCounterRate(invalid, valid, static s => s.AckTotal)); + Assert.Null(QueueSampleHistory.TryComputeCounterRate(valid, invalid, static s => s.AckTotal)); + } + + /// + /// Non-positive elapsed time between sample timestamps returns null. + /// + [Fact] + public void Non_positive_elapsed_returns_null() + { + var a = Valid(5, ackTotal: 1); + var b = Valid(5, ackTotal: 2); + Assert.Null(QueueSampleHistory.TryComputeCounterRate(a, b, static s => s.AckTotal)); + } + + /// + /// History drops oldest entries when capacity is exceeded and reflects the newest samples. + /// + [Fact] + public void History_is_bounded_and_retains_newest() + { + var history = new QueueSampleHistory(3); + for (var i = 0; i < 5; i++) + { + history.AddSuccess(Snap(ready: i, consumers: 1, unacked: 0)); + } + + Assert.Equal(3, history.Count); + var snap = history.Snapshot(); + Assert.Equal(2, snap[0].Snapshot!.MessagesReady); + Assert.Equal(4, snap[2].Snapshot!.MessagesReady); + } + + /// + /// Failed polls record invalid entries with null snapshot rather than synthesizing zero gauges. + /// + [Fact] + public void Failure_does_not_invent_zeros() + { + var history = new QueueSampleHistory(4); + history.AddFailure( + DateTimeOffset.UtcNow, + ManagementPollKind.TransientFailure, + "timeout"); + var entry = Assert.Single(history.Snapshot()); + Assert.False(entry.IsValid); + Assert.Null(entry.Snapshot); + } + + /// Builds a valid history entry with fixed base time plus offset seconds. + /// Seconds added to the canonical sample timestamp. + /// Ack counter total on the embedded snapshot. + /// A valid for rate tests. + private static QueueSampleEntry Valid(int seconds, long ackTotal) + { + var at = DateTimeOffset.Parse("2026-10-25T20:00:00Z").AddSeconds(seconds); + return new QueueSampleEntry( + at, + IsValid: true, + new ManagementQueueSnapshot + { + QueueName = "q", + VirtualHost = "/", + SampledAtUtc = at, + MessagesReady = 0, + MessagesUnacknowledged = 1, + Consumers = 1, + AckTotal = ackTotal, + }, + null, + null); + } + + /// Minimal successful management snapshot for history capacity tests. + /// . + /// . + /// . + /// Optional ack total; omitted when null. + /// Snapshot stamped at the canonical test UTC instant. + private static ManagementQueueSnapshot Snap(long ready, long consumers, long unacked, long? ackTotal = null) => + new() + { + QueueName = "q", + VirtualHost = "/", + SampledAtUtc = DateTimeOffset.Parse("2026-10-25T20:00:00Z"), + MessagesReady = ready, + MessagesUnacknowledged = unacked, + Consumers = consumers, + AckTotal = ackTotal, + }; + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerDispatchStallLifecycleTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerDispatchStallLifecycleTests.cs new file mode 100644 index 00000000..32692ea4 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerDispatchStallLifecycleTests.cs @@ -0,0 +1,520 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.RabbitMq; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// DispatchStall / ProcessingStall / local ConsumerLoss through the real session dispatch path + /// ( → QueuedCount/ActiveCount → + /// → engine), not synthetic observation structs. + /// Fake RabbitMQ channels; blocked tasks are always released in each test. + /// + public sealed class SilentConsumerDispatchStallLifecycleTests + { + /// + /// Holding after dispatch acquire opens instance dispatch stall; + /// healthy shared-queue poll does not open shared dispatch stall for the same scenario. + /// + /// A task that completes when assertions pass and blocked delivery is released. + [Fact] + public async Task Dispatch_blocked_before_active_opens_session_dispatch_stall() + { + var monitorTime = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, monitorTime); + await using var harness = await SessionHarness.StartAsync(connectionNumber: 1, connectionLimit: 1); + + harness.Session.DispatchAcquired = NewSource(); + harness.Session.DispatchAcquireHold = NewSource(); + var processing = harness.Deliver(7); + await harness.Session.DispatchAcquired.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(ArticleWorkConsumerState.Running, harness.Session.State); + Assert.True(harness.Session.Channel is { IsOpen: true }); + Assert.Equal(1, harness.Session.QueuedCount); + Assert.Equal(0, harness.Session.ActiveCount); + Assert.False(harness.Session.IsActiveProcessingStalled(ArticleWorkConsumerService.ProcessingStallThreshold)); + + var observation = SilentConsumerSessionObservation.FromSession( + harness.Session, + ArticleWorkConsumerService.ProcessingStallThreshold); + Assert.Equal(1, observation.QueuedCount); + Assert.Equal(0, observation.ActiveCount); + Assert.Equal(harness.Session.SessionKey, observation.SessionKey); + + _ = engine.ObserveInstance([observation], desiredRunningSessions: 1); + monitorTime.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + var opened = engine + .ObserveInstance( + [ + SilentConsumerSessionObservation.FromSession( + harness.Session, + ArticleWorkConsumerService.ProcessingStallThreshold), + ], + 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall); + + Assert.Equal(ObservationTransition.Opened, opened.Transition); + Assert.Equal(ObservationScope.Instance, opened.Key.Scope); + Assert.Equal(harness.Session.SessionKey, opened.Key.QueueOrBackbone); + Assert.Equal("a.example", engine.InstanceId); + Assert.Contains("DispatchAcquireHold", opened.Detail, StringComparison.Ordinal); + Assert.DoesNotContain( + engine.ObserveSharedQueue( + "backfiller.giganews", + SharedSuccess(consumers: 1, ready: 0, unacked: 1, ackRate: 5), + expectedActive: true), + r => r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Transition == ObservationTransition.Opened); + + harness.Session.DispatchAcquireHold.TrySetResult(); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + } + + /// + /// A running session with no queued or active work stays inactive for dispatch stall through the open window. + /// + /// A task that completes when assertions pass. + [Fact] + public async Task Idle_running_session_does_not_open_dispatch_stall() + { + var monitorTime = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, monitorTime); + await using var harness = await SessionHarness.StartAsync(connectionNumber: 1, connectionLimit: 1); + + Assert.Equal(0, harness.Session.QueuedCount); + Assert.Equal(0, harness.Session.ActiveCount); + + var observation = SilentConsumerSessionObservation.FromSession( + harness.Session, + ArticleWorkConsumerService.ProcessingStallThreshold); + _ = engine.ObserveInstance([observation], 1); + monitorTime.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + var result = engine + .ObserveInstance( + [ + SilentConsumerSessionObservation.FromSession( + harness.Session, + ArticleWorkConsumerService.ProcessingStallThreshold), + ], + 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall); + + Assert.Equal(ConditionSignal.Inactive, result.Signal); + Assert.Equal(ObservationPhase.Clear, result.Phase); + Assert.Contains("idle", result.Detail, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Dispatch stall opens only on the blocked session when a healthy sibling shares the same backbone and connection pool. + /// + /// A task that completes when assertions pass and the blocked delivery is released. + [Fact] + public async Task Blocked_session_dispatch_stall_independent_of_healthy_sibling_same_backbone() + { + var monitorTime = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, monitorTime); + await using var shared = await SharedConnection.StartAsync(); + await using var stalled = await SessionHarness.StartAsync( + shared.Connections, + connectionNumber: 1, + connectionLimit: 2); + await using var healthy = await SessionHarness.StartAsync( + shared.Connections, + connectionNumber: 2, + connectionLimit: 2); + + stalled.Session.DispatchAcquired = NewSource(); + stalled.Session.DispatchAcquireHold = NewSource(); + var processing = stalled.Deliver(7); + await stalled.Session.DispatchAcquired.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(1, stalled.Session.QueuedCount); + Assert.Equal(0, stalled.Session.ActiveCount); + Assert.Equal(0, healthy.Session.QueuedCount); + + SilentConsumerSessionObservation[] Project() => + [ + SilentConsumerSessionObservation.FromSession( + stalled.Session, + ArticleWorkConsumerService.ProcessingStallThreshold), + SilentConsumerSessionObservation.FromSession( + healthy.Session, + ArticleWorkConsumerService.ProcessingStallThreshold), + ]; + + _ = engine.ObserveInstance(Project(), desiredRunningSessions: 2); + monitorTime.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + var results = engine.ObserveInstance(Project(), 2); + + var stalledKey = stalled.Session.SessionKey; + var healthyKey = healthy.Session.SessionKey; + Assert.Equal( + ObservationTransition.Opened, + results.Single(r => + r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == stalledKey).Transition); + Assert.Equal( + ConditionSignal.Inactive, + results.Single(r => + r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == healthyKey).Signal); + + monitorTime.Advance(SilentConsumerMonitorDefaults.DispatchStall.ResolveWindow); + Assert.Equal( + ObservationPhase.Open, + engine.ObserveInstance(Project(), 2) + .Single(r => + r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == stalledKey) + .Phase); + + stalled.Session.DispatchAcquireHold.TrySetResult(); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + } + + /// + /// Releasing dispatch hold on one session resolves its dispatch stall after the resolve window while the other stays open. + /// + /// A task that completes when assertions pass and both deliveries finish. + [Fact] + public async Task Releasing_dispatch_hold_resolves_dispatch_stall_only_for_that_session() + { + var monitorTime = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, monitorTime); + await using var shared = await SharedConnection.StartAsync(); + await using var session1 = await SessionHarness.StartAsync( + shared.Connections, + connectionNumber: 1, + connectionLimit: 2); + await using var session2 = await SessionHarness.StartAsync( + shared.Connections, + connectionNumber: 2, + connectionLimit: 2); + + session1.Session.DispatchAcquired = NewSource(); + session1.Session.DispatchAcquireHold = NewSource(); + session2.Session.DispatchAcquired = NewSource(); + session2.Session.DispatchAcquireHold = NewSource(); + + var p1 = session1.Deliver(1); + var p2 = session2.Deliver(2); + await session1.Session.DispatchAcquired.Task.WaitAsync(TimeSpan.FromSeconds(2)); + await session2.Session.DispatchAcquired.Task.WaitAsync(TimeSpan.FromSeconds(2)); + + SilentConsumerSessionObservation[] Project() => + [ + SilentConsumerSessionObservation.FromSession( + session1.Session, + ArticleWorkConsumerService.ProcessingStallThreshold), + SilentConsumerSessionObservation.FromSession( + session2.Session, + ArticleWorkConsumerService.ProcessingStallThreshold), + ]; + + _ = engine.ObserveInstance(Project(), 2); + monitorTime.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + var opened = engine.ObserveInstance(Project(), 2); + Assert.All( + opened.Where(r => r.Key.Condition == SilentConsumerCondition.DispatchStall), + r => Assert.Equal(ObservationTransition.Opened, r.Transition)); + + // Session 1 recovers; session 2 remains blocked. + session1.Session.DispatchAcquireHold.TrySetResult(); + await p1.WaitAsync(TimeSpan.FromSeconds(2)); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => session1.Session.QueuedCount == 0 && session1.Session.ActiveCount == 0, + TimeSpan.FromSeconds(2)); + + _ = engine.ObserveInstance(Project(), 2); + monitorTime.Advance(SilentConsumerMonitorDefaults.DispatchStall.ResolveWindow); + var after = engine.ObserveInstance(Project(), 2); + + Assert.Equal( + ObservationTransition.Resolved, + after.Single(r => + r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == session1.Session.SessionKey).Transition); + Assert.Equal( + ObservationPhase.Open, + after.Single(r => + r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == session2.Session.SessionKey).Phase); + + session2.Session.DispatchAcquireHold.TrySetResult(); + await p2.WaitAsync(TimeSpan.FromSeconds(2)); + } + + /// + /// Active processing held past opens instance processing stall from real session metrics. + /// + /// A task that completes when assertions pass and the handler gate is released. + [Fact] + public async Task Active_processing_stall_projects_from_real_session_into_instance_alert() + { + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, clock); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "held", + Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn, + ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort, + Started = NewSource(), + Gate = NewSource(), + }; + await using var harness = await SessionHarness.StartAsync( + connectionNumber: 1, + connectionLimit: 1, + handler: handler, + time: clock); + + var processing = harness.Deliver(9); + await handler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(1, harness.Session.ActiveCount); + Assert.True(harness.Session.HasActiveProcessing); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + Assert.True( + harness.Session.IsActiveProcessingStalled(ArticleWorkConsumerService.ProcessingStallThreshold)); + + var observation = SilentConsumerSessionObservation.FromSession( + harness.Session, + ArticleWorkConsumerService.ProcessingStallThreshold); + Assert.True(observation.IsActiveProcessingStalled); + + _ = engine.ObserveInstance([observation], 1); + clock.Advance(SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow); + var opened = engine + .ObserveInstance( + [ + SilentConsumerSessionObservation.FromSession( + harness.Session, + ArticleWorkConsumerService.ProcessingStallThreshold), + ], + 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall); + + Assert.Equal(ObservationTransition.Opened, opened.Transition); + Assert.Equal(ObservationScope.Instance, opened.Key.Scope); + Assert.Equal(harness.Session.SessionKey, opened.Key.QueueOrBackbone); + Assert.Contains("5328", opened.Detail, StringComparison.Ordinal); + + // Monitor does not retire; release handler ourselves. + handler.Gate!.TrySetResult(); + await processing.WaitAsync(TimeSpan.FromSeconds(2)); + } + + /// + /// After session retire, instance consumer loss is active when desired running sessions exceed observed running channels. + /// + /// A task that completes when assertions pass. + [Fact] + public async Task Local_consumer_loss_projects_when_desired_sessions_exist_but_none_running() + { + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, TimeProvider.System); + await using var harness = await SessionHarness.StartAsync(connectionNumber: 1, connectionLimit: 1); + await harness.Session.RetireAsync(); + Assert.Equal(ArticleWorkConsumerState.Stopped, harness.Session.State); + + var observation = SilentConsumerSessionObservation.FromSession( + harness.Session, + ArticleWorkConsumerService.ProcessingStallThreshold); + Assert.False(observation.HasOpenChannel || observation.State == ArticleWorkConsumerState.Running); + + var loss = engine.ObserveInstance([observation], desiredRunningSessions: 2) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ObservationScope.Instance, loss.Key.Scope); + Assert.Equal("local", loss.Key.QueueOrBackbone); + Assert.Equal(ConditionSignal.Active, loss.Signal); + } + + /// Task completion source with continuations off the completing thread (test synchronization). + /// A new unset . + private static TaskCompletionSource NewSource() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Successful shared-queue management poll for cross-scope assertions in dispatch tests. + /// Consumer gauge. + /// Messages ready. + /// Unacknowledged messages. + /// Ack rate. + /// with a Giganews queue snapshot. + private static ManagementPollResult SharedSuccess( + long consumers, + long ready, + long unacked, + double ackRate) => + new() + { + Kind = ManagementPollKind.Success, + Reason = "ok", + Snapshot = new ManagementQueueSnapshot + { + QueueName = "backfiller.giganews", + VirtualHost = "/", + SampledAtUtc = DateTimeOffset.Parse("2026-10-25T20:00:00Z"), + Consumers = consumers, + MessagesReady = ready, + MessagesUnacknowledged = unacked, + AckRate = ackRate, + DeliverGetRate = 0, + }, + }; + + /// + /// Owns one started fake shared by multiple session harnesses on the same connection generation. + /// + private sealed class SharedConnection : IAsyncDisposable + { + /// Wraps an already-started broker service. + /// Started RabbitMQ service from test factory helpers. + private SharedConnection(RabbitMqService connections) => Connections = connections; + + /// Broker service used to create multiple consumer sessions without separate connection factories. + public RabbitMqService Connections { get; } + + /// Starts fake RabbitMQ via . + /// Shared connection holder; dispose to tear down the service. + public static async Task StartAsync() + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + var connections = RabbitMqServiceTests.CreateService(factory); + await connections.StartAsync(CancellationToken.None); + return new SharedConnection(connections); + } + + /// + public async ValueTask DisposeAsync() => await Connections.DisposeAsync(); + } + + /// + /// Starts a real against fake RabbitMQ for end-to-end dispatch and stall projection. + /// + private sealed class SessionHarness : IAsyncDisposable + { + /// Creates a harness around an already-started session and channel. + /// Non-null when this harness started its own . + /// Running consumer session under test. + /// Fake channel cast from the session. + private SessionHarness( + RabbitMqService? ownedConnections, + ArticleWorkConsumerSession session, + FakeBackFillerRabbitMqChannel channel) + { + OwnedConnections = ownedConnections; + Session = session; + Channel = channel; + } + + /// Broker service disposed only when created by this harness (not when injected via shared connection). + private RabbitMqService? OwnedConnections { get; } + + /// Consumer session exercising the production dispatch pipeline. + public ArticleWorkConsumerSession Session { get; } + + /// Fake AMQP channel used to inject deliveries. + public FakeBackFillerRabbitMqChannel Channel { get; } + + /// Delivers a canonical test message with the given delivery tag. + /// Broker delivery tag. + /// The channel deliver task (handler processing). + public Task Deliver(ulong deliveryTag) => + Channel.DeliverAsync(ArticleWorkTestDeliveries.Canonical(deliveryTag)); + + /// Starts a session with a dedicated fake broker connection factory. + /// Session slot within the backbone limit. + /// Max concurrent sessions for the backbone. + /// Optional article handler; default auto-completing success handler. + /// Clock for stall timing; defaults to system time when null. + /// Started harness owning its broker service when connections are not shared. + public static Task StartAsync( + int connectionNumber, + int connectionLimit, + IArticleWorkHandler? handler = null, + TimeProvider? time = null) => + StartAsync(connections: null, connectionNumber, connectionLimit, handler, time); + + /// + /// Starts a session on optional shared ; uses fake RabbitMQ, not a live broker. + /// + /// Shared broker service, or null to create and own one. + /// Session slot within the backbone limit. + /// Max concurrent sessions for the backbone. + /// Optional article handler. + /// Clock for stall timing. + /// Started harness with session channel exposed as . + public static async Task StartAsync( + IRabbitMqService? connections, + int connectionNumber, + int connectionLimit, + IArticleWorkHandler? handler = null, + TimeProvider? time = null) + { + // Only own the broker service when this harness created it. + RabbitMqService? owned = null; + IRabbitMqService service; + if (connections is null) + { + var factory = new FakeBackFillerRabbitMqConnectionFactory(); + owned = RabbitMqServiceTests.CreateService(factory); + await owned.StartAsync(CancellationToken.None); + service = owned; + } + else + { + service = connections; + } + + handler ??= new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.Success, + Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn, + ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort, + Gate = NewSource(), + }; + // Auto-complete fast handlers that are not intentionally held. + if (handler is ControllableArticleWorkHandler controllable + && controllable.Gate is not null + && controllable.Started is null + && controllable.Outcome == ArticleWorkOutcome.Success) + { + controllable.Gate.TrySetResult(); + } + + var session = new ArticleWorkConsumerSession( + "Giganews", + prefetch: 1, + new ArticleWorkDeliveryPipeline( + handler, + new RecordingArticleWorkResponsePublisher { CompletesSuccessPublication = true }, + 1024), + service, + NullLogger.Instance, + new BackFillerShutdownRuntimeOptions(TimeSpan.FromSeconds(30), DrainQueuedWork: true, FinishActiveArticles: true), + connectionNumber, + connectionLimit, + time); + await session.StartAsync(CancellationToken.None); + var channel = Assert.IsType(session.Channel); + return new SessionHarness(owned, session, channel); + } + + /// Disposes the session and owned broker service when applicable. + /// ValueTask completing when disposal finishes. + public async ValueTask DisposeAsync() + { + await Session.DisposeAsync(); + if (OwnedConnections is not null) + { + await OwnedConnections.DisposeAsync(); + } + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluatorAndEngineTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluatorAndEngineTests.cs new file mode 100644 index 00000000..7f5e2954 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluatorAndEngineTests.cs @@ -0,0 +1,340 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// Predicate-level contracts plus + /// open/resolve timing, Unknown freeze while shared alerts stay open, and event 9113 observation-unavailable rate limiting. + /// + public sealed class SilentConsumerEvaluatorAndEngineTests + { + /// + /// Shared processing stall is inactive when sample history shows positive ack rate with consumers and unacked work present. + /// + [Fact] + public void Shared_processing_stall_healthy_ack_is_inactive() + { + var history = History(Snap(consumers: 1, ready: 0, unacked: 1, ackRate: 1.0)); + Assert.Equal( + ConditionSignal.Inactive, + SilentConsumerEvaluators.EvaluateSharedProcessingStall( + history, + SilentConsumerMonitorDefaults.ProcessingStall, + out _)); + } + + /// + /// Zero ack rate with consumers and unacked messages yields an active shared processing-stall signal. + /// + [Fact] + public void Shared_processing_stall_no_progress_is_active() + { + var history = History(Snap(consumers: 1, ready: 0, unacked: 1, ackRate: 0.0)); + Assert.Equal( + ConditionSignal.Active, + SilentConsumerEvaluators.EvaluateSharedProcessingStall( + history, + SilentConsumerMonitorDefaults.ProcessingStall, + out _)); + } + + /// + /// Instance processing stall follows the session stall flag only; healthy active work is inactive and detail references the stall threshold. + /// + [Fact] + public void Instance_processing_stall_uses_existing_stall_predicate_only() + { + Assert.Equal( + ConditionSignal.Active, + SilentConsumerEvaluators.EvaluateInstanceProcessingStall( + Running("giganews", stalled: true, queued: 0, active: 1), + out var detail)); + Assert.Contains("5328", detail, StringComparison.Ordinal); + Assert.Equal( + ConditionSignal.Inactive, + SilentConsumerEvaluators.EvaluateInstanceProcessingStall( + Running("giganews", stalled: false, queued: 0, active: 1), + out _)); + } + + /// + /// Dispatch stall is active when queued work exists before active processing; idle running sessions are inactive. + /// + [Fact] + public void Instance_dispatch_stall_detects_queued_before_active() + { + Assert.Equal( + ConditionSignal.Active, + SilentConsumerEvaluators.EvaluateInstanceDispatchStall( + Running("giganews", stalled: false, queued: 2, active: 0), + out _)); + Assert.Equal( + ConditionSignal.Inactive, + SilentConsumerEvaluators.EvaluateInstanceDispatchStall( + Running("giganews", stalled: false, queued: 0, active: 0), + out var idleDetail)); + Assert.Contains("idle", idleDetail, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Engine opens dispatch stall per session key after the dispatch open window with queued-before-active observation. + /// + [Fact] + public void Engine_opens_per_session_dispatch_stall() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + _ = engine.ObserveInstance([Running("Giganews", stalled: false, queued: 3, active: 0, connectionNumber: 1)], 1); + time.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + var opened = engine.ObserveInstance([Running("Giganews", stalled: false, queued: 3, active: 0, connectionNumber: 1)], 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall); + Assert.Equal(ObservationTransition.Opened, opened.Transition); + Assert.Equal( + ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 1), + opened.Key.QueueOrBackbone); + } + + /// + /// Shared processing stall opens once after the open window, continues while active, and resolves after recovery and the resolve window. + /// + [Fact] + public void Engine_opens_shared_processing_stall_once_and_resolves_on_recovery() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + + ObservationResult LastStall(IReadOnlyList results) => + results.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.Scope == ObservationScope.SharedQueue); + + Assert.Equal( + ObservationTransition.None, + LastStall(engine.ObserveSharedQueue("backfiller.giganews", Success(1, 0, 1, 0), true)).Transition); + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow); + Assert.Equal( + ObservationTransition.Opened, + LastStall(engine.ObserveSharedQueue("backfiller.giganews", Success(1, 0, 1, 0), true)).Transition); + Assert.Equal( + ObservationTransition.Continued, + LastStall(engine.ObserveSharedQueue("backfiller.giganews", Success(1, 0, 0, 1), true)).Transition); + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.ResolveWindow); + Assert.Equal( + ObservationTransition.Resolved, + LastStall(engine.ObserveSharedQueue("backfiller.giganews", Success(1, 0, 0, 1), true)).Transition); + } + + /// + /// Open shared consumer-loss stays frozen on Unknown polls; local dispatch stall can still open; resumed valid samples continue then resolve normally. + /// + [Fact] + public void Open_shared_alert_stays_open_on_unknown_then_reevaluates_when_management_resumes() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + + ObservationResult LastConsumerLoss(IReadOnlyList results) => + results.Single(r => + r.Key.Condition == SilentConsumerCondition.ConsumerLoss + && r.Key.Scope == ObservationScope.SharedQueue); + + _ = engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), expectedActive: true); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + Assert.Equal( + ObservationTransition.Opened, + LastConsumerLoss(engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true)).Transition); + + var unknownPoll = new ManagementPollResult + { + Kind = ManagementPollKind.TransientFailure, + Reason = "timeout", + }; + var duringOutage = LastConsumerLoss(engine.ObserveSharedQueue("backfiller.giganews", unknownPoll, true)); + Assert.Equal(ConditionSignal.Unknown, duringOutage.Signal); + Assert.Equal(ObservationTransition.None, duringOutage.Transition); + Assert.Equal(ObservationPhase.Open, duringOutage.Phase); + + // Local monitoring must still be able to open while shared stays frozen Open. + _ = engine.ObserveInstance( + [ + new SilentConsumerSessionObservation( + "Giganews", + 1, + ArticleWorkConsumerState.Running, + QueuedCount: 2, + ActiveCount: 0, + HasOpenChannel: true, + IsActiveProcessingStalled: false), + ], + desiredRunningSessions: 1); + time.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + Assert.Equal( + ObservationTransition.Opened, + engine.ObserveInstance( + [ + new SilentConsumerSessionObservation( + "Giganews", + 1, + ArticleWorkConsumerState.Running, + QueuedCount: 2, + ActiveCount: 0, + HasOpenChannel: true, + IsActiveProcessingStalled: false), + ], + 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall) + .Transition); + Assert.Equal( + ObservationPhase.Open, + LastConsumerLoss(engine.ObserveSharedQueue("backfiller.giganews", unknownPoll, true)).Phase); + + // Valid samples resume: still Active → Continued (not Resolved by the outage). + Assert.Equal( + ObservationTransition.Continued, + LastConsumerLoss(engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true)).Transition); + Assert.Equal(ObservationPhase.Open, LastConsumerLoss( + engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true)).Phase); + + // Conclusive clear after resume uses the normal resolve window — not Unknown-as-recovery. + _ = engine.ObserveSharedQueue("backfiller.giganews", Success(1, 5, 0, 0), true); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.ResolveWindow); + var resolved = LastConsumerLoss(engine.ObserveSharedQueue("backfiller.giganews", Success(1, 5, 0, 0), true)); + Assert.Equal(ObservationTransition.Resolved, resolved.Transition); + Assert.DoesNotContain("removed from catalog", resolved.Detail ?? string.Empty, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Event 9113 logs once per observation key per open-window interval; distinct queue keys and advanced time emit additional entries. + /// + [Fact] + public void Observation_unavailable_while_open_is_rate_limited_per_observation_key() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var logger = new CollectingLogger(); + var engine = new SilentConsumerMonitorEngine(() => "a.example", logger, time); + var unknownPoll = new ManagementPollResult + { + Kind = ManagementPollKind.TransientFailure, + Reason = "timeout", + }; + + _ = engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), expectedActive: true); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + _ = engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true); + + _ = engine.ObserveSharedQueue("backfiller.giganews", unknownPoll, true); + Assert.Equal(1, logger.Entries.Count(e => e.EventId.Id == 9113)); + + // Same observation key within the open-window interval: suppressed. + _ = engine.ObserveSharedQueue("backfiller.giganews", unknownPoll, true); + Assert.Equal(1, logger.Entries.Count(e => e.EventId.Id == 9113)); + + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow); + _ = engine.ObserveSharedQueue("backfiller.giganews", unknownPoll, true); + Assert.Equal(2, logger.Entries.Count(e => e.EventId.Id == 9113)); + + // Distinct queue key is not suppressed by the first queue's limiter. + _ = engine.ObserveSharedQueue("backfiller.eweka", Success(0, 5, 0, 0), true); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + _ = engine.ObserveSharedQueue("backfiller.eweka", Success(0, 5, 0, 0), true); + _ = engine.ObserveSharedQueue("backfiller.eweka", unknownPoll, true); + Assert.Contains( + logger.Entries, + e => e.EventId.Id == 9113 && e.Message.Contains("backfiller.eweka", StringComparison.Ordinal)); + } + + /// + /// Invalid sample history yields for shared processing stall evaluation. + /// + [Fact] + public void Missing_metrics_are_unknown() + { + var invalid = new List + { + new( + DateTimeOffset.UtcNow, + IsValid: false, + Snapshot: null, + ManagementPollKind.TransientFailure, + "timeout"), + }; + Assert.Equal( + ConditionSignal.Unknown, + SilentConsumerEvaluators.EvaluateSharedProcessingStall( + invalid, + SilentConsumerMonitorDefaults.ProcessingStall, + out _)); + } + + /// Synthetic running session observation for evaluator and engine tests. + /// Provider backbone name. + /// Whether active processing exceeds the stall threshold. + /// Queued delivery count before active processing. + /// In-flight active article count. + /// Connection slot within the backbone. + /// Observation with an open channel and running state. + private static SilentConsumerSessionObservation Running( + string backbone, + bool stalled, + int queued, + int active, + int connectionNumber = 1) => + new( + backbone, + connectionNumber, + ArticleWorkConsumerState.Running, + queued, + active, + HasOpenChannel: true, + IsActiveProcessingStalled: stalled); + + /// Successful management poll wrapping a synthetic snapshot. + /// Consumer count gauge. + /// Messages ready gauge. + /// Unacknowledged message gauge. + /// Instant ack rate from management API. + /// result for engine shared-queue tests. + private static ManagementPollResult Success(long consumers, long ready, long unacked, double ackRate) => + new() + { + Kind = ManagementPollKind.Success, + Reason = "ok", + Snapshot = Snap(consumers, ready, unacked, ackRate), + }; + + /// Single-entry history list for shared evaluator tests. + /// Valid snapshot to wrap. + /// One valid at the snapshot timestamp. + private static IReadOnlyList History(ManagementQueueSnapshot snap) => + [ + new(snap.SampledAtUtc, true, snap, null, null), + ]; + + /// Minimal queue snapshot for evaluator inputs. + /// Consumer count. + /// Messages ready. + /// Unacknowledged messages. + /// Optional ack rate. + /// Snapshot with fixed queue name and sample time. + private static ManagementQueueSnapshot Snap( + long consumers, + long ready, + long unacked, + double? ackRate) => + new() + { + QueueName = "q", + VirtualHost = "/", + SampledAtUtc = DateTimeOffset.Parse("2026-10-25T20:00:00Z"), + Consumers = consumers, + MessagesReady = ready, + MessagesUnacknowledged = unacked, + AckRate = ackRate, + DeliverGetRate = 0, + }; + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptionsValidatorTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptionsValidatorTests.cs new file mode 100644 index 00000000..62e6d43d --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorOptionsValidatorTests.cs @@ -0,0 +1,22 @@ +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// Validates that accepts both enabled and disabled monitor flags + /// without inspecting RabbitMQ management or other infrastructure on this options type. + /// + public sealed class SilentConsumerMonitorOptionsValidatorTests + { + /// + /// succeeds for Enabled true and false. + /// + [Fact] + public void Enabled_and_disabled_both_succeed() + { + var validator = new SilentConsumerMonitorOptionsValidator(); + Assert.True(validator.Validate(null, new SilentConsumerMonitorOptions { Enabled = true }).Succeeded); + Assert.True(validator.Validate(null, new SilentConsumerMonitorOptions { Enabled = false }).Succeeded); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorTargetsTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorTargetsTests.cs new file mode 100644 index 00000000..35d78af6 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorTargetsTests.cs @@ -0,0 +1,76 @@ +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; +using VectorNNTPD.BackFillerService.Nntp; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// maps provider catalog entries and backbone usable capacity + /// to per-queue monitor targets, including ExpectedActive when capacity is positive. + /// + public sealed class SilentConsumerMonitorTargetsTests + { + /// + /// Providers with positive usable capacity become expected-active queue targets; zero-capacity providers are included + /// with ExpectedActive false; zero MaxSessions catalog entries are omitted. + /// + [Fact] + public void Resolves_catalog_queues_without_json_list() + { + var catalog = new StaticBackFillerProviderCatalog( + [ + Provider("Giganews", maxSessions: 2), + Provider("Eweka", maxSessions: 1), + Provider("Idle", maxSessions: 0), + ]); + var capacity = new BackboneUsableCapacityState(); + capacity.PublishSnapshot(new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Giganews"] = 2, + ["Eweka"] = 0, + }); + + var targets = SilentConsumerMonitorTargets.Resolve(catalog, capacity); + Assert.Equal(2, targets.Count); + Assert.Contains(targets, t => t.QueueName == "backfiller.giganews" && t.ExpectedActive); + Assert.Contains(targets, t => t.QueueName == "backfiller.eweka" && !t.ExpectedActive); + Assert.DoesNotContain(targets, t => t.Backbone.Equals("Idle", StringComparison.OrdinalIgnoreCase)); + } + + /// + /// Adding a provider to the catalog yields a new queue target on the next resolve without requiring a JSON queue list. + /// + [Fact] + public void Catalog_change_adds_new_queue_target() + { + var capacity = new BackboneUsableCapacityState(); + capacity.PublishSnapshot(new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Giganews"] = 1, + ["Eweka"] = 1, + }); + + var first = SilentConsumerMonitorTargets.Resolve( + new StaticBackFillerProviderCatalog([Provider("Giganews", 1)]), + capacity); + Assert.Equal(["backfiller.giganews"], first.Select(t => t.QueueName)); + + var second = SilentConsumerMonitorTargets.Resolve( + new StaticBackFillerProviderCatalog( + [ + Provider("Giganews", 1), + Provider("Eweka", 1), + ]), + capacity); + Assert.Equal( + ["backfiller.eweka", "backfiller.giganews"], + second.Select(t => t.QueueName).OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + + /// Minimal catalog row for target-resolution tests. + /// Provider backbone name. + /// Catalog MaxSessions cap. + /// A with fixed host, port, and credentials. + private static BackFillerProviderDefinition Provider(string backbone, int maxSessions) => + new(backbone, "host", 563, true, "u", "p", MinSessions: 0, MaxSessions: maxSessions); + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMultiInstanceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMultiInstanceTests.cs new file mode 100644 index 00000000..92fdf8ad --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMultiInstanceTests.cs @@ -0,0 +1,540 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.BackFillerService.Accounts; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.Common.Messaging.RabbitMq; +using VectorNNTPD.Common.NntpDb; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// Multi-instance / multi-session isolation, shared-queue fleet semantics, prune/reintroduction, + /// and credential-projection helpers. Uses engine fakes (no live broker). + /// + public sealed class SilentConsumerMultiInstanceTests + { + /// + /// Case A: one instance opens local processing stall while a sibling instance and shared aggregate ack progress stay healthy. + /// + [Fact] + public void CaseA_local_processing_stall_opens_while_sibling_aggregate_ack_continues() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var stalled = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var healthy = new SilentConsumerMonitorEngine(() => "b.example", NullLogger.Instance, time); + + _ = stalled.ObserveInstance([Session("Giganews", 1, stalled: true, queued: 0, active: 1)], desiredRunningSessions: 1); + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow); + var local = stalled.ObserveInstance([Session("Giganews", 1, stalled: true, queued: 0, active: 1)], 1) + .Single(r => r.Key is { Scope: ObservationScope.Instance, Condition: SilentConsumerCondition.ProcessingStall }); + Assert.Equal(ObservationTransition.Opened, local.Transition); + Assert.Equal(ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 1), local.Key.QueueOrBackbone); + Assert.Contains("sibling", local.Detail, StringComparison.OrdinalIgnoreCase); + + // Sibling sees healthy local state and aggregate ACK progress — must not open ProcessingStall. + Assert.Equal( + ConditionSignal.Inactive, + healthy.ObserveInstance([Session("Giganews", 1, stalled: false, queued: 0, active: 1)], 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall) + .Signal); + Assert.Equal( + ConditionSignal.Inactive, + healthy.ObserveSharedQueue("backfiller.giganews", Success(consumers: 2, ready: 0, unacked: 1, ackRate: 5), expectedActive: true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall) + .Signal); + + // Aggregate progress must not clear the stalled instance's local observation. + Assert.Equal( + ObservationPhase.Open, + stalled.ObserveInstance([Session("Giganews", 1, stalled: true, queued: 0, active: 1)], 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall) + .Phase); + } + + /// + /// Case B: a running session with zero queued and zero active work is not classified as dispatch stall. + /// + [Fact] + public void CaseB_idle_instance_with_zero_queued_is_not_dispatch_stall() + { + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, TimeProvider.System); + var result = engine.ObserveInstance([Session("Giganews", 1, stalled: false, queued: 0, active: 0)], desiredRunningSessions: 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall); + Assert.Equal(ConditionSignal.Inactive, result.Signal); + Assert.Contains("idle", result.Detail, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Case C: queued work before active count opens dispatch stall after the open window. + /// + [Fact] + public void CaseC_dispatch_stall_before_active_count_increments() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + _ = engine.ObserveInstance([Session("Giganews", 1, stalled: false, queued: 2, active: 0)], 1); + time.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + var opened = engine.ObserveInstance([Session("Giganews", 1, stalled: false, queued: 2, active: 0)], 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall); + Assert.Equal(ObservationTransition.Opened, opened.Transition); + Assert.Contains("DispatchAcquireHold", opened.Detail, StringComparison.Ordinal); + } + + /// + /// Case D: zero consumers with ready backlog opens shared consumer loss when the queue is expected active. + /// + [Fact] + public void CaseD_shared_consumer_loss_when_all_consumers_gone_with_backlog() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + _ = engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), expectedActive: true); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + Assert.Equal( + ObservationTransition.Opened, + engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), expectedActive: true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss) + .Transition); + } + + /// + /// Case D variant: one remaining consumer with backlog is inactive fleet consumer loss. + /// + [Fact] + public void CaseD_one_remaining_consumer_is_not_fleet_consumer_loss() + { + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, TimeProvider.System); + var shared = engine.ObserveSharedQueue("backfiller.giganews", Success(1, 5, 0, 0), expectedActive: true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ConditionSignal.Inactive, shared.Signal); + Assert.Contains("not fleet-wide", shared.Detail, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Case E: local consumer loss activates with desired sessions but no running observations; healthy shared metrics stay inactive. + /// + [Fact] + public void CaseE_local_consumer_loss_independent_of_aggregate_rates() + { + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, TimeProvider.System); + var local = engine.ObserveInstance([], desiredRunningSessions: 2) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ObservationScope.Instance, local.Key.Scope); + Assert.Equal(ConditionSignal.Active, local.Signal); + + var shared = engine.ObserveSharedQueue("backfiller.giganews", Success(2, 0, 0, ackRate: 10), expectedActive: true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ConditionSignal.Inactive, shared.Signal); + } + + /// + /// Case F: two monitor engines on distinct instance ids both open the same shared consumer-loss condition independently. + /// + [Fact] + public void CaseF_two_monitors_duplicate_shared_observation_without_attributing_to_local_consumer() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var a = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var b = new SilentConsumerMonitorEngine(() => "b.example", NullLogger.Instance, time); + _ = a.ObserveSharedQueue("backfiller.giganews", Success(0, 4, 0, 0), true); + _ = b.ObserveSharedQueue("backfiller.giganews", Success(0, 4, 0, 0), true); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + var aOpen = a.ObserveSharedQueue("backfiller.giganews", Success(0, 4, 0, 0), true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + var bOpen = b.ObserveSharedQueue("backfiller.giganews", Success(0, 4, 0, 0), true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ObservationTransition.Opened, aOpen.Transition); + Assert.Equal(ObservationTransition.Opened, bOpen.Transition); + Assert.Equal(ObservationScope.SharedQueue, aOpen.Key.Scope); + Assert.NotEqual(a.InstanceId, b.InstanceId); + } + + /// + /// A healthy sibling session on the same backbone does not suppress or resolve another session's processing stall. + /// + [Fact] + public void Healthy_sibling_session_same_backbone_does_not_suppress_processing_stall() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var key1 = ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 1); + var key2 = ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 2); + var sessions = new[] + { + Session("Giganews", 1, stalled: true, queued: 0, active: 1), + Session("Giganews", 2, stalled: false, queued: 0, active: 1), + }; + + _ = engine.ObserveInstance(sessions, desiredRunningSessions: 2); + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow); + var results = engine.ObserveInstance(sessions, 2); + + var stalled = results.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key1); + var healthy = results.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key2); + + Assert.Equal(ObservationTransition.Opened, stalled.Transition); + Assert.Equal(ObservationPhase.Open, stalled.Phase); + Assert.Equal(ConditionSignal.Inactive, healthy.Signal); + Assert.Equal(ObservationPhase.Clear, healthy.Phase); + + // Same-cycle healthy sibling must not begin resolving the stalled session. + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.ResolveWindow); + var continued = engine.ObserveInstance(sessions, 2) + .Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key1); + Assert.Equal(ObservationPhase.Open, continued.Phase); + Assert.Equal(ObservationTransition.Continued, continued.Transition); + } + + /// + /// A healthy idle sibling does not clear dispatch stall on a blocked session on the same backbone. + /// + [Fact] + public void Healthy_sibling_session_same_backbone_does_not_suppress_dispatch_stall() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var key1 = ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 1); + var key2 = ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 2); + var sessions = new[] + { + Session("Giganews", 1, stalled: false, queued: 2, active: 0), + Session("Giganews", 2, stalled: false, queued: 0, active: 0), + }; + + _ = engine.ObserveInstance(sessions, 2); + time.Advance(SilentConsumerMonitorDefaults.DispatchStall.OpenWindow); + var results = engine.ObserveInstance(sessions, 2); + + Assert.Equal( + ObservationTransition.Opened, + results.Single(r => + r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == key1).Transition); + Assert.Equal( + ConditionSignal.Inactive, + results.Single(r => + r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == key2).Signal); + + time.Advance(SilentConsumerMonitorDefaults.DispatchStall.ResolveWindow); + Assert.Equal( + ObservationPhase.Open, + engine.ObserveInstance(sessions, 2) + .Single(r => + r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == key1) + .Phase); + } + + /// + /// When one of two stalled sessions recovers, only that session's processing stall resolves after the resolve window. + /// + [Fact] + public void Independent_recovery_resolves_only_recovered_session() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var key1 = ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 1); + var key2 = ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 2); + + var bothStalled = new[] + { + Session("Giganews", 1, stalled: true, queued: 0, active: 1), + Session("Giganews", 2, stalled: true, queued: 0, active: 1), + }; + _ = engine.ObserveInstance(bothStalled, 2); + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow); + var opened = engine.ObserveInstance(bothStalled, 2); + Assert.Equal( + ObservationTransition.Opened, + opened.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key1).Transition); + Assert.Equal( + ObservationTransition.Opened, + opened.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key2).Transition); + + // Session 1 recovers; session 2 remains stalled. + var mixed = new[] + { + Session("Giganews", 1, stalled: false, queued: 0, active: 1), + Session("Giganews", 2, stalled: true, queued: 0, active: 1), + }; + _ = engine.ObserveInstance(mixed, 2); + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.ResolveWindow); + var after = engine.ObserveInstance(mixed, 2); + + var recovered = after.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key1); + var stillOpen = after.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key2); + + Assert.Equal(ObservationTransition.Resolved, recovered.Transition); + Assert.Equal(ObservationPhase.Clear, recovered.Phase); + Assert.Equal(ObservationPhase.Open, stillOpen.Phase); + Assert.Equal(ObservationTransition.Continued, stillOpen.Transition); + } + + /// + /// Removing a session from observations inactivates its stall and resolves it after the window while the remaining session stays open. + /// + [Fact] + public void Session_removal_prunes_only_removed_session_identity() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var key1 = ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 1); + var key2 = ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 2); + + var both = new[] + { + Session("Giganews", 1, stalled: true, queued: 0, active: 1), + Session("Giganews", 2, stalled: true, queued: 0, active: 1), + }; + _ = engine.ObserveInstance(both, 2); + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow); + _ = engine.ObserveInstance(both, 2); + + // Session 1 disappears from the desired set; session 2 remains stalled. + var remaining = new[] { Session("Giganews", 2, stalled: true, queued: 0, active: 1) }; + var afterRemoval = engine.ObserveInstance(remaining, 1); + var removed = afterRemoval.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key1); + var kept = afterRemoval.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key2); + + Assert.Equal(ConditionSignal.Inactive, removed.Signal); + Assert.Contains("session removed", removed.Detail, StringComparison.OrdinalIgnoreCase); + Assert.Equal(ObservationPhase.Open, kept.Phase); + Assert.Equal(ConditionSignal.Active, kept.Signal); + + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.ResolveWindow); + var afterResolve = engine.ObserveInstance(remaining, 1); + Assert.Equal( + ObservationTransition.Resolved, + afterResolve.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key1).Transition); + Assert.Equal( + ObservationPhase.Open, + afterResolve.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key2).Phase); + } + + /// + /// After , re-observing a queue gets fresh state (None then Opened, not Continued). + /// + [Fact] + public void Provider_reintroduction_recreates_shared_observation_state() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + + _ = engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), expectedActive: true); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + Assert.Equal( + ObservationTransition.Opened, + engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss) + .Transition); + + var pruned = engine.PruneQueues([]); + Assert.Contains( + pruned, + r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss + && r.Transition == ObservationTransition.Resolved + && r.Detail is not null + && r.Detail.Contains("removed from catalog", StringComparison.OrdinalIgnoreCase)); + + // Reintroduce: history/machines must be fresh — first Active tick is None, then Opened (not Continued). + Assert.Equal( + ObservationTransition.None, + engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss) + .Transition); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + var reopened = engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ObservationTransition.Opened, reopened.Transition); + Assert.Equal(ObservationPhase.Open, reopened.Phase); + Assert.Equal(ObservationScope.SharedQueue, reopened.Key.Scope); + } + + /// + /// A healthy session on one backbone does not suppress processing stall on another backbone on the same monitor instance. + /// + [Fact] + public void Healthy_backbone_does_not_clear_stalled_backbone_on_same_instance() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var sessions = new[] + { + Session("Giganews", 1, stalled: true, queued: 0, active: 1), + Session("Eweka", 1, stalled: false, queued: 0, active: 1), + }; + _ = engine.ObserveInstance(sessions, 2); + time.Advance(SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow); + var results = engine.ObserveInstance(sessions, 2); + Assert.Equal( + ObservationTransition.Opened, + results.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == ArticleWorkConsumerSession.ComposeSessionKey("Giganews", 1)).Transition); + Assert.Equal( + ConditionSignal.Inactive, + results.Single(r => + r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == ArticleWorkConsumerSession.ComposeSessionKey("Eweka", 1)).Signal); + } + + /// + /// Shared consumer loss is inactive when expectedActive is false even with zero consumers and backlog. + /// + [Fact] + public void No_expected_capacity_suppresses_shared_consumer_loss() + { + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, TimeProvider.System); + Assert.Equal( + ConditionSignal.Inactive, + engine.ObserveSharedQueue("backfiller.giganews", Success(0, 9, 0, 0), expectedActive: false) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss) + .Signal); + } + + /// + /// Pruning all queues resolves open shared consumer loss with catalog-removal detail. + /// + [Fact] + public void Provider_removal_drives_shared_observation_toward_resolve() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T20:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + _ = engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true); + time.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + Assert.Equal( + ObservationTransition.Opened, + engine.ObserveSharedQueue("backfiller.giganews", Success(0, 5, 0, 0), true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss) + .Transition); + + var pruned = engine.PruneQueues([]); + Assert.Contains( + pruned, + r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss + && r.Transition == ObservationTransition.Resolved + && r.Detail is not null + && r.Detail.Contains("removed from catalog", StringComparison.OrdinalIgnoreCase)); + } + + /// + /// mutates live without changing management URL. + /// + [Fact] + public void Shared_config_projects_credentials_onto_live_rabbit_options_used_by_management_client() + { + var rabbit = new RabbitMqOptions + { + Username = null, + Password = null, + Management = new RabbitMqManagementOptions { BaseUrl = "http://198.18.0.3:15672" }, + }; + var shared = new NntpSharedConfiguration( + MaxArticleBytes: 1, + SiteName: "site", + PrometheusUrl: "http://p", + AcmeDirectoryUrl: "https://acme", + AcmeRenewalThresholdDays: 30, + CloudFlareZoneId: "zone", + DnsSuffix: "example.net", + AcmeAccount: "a@example.net", + AcmeCertificatePassword: "x", + CloudFlareApiKey: "k", + RabbitMqPassword: "secret-pass", + RabbitMqUsername: "nntparticles"); + shared.CopyRabbitMqCredentialsTo(rabbit); + Assert.Equal("nntparticles", rabbit.Username); + Assert.Equal("secret-pass", rabbit.Password); + Assert.Equal("http://198.18.0.3:15672", rabbit.Management.BaseUrl); + } + + /// + /// Resolved monitor targets use the same queue naming convention as provider entity composition. + /// + [Fact] + public void Catalog_targets_match_compose_provider_entity_queue_names() + { + var catalog = new ProviderConfigurationCatalog(); + catalog.Publish( + [ + new BackFillerProviderDefinition("Giganews", "h", 563, true, "u", "p", 0, 2), + ]); + var capacity = new BackboneUsableCapacityState(); + capacity.PublishSnapshot(new Dictionary(StringComparer.OrdinalIgnoreCase) { ["Giganews"] = 2 }); + var target = Assert.Single(SilentConsumerMonitorTargets.Resolve(catalog, capacity)); + Assert.Equal("backfiller.giganews", target.QueueName); + Assert.True(target.ExpectedActive); + } + + /// Synthetic session observation for multi-instance engine scenarios. + /// Provider backbone. + /// Connection index within the backbone. + /// Active processing stall flag. + /// Queued-before-active count. + /// Active in-flight count. + /// Running observation with open channel. + private static SilentConsumerSessionObservation Session( + string backbone, + int connectionNumber, + bool stalled, + int queued, + int active) => + new( + backbone, + connectionNumber, + ArticleWorkConsumerState.Running, + queued, + active, + HasOpenChannel: true, + IsActiveProcessingStalled: stalled); + + /// Successful shared-queue poll with synthetic snapshot for fleet semantics tests. + /// Management consumer gauge. + /// Messages ready. + /// Unacknowledged messages. + /// Ack rate from management API. + /// poll result. + private static ManagementPollResult Success(long consumers, long ready, long unacked, double ackRate) => + new() + { + Kind = ManagementPollKind.Success, + Reason = "ok", + Snapshot = new ManagementQueueSnapshot + { + QueueName = "q", + VirtualHost = "/", + SampledAtUtc = DateTimeOffset.Parse("2026-10-25T20:00:00Z"), + Consumers = consumers, + MessagesReady = ready, + MessagesUnacknowledged = unacked, + AckRate = ackRate, + DeliverGetRate = 0, + }, + }; + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs index 1b7ce2e0..2c994541 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs @@ -7,6 +7,7 @@ using VectorNNTPD.BackFillerService.Accounts; using VectorNNTPD.BackFillerService.Acme; using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; using VectorNNTPD.BackFillerService.Configuration; using VectorNNTPD.Common.Core; using VectorNNTPD.BackFillerService.Hosting; @@ -35,7 +36,11 @@ namespace VectorNNTPD.BackFillerService.Tests.Hosting /// public sealed class BackFillerHostCompositionTests { - /// Hosted service order, application-service triple (DNS, ACME, listener), and key singleton aliases match the current composition contract. + /// + /// Hosted service order (eleven BackFiller-hosted services ending with + /// after ), + /// application-service triple (DNS, ACME, listener), and key singleton aliases match the current composition contract. + /// [Fact] public void AddBackFillerHosting_registers_rabbitmq_then_article_work_consumers() { @@ -45,7 +50,7 @@ public void AddBackFillerHosting_registers_rabbitmq_then_article_work_consumers( var hosted = host.Services.GetServices() .Where(static service => service.GetType().Assembly == typeof(BackFillerServiceCollectionExtensions).Assembly) .ToArray(); - Assert.Equal(10, hosted.Length); + Assert.Equal(11, hosted.Length); Assert.IsType(hosted[0]); Assert.IsType(hosted[1]); Assert.IsType(hosted[2]); @@ -59,6 +64,7 @@ public void AddBackFillerHosting_registers_rabbitmq_then_article_work_consumers( Assert.Same(host.Services.GetRequiredService(), hosted[7]); Assert.Same(host.Services.GetRequiredService(), hosted[8]); Assert.Same(host.Services.GetRequiredService(), hosted[9]); + Assert.IsType(hosted[10]); Assert.Same( host.Services.GetRequiredService(), host.Services.GetRequiredService()); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/VectorNNTPD.BackFillerService.Tests.csproj b/tests/VectorNNTPD.BackFillerService.Tests/VectorNNTPD.BackFillerService.Tests.csproj index 78db28eb..3b5bce24 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/VectorNNTPD.BackFillerService.Tests.csproj +++ b/tests/VectorNNTPD.BackFillerService.Tests/VectorNNTPD.BackFillerService.Tests.csproj @@ -19,6 +19,7 @@ + all From d75cd761158f8a7a7519d2bbc13f6ffaf7e52b98 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Mon, 26 Oct 2026 07:20:04 +0200 Subject: [PATCH 3/3] Fix CodeQL findings in SilentConsumerMonitor - Correct HttpResponseMessage ownership in test handler - Simplify healthy-session and consumer-loss counts using LINQ - Preserve existing locking and monitoring semantics --- .../ArticleWork/ArticleWorkConsumerService.cs | 10 ++-------- .../SilentConsumerEvaluators.cs | 10 ++-------- .../ManagementQueueClientTests.cs | 17 +++++++++++------ 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs index c947fb1a..49a44327 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerService.cs @@ -604,16 +604,10 @@ private static bool IsHealthyRunningSession(ArticleWorkConsumerSession session) private void ObserveInfrastructureUnavailable(int desiredSlotCount) { var streak = Interlocked.Increment(ref _consecutiveInfrastructureUnavailable); - var healthy = 0; + int healthy; lock (_gate) { - foreach (var session in _sessions.Values) - { - if (IsHealthyRunningSession(session)) - { - healthy++; - } - } + healthy = _sessions.Values.Count(IsHealthyRunningSession); } ArticleWorkLogMessages.ConsumerInfrastructureUnavailable( diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluators.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluators.cs index 459e298a..c048179c 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluators.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerEvaluators.cs @@ -84,14 +84,8 @@ public static ConditionSignal EvaluateInstanceConsumerLoss( return ConditionSignal.Inactive; } - var running = 0; - foreach (var session in sessions) - { - if (session.State == ArticleWorkConsumerState.Running && session.HasOpenChannel) - { - running++; - } - } + var running = sessions.Count( + static session => session.State == ArticleWorkConsumerState.Running && session.HasOpenChannel); if (running > 0) { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ManagementQueueClientTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ManagementQueueClientTests.cs index b08a15ef..48325049 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ManagementQueueClientTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/ManagementQueueClientTests.cs @@ -354,6 +354,7 @@ protected override async Task SendAsync( await Task.Delay(Delay, cancellationToken); } + // Returned to HttpClient; ManagementQueueClient.GetQueueAsync disposes with using. return new HttpResponseMessage(_status) { Content = new StringContent(_body, Encoding.UTF8, "application/json"), @@ -372,18 +373,22 @@ private sealed class CapturingHandler(HttpStatusCode status, string body) : Http public List AuthorizationParameters { get; } = []; /// - protected override Task SendAsync( + protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { Assert.NotNull(request.Headers.Authorization); Assert.Equal("Basic", request.Headers.Authorization.Scheme); AuthorizationParameters.Add(request.Headers.Authorization.Parameter ?? string.Empty); - return Task.FromResult( - new HttpResponseMessage(status) - { - Content = new StringContent(body, Encoding.UTF8, "application/json"), - }); + + // Prefer returning the message from an async method (not Task.FromResult(new ...)) + // so ownership transfers to HttpClient the same way as StubHandler / quorum fakes. + // ManagementQueueClient.GetQueueAsync disposes the response with using. + await Task.CompletedTask.ConfigureAwait(false); + return new HttpResponseMessage(status) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }; } } }