diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabHarness.cs new file mode 100644 index 00000000..6399a14c --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabHarness.cs @@ -0,0 +1,322 @@ +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using RabbitMQ.Client; +using VectorNNTPD.BackFillerService.Accounts; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.RabbitMq; +using VectorNNTPD.BackFillerService.Tests.Stress; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// Lab-only helpers: live RabbitMQ consumer sessions, StressRunId publish/cleanup, Management snapshots. + /// Not used by ordinary CI runs (callers must gate on ). + /// + internal static class SilentConsumerLabHarness + { + /// Giganews Article Work queue used for lab scenarios. + internal const string GiganewsQueue = "backfiller.giganews"; + + /// Backbone label matching . + internal const string Backbone = "Giganews"; + + /// Creates a named HttpClient factory for . + internal static ServiceProvider CreateHttpServices() + { + var services = new ServiceCollection(); + services.AddHttpClient(ManagementQueueClient.HttpClientName); + return services.BuildServiceProvider(); + } + + /// + /// Starts against the lab broker with a unique client-provided name. + /// + /// Live options (hosts/credentials). + /// AMQP client-provided name (visible in Management). + /// Cancels start. + /// Started service; caller disposes. + internal static async Task StartLiveRabbitAsync( + RabbitMqOptions rabbitOptions, + string connectionName, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(rabbitOptions); + ArgumentException.ThrowIfNullOrWhiteSpace(connectionName); + var factory = new RabbitMqClientConnectionFactory(NullLogger.Instance); + var service = new RabbitMqService( + factory, + Options.Create(rabbitOptions), + new DelegateRabbitMqConnectionNameProvider(() => connectionName), + NullLogger.Instance); + await service.StartAsync(cancellationToken).ConfigureAwait(false); + return service; + } + + /// + /// Builds a MaxSessions=1 Giganews consumer with injectable handler and shared fake clock. + /// + /// Live or fake RabbitMQ service. + /// Article handler (typically ). + /// Clock for stall predicates and reconcile. + /// + /// When false (default for lab), the background reconcile loop parks until cancel so FakeTimeProvider + /// advances do not auto-retire sessions; callers invoke explicitly. + /// + /// Unstarted consumer service. + internal static ArticleWorkConsumerService CreateConsumer( + IRabbitMqService connections, + IArticleWorkHandler handler, + TimeProvider time, + bool autoReconcile = false) + { + ArgumentNullException.ThrowIfNull(connections); + ArgumentNullException.ThrowIfNull(handler); + ArgumentNullException.ThrowIfNull(time); + var catalog = new StaticBackFillerProviderCatalog( + [ + new BackFillerProviderDefinition( + Backbone, + "127.0.0.1", + 119, + false, + "nntp-user", + "p", + MinSessions: 0, + MaxSessions: 1), + ]); + var capacity = new BackboneUsableCapacityState(); + capacity.PublishSnapshot( + new Dictionary(StringComparer.OrdinalIgnoreCase) { [Backbone] = 1 }); + Func? loopBody = autoReconcile + ? null + : static async ct => + { + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Parked until host stop. + } + }; + return new ArticleWorkConsumerService( + connections, + RabbitMqServiceTests.CreateFastRuntime(), + handler, + new RecordingArticleWorkResponsePublisher { CompletesSuccessPublication = true }, + NullLogger.Instance, + catalog, + capacity, + time, + loopBody); + } + + /// + /// Observes Inactive once (starts resolve clock), advances , observes again. + /// + internal static ObservationResult ResolveInstanceCondition( + SilentConsumerMonitorEngine engine, + IReadOnlyList sessions, + int desired, + FakeTimeProvider clock, + SilentConsumerCondition condition, + TimeSpan resolveWindow) + { + ArgumentNullException.ThrowIfNull(engine); + ArgumentNullException.ThrowIfNull(sessions); + ArgumentNullException.ThrowIfNull(clock); + ObservationResult First() + { + var projected = sessions + .Select( + s => SilentConsumerSessionObservation.FromSession( + s, + ArticleWorkConsumerService.ProcessingStallThreshold)) + .ToArray(); + return engine.ObserveInstance(projected, desired).Single(r => r.Key.Condition == condition); + } + + var first = First(); + Assert.True( + first.Transition is ObservationTransition.Continued or ObservationTransition.None, + $"Expected resolve-clock start, got {first.Transition} signal={first.Signal}"); + clock.Advance(resolveWindow); + return First(); + } + + /// Waits until the consumer has one Running session with an open channel. + /// Started consumer. + /// Safety bound. + /// The running session. + internal static async Task WaitForRunningSessionAsync( + ArticleWorkConsumerService consumer, + TimeSpan timeout) + { + ArgumentNullException.ThrowIfNull(consumer); + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + var session = consumer.Sessions.FirstOrDefault( + static s => s.State == ArticleWorkConsumerState.Running && s.Channel is { IsOpen: true }); + if (session is not null) + { + return session; + } + + await Task.Delay(100).ConfigureAwait(false); + } + + throw new TimeoutException("Timed out waiting for a Running Article Work consumer session."); + } + + /// + /// Publishes one canonical Article Work JSON body with a unique request id and StressRunId header. + /// + /// Broker credentials/host. + /// Owned run id for cleanup. + /// Cancels publish. + /// The correlation id used on the message. + internal static async Task PublishOwnedWorkAsync( + StressHarnessOptions stress, + string stressRunId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stress); + ArgumentException.ThrowIfNullOrWhiteSpace(stressRunId); + var factory = new ConnectionFactory + { + HostName = stress.RabbitMqHost, + Port = stress.RabbitMqPort, + VirtualHost = stress.RabbitMqVirtualHost, + UserName = stress.RabbitMqUsername, + Password = stress.RabbitMqPassword, + RequestedHeartbeat = TimeSpan.FromSeconds(30), + AutomaticRecoveryEnabled = false, + ClientProvidedName = "VectorNNTPD.BackFillerService.Tests:silent-consumer-lab-pub", + }; + await using var connection = await factory.CreateConnectionAsync(cancellationToken).ConfigureAwait(false); + await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + var requestId = Guid.NewGuid(); + var correlationId = Guid.NewGuid().ToString("D"); + var json = + $"{{\"version\":1,\"requestId\":\"{requestId:D}\",\"messageId\":\"<12345@example.invalid>\",\"backbone\":\"Giganews\"}}"; + var props = new BasicProperties + { + ContentType = ArticleWorkRequestParser.JsonContentType, + CorrelationId = correlationId, + ReplyTo = "silent-consumer-lab.unused-reply", + DeliveryMode = DeliveryModes.Persistent, + Headers = new Dictionary + { + ["RequestId"] = requestId.ToString("D"), + [StressOwnedWorkMessageCleanup.StressRunIdHeader] = stressRunId, + }, + }; + await channel.BasicPublishAsync( + exchange: GiganewsQueue, + routingKey: GiganewsQueue, + mandatory: false, + basicProperties: props, + body: Encoding.UTF8.GetBytes(json), + cancellationToken: cancellationToken) + .ConfigureAwait(false); + return correlationId; + } + + /// Best-effort StressRunId-scoped cleanup (never purges foreign messages). + /// Broker options. + /// Owned run id. + /// Cancels cleanup. + /// Owned messages discarded. + internal static Task CleanupOwnedAsync( + StressHarnessOptions stress, + string stressRunId, + CancellationToken cancellationToken) => + StressOwnedWorkMessageCleanup.RemoveOwnedAsync( + stress, + Backbone, + stressRunId, + cancellationToken); + + /// Reads Management gauges for (no secrets logged). + /// Broker host/credentials. + /// Cancels HTTP. + /// Ready, unacked, and consumer counts. + internal static async Task<(long Ready, long Unacked, long Consumers)> SampleQueueAsync( + StressHarnessOptions stress, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stress); + using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; + var token = Convert.ToBase64String( + Encoding.ASCII.GetBytes($"{stress.RabbitMqUsername}:{stress.RabbitMqPassword}")); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", token); + var url = + $"http://{stress.RabbitMqHost}:15672/api/queues/%2F/{Uri.EscapeDataString(GiganewsQueue)}"; + using var response = await client.GetAsync(url, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken) + .ConfigureAwait(false); + var root = doc.RootElement; + var ready = root.TryGetProperty("messages_ready", out var r) ? r.GetInt64() : -1; + var unacked = root.TryGetProperty("messages_unacknowledged", out var u) ? u.GetInt64() : -1; + var consumers = root.TryGetProperty("consumers", out var c) ? c.GetInt64() : -1; + return (ready, unacked, consumers); + } + + /// + /// Projects sessions and advances the monitor clock through the open window for one condition evaluate pass. + /// + /// Monitor engine under test. + /// Live sessions. + /// Desired running session count. + /// Shared fake clock (must match session clocks for processing stall). + /// Open window to advance after the priming observe. + /// Results from the second observe (after open window). + internal static IReadOnlyList ObserveInstanceAfterOpenWindow( + SilentConsumerMonitorEngine engine, + IReadOnlyList sessions, + int desired, + FakeTimeProvider clock, + TimeSpan openWindow) + { + ArgumentNullException.ThrowIfNull(engine); + ArgumentNullException.ThrowIfNull(sessions); + ArgumentNullException.ThrowIfNull(clock); + var projected = sessions + .Select( + s => SilentConsumerSessionObservation.FromSession( + s, + ArticleWorkConsumerService.ProcessingStallThreshold)) + .ToArray(); + _ = engine.ObserveInstance(projected, desired); + clock.Advance(openWindow); + projected = sessions + .Select( + s => SilentConsumerSessionObservation.FromSession( + s, + ArticleWorkConsumerService.ProcessingStallThreshold)) + .ToArray(); + return engine.ObserveInstance(projected, desired); + } + + /// Builds runtime options used only for monitor HostedService FQDN stamping in lab tests. + internal static BackFillerRuntimeOptions CreateMonitorRuntime() => + RabbitMqServiceTests.CreateFastRuntime(); + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabOptions.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabOptions.cs new file mode 100644 index 00000000..fd00505a --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabOptions.cs @@ -0,0 +1,87 @@ +using VectorNNTPD.BackFillerService.Tests.Stress; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// Opt-in controls for lab SilentConsumerMonitor runtime validation against the shared lab broker. + /// Disabled unless is exactly 1. + /// + internal static class SilentConsumerLabOptions + { + /// Master gate. Must be 1; any other value (including unset) keeps tests no-ops. + internal const string EnableEnvironmentVariable = "VECTORNNTPD_SILENT_CONSUMER_LAB"; + + /// Optional RabbitMQ host override (defaults to stress/lab host 198.18.0.3). + internal const string RabbitHostEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_RABBIT_HOST"; + + /// Returns whether lab runtime scenarios should execute. + internal static bool IsEnabled() => + string.Equals( + Environment.GetEnvironmentVariable(EnableEnvironmentVariable), + "1", + StringComparison.Ordinal); + + /// + /// Builds stress-compatible broker options from lab credentials without enabling the E2E stress master flag. + /// + /// Options suitable for and owned-message cleanup. + /// Thrown when RabbitMQ credentials are missing. + internal static StressHarnessOptions CreateStressBrokerOptions() + { + var username = Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__USERNAME"); + var password = Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__PASSWORD"); + if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) + { + throw new InvalidOperationException( + "VECTOR__RABBITMQ__USERNAME and VECTOR__RABBITMQ__PASSWORD are required for SilentConsumer lab runs."); + } + + var host = Environment.GetEnvironmentVariable(RabbitHostEnvironmentVariable); + if (string.IsNullOrWhiteSpace(host)) + { + host = "198.18.0.3"; + } + + return new StressHarnessOptions + { + RabbitMqHost = host.Trim(), + RabbitMqPort = 5672, + RabbitMqVirtualHost = "/", + RabbitMqUsername = username, + RabbitMqPassword = password, + WorkReplyTimeout = TimeSpan.FromMinutes(5), + ReplyQueueDisposeDrainTimeout = TimeSpan.FromSeconds(30), + }; + } + + /// + /// Live for / Management client (secrets never logged). + /// + /// Broker identity and credentials. + /// Mutable options instance (BaseUrl can be cleared for Management-outage inject). + internal static RabbitMqOptions CreateRabbitMqOptions(StressHarnessOptions stress) + { + ArgumentNullException.ThrowIfNull(stress); + return new RabbitMqOptions + { + Hosts = [stress.RabbitMqHost], + Port = stress.RabbitMqPort, + VirtualHost = stress.RabbitMqVirtualHost, + Username = stress.RabbitMqUsername, + Password = stress.RabbitMqPassword, + EnableSsl = false, + MinConnections = 1, + MaxConnections = 4, + ChannelPoolSize = 64, + PoolReconnectBaseDelayMs = 50, + PoolReconnectMaxDelayMs = 250, + Management = new RabbitMqManagementOptions + { + BaseUrl = $"http://{stress.RabbitMqHost}:15672", + RequestTimeoutSeconds = 5, + }, + }; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabRuntimeTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabRuntimeTests.cs new file mode 100644 index 00000000..5ac6a8b9 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabRuntimeTests.cs @@ -0,0 +1,464 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor +{ + /// + /// Opt-in lab runtime validation for SilentConsumerMonitor against the shared lab RabbitMQ broker. + /// Disabled unless VECTORNNTPD_SILENT_CONSUMER_LAB=1. Does not start OS BackFiller processes; + /// uses production session/consumer/monitor types with / + /// seams (no production code changes). + /// + public sealed class SilentConsumerLabRuntimeTests + { + /// + /// Scenario C: DispatchAcquireHold opens instance DispatchStall; release resolves; Event 5328 path stays idle. + /// + [Fact] + public async Task ScenarioC_dispatch_stall_detects_and_resolves_on_lab_broker() + { + if (!SilentConsumerLabOptions.IsEnabled()) + { + return; + } + + var stress = SilentConsumerLabOptions.CreateStressBrokerOptions(); + var runId = $"scm-lab-c-{Guid.NewGuid():N}"; + var rabbitOptions = SilentConsumerLabOptions.CreateRabbitMqOptions(stress); + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:00:00Z")); + var logger = new CollectingLogger(); + var engine = new SilentConsumerMonitorEngine(() => "lab-bf-c.example", logger, clock); + await using var connections = await SilentConsumerLabHarness.StartLiveRabbitAsync( + rabbitOptions, + "VectorNNTPD.BackFillerService:silent-consumer-lab-c", + CancellationToken.None); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "lab-dispatch", + Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn, + ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort, + Gate = NewSource(), + }; + handler.Gate!.TrySetResult(); + await using var consumer = SilentConsumerLabHarness.CreateConsumer(connections, handler, clock); + await consumer.StartAsync(CancellationToken.None); + var session = await SilentConsumerLabHarness.WaitForRunningSessionAsync( + consumer, + TimeSpan.FromSeconds(60)); + + var before = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + Console.WriteLine( + $"ScenarioC start utc={DateTimeOffset.UtcNow:o} ready={before.Ready} unacked={before.Unacked} consumers={before.Consumers}"); + + session.DispatchAcquired = NewSource(); + session.DispatchAcquireHold = NewSource(); + await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None); + await session.DispatchAcquired.Task.WaitAsync(TimeSpan.FromSeconds(30)); + Assert.Equal(1, session.QueuedCount); + Assert.Equal(0, session.ActiveCount); + Assert.False(session.IsActiveProcessingStalled(ArticleWorkConsumerService.ProcessingStallThreshold)); + + var mid = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + Console.WriteLine( + $"ScenarioC held utc={DateTimeOffset.UtcNow:o} ready={mid.Ready} unacked={mid.Unacked} consumers={mid.Consumers} queued={session.QueuedCount} active={session.ActiveCount}"); + + var opened = SilentConsumerLabHarness + .ObserveInstanceAfterOpenWindow( + engine, + [session], + desired: 1, + clock, + SilentConsumerMonitorDefaults.DispatchStall.OpenWindow) + .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall); + Assert.Equal(ObservationTransition.Opened, opened.Transition); + Assert.Equal(ObservationScope.Instance, opened.Key.Scope); + Assert.Contains("DispatchAcquireHold", opened.Detail, StringComparison.Ordinal); + Assert.Contains( + logger.Entries, + e => e.EventId.Id == 9110 && e.Message.Contains("DispatchStall", StringComparison.Ordinal)); + + session.DispatchAcquireHold.TrySetResult(); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => session.QueuedCount == 0 && session.ActiveCount == 0, + TimeSpan.FromSeconds(30)); + + var resolved = SilentConsumerLabHarness.ResolveInstanceCondition( + engine, + [session], + desired: 1, + clock, + SilentConsumerCondition.DispatchStall, + SilentConsumerMonitorDefaults.DispatchStall.ResolveWindow); + Assert.Equal(ObservationTransition.Resolved, resolved.Transition); + Assert.Contains(logger.Entries, e => e.EventId.Id == 9112); + + var discarded = await SilentConsumerLabHarness.CleanupOwnedAsync( + stress, + runId, + CancellationToken.None); + var after = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + Console.WriteLine( + $"ScenarioC end utc={DateTimeOffset.UtcNow:o} discarded={discarded} ready={after.Ready} unacked={after.Unacked} consumers={after.Consumers}"); + await consumer.StopAsync(CancellationToken.None); + } + + /// + /// Scenario B: handler Gate opens ProcessingStall on one instance; healthy sibling stays clear; reconcile retires. + /// + [Fact] + public async Task ScenarioB_processing_stall_sibling_healthy_and_reconcile_recovers() + { + if (!SilentConsumerLabOptions.IsEnabled()) + { + return; + } + + var stress = SilentConsumerLabOptions.CreateStressBrokerOptions(); + var runId = $"scm-lab-b-{Guid.NewGuid():N}"; + var rabbitOptions = SilentConsumerLabOptions.CreateRabbitMqOptions(stress); + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:10:00Z")); + var stalledLogger = new CollectingLogger(); + var healthyLogger = new CollectingLogger(); + var stalledEngine = new SilentConsumerMonitorEngine(() => "lab-bf-b-stalled.example", stalledLogger, clock); + var healthyEngine = new SilentConsumerMonitorEngine(() => "lab-bf-b-healthy.example", healthyLogger, clock); + + await using var stalledConnections = await SilentConsumerLabHarness.StartLiveRabbitAsync( + rabbitOptions, + "VectorNNTPD.BackFillerService:silent-consumer-lab-b-stalled", + CancellationToken.None); + await using var healthyConnections = await SilentConsumerLabHarness.StartLiveRabbitAsync( + rabbitOptions, + "VectorNNTPD.BackFillerService:silent-consumer-lab-b-healthy", + CancellationToken.None); + + var stalledHandler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "lab-held", + Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn, + ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort, + Started = NewSource(), + Gate = NewSource(), + }; + var healthyHandler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "lab-ok", + Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn, + ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort, + Gate = NewSource(), + }; + healthyHandler.Gate!.TrySetResult(); + + await using var stalledConsumer = SilentConsumerLabHarness.CreateConsumer( + stalledConnections, + stalledHandler, + clock); + await using var healthyConsumer = SilentConsumerLabHarness.CreateConsumer( + healthyConnections, + healthyHandler, + clock); + // Start stalled consumer first so the held delivery is routed to it (StartAsync is one-shot). + await stalledConsumer.StartAsync(CancellationToken.None); + var stalledSession = await SilentConsumerLabHarness.WaitForRunningSessionAsync( + stalledConsumer, + TimeSpan.FromSeconds(60)); + + var before = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + Console.WriteLine( + $"ScenarioB start utc={DateTimeOffset.UtcNow:o} ready={before.Ready} unacked={before.Unacked} consumers={before.Consumers}"); + + await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None); + await stalledHandler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(30)); + Assert.Equal(1, stalledSession.ActiveCount); + Assert.True(stalledSession.HasActiveProcessing); + + await healthyConsumer.StartAsync(CancellationToken.None); + var healthySession = await SilentConsumerLabHarness.WaitForRunningSessionAsync( + healthyConsumer, + TimeSpan.FromSeconds(60)); + await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None); + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => healthyHandler.HandleCount >= 1, + TimeSpan.FromSeconds(30)); + + clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold); + Assert.True(stalledSession.IsActiveProcessingStalled(ArticleWorkConsumerService.ProcessingStallThreshold)); + Assert.False(healthySession.IsActiveProcessingStalled(ArticleWorkConsumerService.ProcessingStallThreshold)); + + var mid = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + Console.WriteLine( + $"ScenarioB stalled utc={DateTimeOffset.UtcNow:o} ready={mid.Ready} unacked={mid.Unacked} consumers={mid.Consumers}"); + + var stalledOpened = SilentConsumerLabHarness + .ObserveInstanceAfterOpenWindow( + stalledEngine, + [stalledSession], + desired: 1, + clock, + SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow) + .Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall); + Assert.Equal(ObservationTransition.Opened, stalledOpened.Transition); + Assert.Contains("5328", stalledOpened.Detail, StringComparison.Ordinal); + Assert.Contains(stalledLogger.Entries, e => e.EventId.Id == 9110); + + var healthyProcessing = healthyEngine + .ObserveInstance( + [ + SilentConsumerSessionObservation.FromSession( + healthySession, + ArticleWorkConsumerService.ProcessingStallThreshold), + ], + desiredRunningSessions: 1) + .Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall); + Assert.Equal(ConditionSignal.Inactive, healthyProcessing.Signal); + Assert.DoesNotContain(healthyLogger.Entries, e => e.EventId.Id == 9110); + + // Monitor remains observational: reconcile owns recovery. + await stalledConsumer.ReconcileAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(30)); + Assert.Equal(ArticleWorkConsumerState.Stopped, stalledSession.State); + stalledHandler.Gate!.TrySetResult(); + + var discarded = await SilentConsumerLabHarness.CleanupOwnedAsync( + stress, + runId, + CancellationToken.None); + var after = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + Console.WriteLine( + $"ScenarioB end utc={DateTimeOffset.UtcNow:o} discarded={discarded} ready={after.Ready} unacked={after.Unacked} consumers={after.Consumers}"); + await healthyConsumer.StopAsync(CancellationToken.None); + await stalledConsumer.StopAsync(CancellationToken.None); + } + + /// + /// Scenario E: consumers drop with ready backlog → shared ConsumerLoss opens; restore consumers → resolves. + /// + [Fact] + public async Task ScenarioE_consumer_loss_shared_alert_and_restore() + { + if (!SilentConsumerLabOptions.IsEnabled()) + { + return; + } + + var stress = SilentConsumerLabOptions.CreateStressBrokerOptions(); + var runId = $"scm-lab-e-{Guid.NewGuid():N}"; + var rabbitOptions = SilentConsumerLabOptions.CreateRabbitMqOptions(stress); + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:20:00Z")); + var logger = new CollectingLogger(); + var engine = new SilentConsumerMonitorEngine(() => "lab-bf-e.example", logger, clock); + + // Seed ready backlog with no consumer (StartAsync is one-shot — restore uses a fresh consumer). + var before = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + Console.WriteLine( + $"ScenarioE start utc={DateTimeOffset.UtcNow:o} ready={before.Ready} unacked={before.Unacked} consumers={before.Consumers}"); + await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None); + await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None); + + var lossShape = await WaitForQueueAsync( + stress, + static s => s.Consumers == 0 && s.Ready >= 1, + TimeSpan.FromSeconds(30)); + Console.WriteLine( + $"ScenarioE loss utc={DateTimeOffset.UtcNow:o} ready={lossShape.Ready} unacked={lossShape.Unacked} consumers={lossShape.Consumers}"); + + using var http = SilentConsumerLabHarness.CreateHttpServices(); + var management = new ManagementQueueClient( + http.GetRequiredService(), + Options.Create(rabbitOptions), + clock); + var poll = await management.GetQueueAsync("/", SilentConsumerLabHarness.GiganewsQueue, CancellationToken.None); + Assert.Equal(ManagementPollKind.Success, poll.Kind); + Assert.NotNull(poll.Snapshot); + + _ = engine.ObserveSharedQueue( + SilentConsumerLabHarness.GiganewsQueue, + poll, + expectedActive: true); + clock.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow); + poll = await management.GetQueueAsync("/", SilentConsumerLabHarness.GiganewsQueue, CancellationToken.None); + var opened = engine + .ObserveSharedQueue(SilentConsumerLabHarness.GiganewsQueue, poll, expectedActive: true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ObservationTransition.Opened, opened.Transition); + Assert.Equal(ObservationScope.SharedQueue, opened.Key.Scope); + Assert.Contains(logger.Entries, e => e.EventId.Id == 9110); + + await using var connections = await SilentConsumerLabHarness.StartLiveRabbitAsync( + rabbitOptions, + "VectorNNTPD.BackFillerService:silent-consumer-lab-e", + CancellationToken.None); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "lab-e", + Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn, + ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort, + Gate = NewSource(), + }; + handler.Gate!.TrySetResult(); + await using var consumer = SilentConsumerLabHarness.CreateConsumer(connections, handler, clock); + await consumer.StartAsync(CancellationToken.None); + _ = await SilentConsumerLabHarness.WaitForRunningSessionAsync(consumer, TimeSpan.FromSeconds(60)); + var restored = await WaitForQueueAsync( + stress, + static s => s.Consumers >= 1 && s.Ready == 0 && s.Unacked == 0, + TimeSpan.FromSeconds(60)); + Console.WriteLine( + $"ScenarioE restored utc={DateTimeOffset.UtcNow:o} ready={restored.Ready} unacked={restored.Unacked} consumers={restored.Consumers}"); + + poll = await management.GetQueueAsync("/", SilentConsumerLabHarness.GiganewsQueue, CancellationToken.None); + var resolveStart = engine + .ObserveSharedQueue(SilentConsumerLabHarness.GiganewsQueue, poll, expectedActive: true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ConditionSignal.Inactive, resolveStart.Signal); + clock.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.ResolveWindow); + poll = await management.GetQueueAsync("/", SilentConsumerLabHarness.GiganewsQueue, CancellationToken.None); + var resolved = engine + .ObserveSharedQueue(SilentConsumerLabHarness.GiganewsQueue, poll, expectedActive: true) + .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss); + Assert.Equal(ObservationTransition.Resolved, resolved.Transition); + Assert.Contains(logger.Entries, e => e.EventId.Id == 9112); + + var discarded = await SilentConsumerLabHarness.CleanupOwnedAsync( + stress, + runId, + CancellationToken.None); + Console.WriteLine($"ScenarioE cleanup discarded={discarded}"); + await consumer.StopAsync(CancellationToken.None); + } + + /// + /// Scenario D: clear Management BaseUrl on the options instance (process-local) → Event 9123; no shared Management poll. + /// Does not take down the lab Management HTTP service. + /// + [Fact] + public async Task ScenarioD_management_baseurl_cleared_emits_9123_without_shared_outage() + { + if (!SilentConsumerLabOptions.IsEnabled()) + { + return; + } + + var stress = SilentConsumerLabOptions.CreateStressBrokerOptions(); + var rabbitOptions = SilentConsumerLabOptions.CreateRabbitMqOptions(stress); + var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:30:00Z")); + var engineLogger = new CollectingLogger(); + var hostedLogger = new CollectingLogger(); + var engine = new SilentConsumerMonitorEngine(() => "lab-bf-d.example", engineLogger, clock); + + await using var connections = await SilentConsumerLabHarness.StartLiveRabbitAsync( + rabbitOptions, + "VectorNNTPD.BackFillerService:silent-consumer-lab-d", + CancellationToken.None); + var handler = new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ArticleNotFound, + Error = "lab-d", + Gate = NewSource(), + }; + handler.Gate!.TrySetResult(); + await using var consumer = SilentConsumerLabHarness.CreateConsumer(connections, handler, clock); + await consumer.StartAsync(CancellationToken.None); + _ = await SilentConsumerLabHarness.WaitForRunningSessionAsync(consumer, TimeSpan.FromSeconds(60)); + + using var http = SilentConsumerLabHarness.CreateHttpServices(); + var rabbitOpts = Options.Create(rabbitOptions); + var management = new ManagementQueueClient( + http.GetRequiredService(), + rabbitOpts, + clock); + var catalog = new StaticBackFillerProviderCatalog( + [ + new BackFillerProviderDefinition( + SilentConsumerLabHarness.Backbone, + "127.0.0.1", + 119, + false, + "u", + "p", + MinSessions: 0, + MaxSessions: 1), + ]); + var capacity = new BackboneUsableCapacityState(); + capacity.PublishSnapshot( + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [SilentConsumerLabHarness.Backbone] = 1, + }); + var runtime = SilentConsumerLabHarness.CreateMonitorRuntime(); + var hosted = new SilentConsumerMonitorHostedService( + engine, + management, + consumer, + capacity, + catalog, + Options.Create(new SilentConsumerMonitorOptions { Enabled = true }), + rabbitOpts, + runtime, + hostedLogger, + clock); + + // Isolated inject: clear BaseUrl on the shared options object (not broker Management). + rabbitOptions.Management!.BaseUrl = string.Empty; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(3)); + await hosted.StartAsync(cts.Token); + try + { + await ArticleWorkTestDeliveries.WaitUntilAsync( + () => hostedLogger.Entries.Any(e => e.EventId.Id == 9123), + TimeSpan.FromSeconds(5)); + } + finally + { + await hosted.StopAsync(CancellationToken.None); + } + + Assert.Contains(hostedLogger.Entries, e => e.EventId.Id == 9123); + Console.WriteLine($"ScenarioD 9123 utc={DateTimeOffset.UtcNow:o} detail={hostedLogger.Entries.First(e => e.EventId.Id == 9123).Message}"); + + // Lab Management remains reachable for operators / NNTPD. + rabbitOptions.Management.BaseUrl = $"http://{stress.RabbitMqHost}:15672"; + var sample = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + Assert.True(sample.Consumers >= 0); + await consumer.StopAsync(CancellationToken.None); + } + + /// Creates an unset TCS with async continuations. + private static TaskCompletionSource NewSource() => + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Polls Management until holds or timeout. + private static async Task<(long Ready, long Unacked, long Consumers)> WaitForQueueAsync( + Stress.StressHarnessOptions stress, + Func<(long Ready, long Unacked, long Consumers), bool> predicate, + TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + (long Ready, long Unacked, long Consumers) last = default; + while (DateTime.UtcNow < deadline) + { + last = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None); + if (predicate(last)) + { + return last; + } + + await Task.Delay(250); + } + + throw new TimeoutException( + $"Queue predicate not met (ready={last.Ready} unacked={last.Unacked} consumers={last.Consumers})."); + } + } +}