diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabHarness.cs
new file mode 100644
index 00000000..6399a14c
--- /dev/null
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabHarness.cs
@@ -0,0 +1,322 @@
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Time.Testing;
+using RabbitMQ.Client;
+using VectorNNTPD.BackFillerService.Accounts;
+using VectorNNTPD.BackFillerService.ArticleWork;
+using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor;
+using VectorNNTPD.BackFillerService.Configuration;
+using VectorNNTPD.BackFillerService.Nntp;
+using VectorNNTPD.BackFillerService.Tests.Fixtures;
+using VectorNNTPD.BackFillerService.Tests.RabbitMq;
+using VectorNNTPD.BackFillerService.Tests.Stress;
+using VectorNNTPD.BackFillerService.Tests.TestDoubles;
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
+namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor
+{
+ ///
+ /// Lab-only helpers: live RabbitMQ consumer sessions, StressRunId publish/cleanup, Management snapshots.
+ /// Not used by ordinary CI runs (callers must gate on ).
+ ///
+ internal static class SilentConsumerLabHarness
+ {
+ /// Giganews Article Work queue used for lab scenarios.
+ internal const string GiganewsQueue = "backfiller.giganews";
+
+ /// Backbone label matching .
+ internal const string Backbone = "Giganews";
+
+ /// Creates a named HttpClient factory for .
+ internal static ServiceProvider CreateHttpServices()
+ {
+ var services = new ServiceCollection();
+ services.AddHttpClient(ManagementQueueClient.HttpClientName);
+ return services.BuildServiceProvider();
+ }
+
+ ///
+ /// Starts against the lab broker with a unique client-provided name.
+ ///
+ /// Live options (hosts/credentials).
+ /// AMQP client-provided name (visible in Management).
+ /// Cancels start.
+ /// Started service; caller disposes.
+ internal static async Task StartLiveRabbitAsync(
+ RabbitMqOptions rabbitOptions,
+ string connectionName,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(rabbitOptions);
+ ArgumentException.ThrowIfNullOrWhiteSpace(connectionName);
+ var factory = new RabbitMqClientConnectionFactory(NullLogger.Instance);
+ var service = new RabbitMqService(
+ factory,
+ Options.Create(rabbitOptions),
+ new DelegateRabbitMqConnectionNameProvider(() => connectionName),
+ NullLogger.Instance);
+ await service.StartAsync(cancellationToken).ConfigureAwait(false);
+ return service;
+ }
+
+ ///
+ /// Builds a MaxSessions=1 Giganews consumer with injectable handler and shared fake clock.
+ ///
+ /// Live or fake RabbitMQ service.
+ /// Article handler (typically ).
+ /// Clock for stall predicates and reconcile.
+ ///
+ /// When false (default for lab), the background reconcile loop parks until cancel so FakeTimeProvider
+ /// advances do not auto-retire sessions; callers invoke explicitly.
+ ///
+ /// Unstarted consumer service.
+ internal static ArticleWorkConsumerService CreateConsumer(
+ IRabbitMqService connections,
+ IArticleWorkHandler handler,
+ TimeProvider time,
+ bool autoReconcile = false)
+ {
+ ArgumentNullException.ThrowIfNull(connections);
+ ArgumentNullException.ThrowIfNull(handler);
+ ArgumentNullException.ThrowIfNull(time);
+ var catalog = new StaticBackFillerProviderCatalog(
+ [
+ new BackFillerProviderDefinition(
+ Backbone,
+ "127.0.0.1",
+ 119,
+ false,
+ "nntp-user",
+ "p",
+ MinSessions: 0,
+ MaxSessions: 1),
+ ]);
+ var capacity = new BackboneUsableCapacityState();
+ capacity.PublishSnapshot(
+ new Dictionary(StringComparer.OrdinalIgnoreCase) { [Backbone] = 1 });
+ Func? loopBody = autoReconcile
+ ? null
+ : static async ct =>
+ {
+ try
+ {
+ await Task.Delay(Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ // Parked until host stop.
+ }
+ };
+ return new ArticleWorkConsumerService(
+ connections,
+ RabbitMqServiceTests.CreateFastRuntime(),
+ handler,
+ new RecordingArticleWorkResponsePublisher { CompletesSuccessPublication = true },
+ NullLogger.Instance,
+ catalog,
+ capacity,
+ time,
+ loopBody);
+ }
+
+ ///
+ /// Observes Inactive once (starts resolve clock), advances , observes again.
+ ///
+ internal static ObservationResult ResolveInstanceCondition(
+ SilentConsumerMonitorEngine engine,
+ IReadOnlyList sessions,
+ int desired,
+ FakeTimeProvider clock,
+ SilentConsumerCondition condition,
+ TimeSpan resolveWindow)
+ {
+ ArgumentNullException.ThrowIfNull(engine);
+ ArgumentNullException.ThrowIfNull(sessions);
+ ArgumentNullException.ThrowIfNull(clock);
+ ObservationResult First()
+ {
+ var projected = sessions
+ .Select(
+ s => SilentConsumerSessionObservation.FromSession(
+ s,
+ ArticleWorkConsumerService.ProcessingStallThreshold))
+ .ToArray();
+ return engine.ObserveInstance(projected, desired).Single(r => r.Key.Condition == condition);
+ }
+
+ var first = First();
+ Assert.True(
+ first.Transition is ObservationTransition.Continued or ObservationTransition.None,
+ $"Expected resolve-clock start, got {first.Transition} signal={first.Signal}");
+ clock.Advance(resolveWindow);
+ return First();
+ }
+
+ /// Waits until the consumer has one Running session with an open channel.
+ /// Started consumer.
+ /// Safety bound.
+ /// The running session.
+ internal static async Task WaitForRunningSessionAsync(
+ ArticleWorkConsumerService consumer,
+ TimeSpan timeout)
+ {
+ ArgumentNullException.ThrowIfNull(consumer);
+ var deadline = DateTime.UtcNow + timeout;
+ while (DateTime.UtcNow < deadline)
+ {
+ var session = consumer.Sessions.FirstOrDefault(
+ static s => s.State == ArticleWorkConsumerState.Running && s.Channel is { IsOpen: true });
+ if (session is not null)
+ {
+ return session;
+ }
+
+ await Task.Delay(100).ConfigureAwait(false);
+ }
+
+ throw new TimeoutException("Timed out waiting for a Running Article Work consumer session.");
+ }
+
+ ///
+ /// Publishes one canonical Article Work JSON body with a unique request id and StressRunId header.
+ ///
+ /// Broker credentials/host.
+ /// Owned run id for cleanup.
+ /// Cancels publish.
+ /// The correlation id used on the message.
+ internal static async Task PublishOwnedWorkAsync(
+ StressHarnessOptions stress,
+ string stressRunId,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(stress);
+ ArgumentException.ThrowIfNullOrWhiteSpace(stressRunId);
+ var factory = new ConnectionFactory
+ {
+ HostName = stress.RabbitMqHost,
+ Port = stress.RabbitMqPort,
+ VirtualHost = stress.RabbitMqVirtualHost,
+ UserName = stress.RabbitMqUsername,
+ Password = stress.RabbitMqPassword,
+ RequestedHeartbeat = TimeSpan.FromSeconds(30),
+ AutomaticRecoveryEnabled = false,
+ ClientProvidedName = "VectorNNTPD.BackFillerService.Tests:silent-consumer-lab-pub",
+ };
+ await using var connection = await factory.CreateConnectionAsync(cancellationToken).ConfigureAwait(false);
+ await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken)
+ .ConfigureAwait(false);
+ var requestId = Guid.NewGuid();
+ var correlationId = Guid.NewGuid().ToString("D");
+ var json =
+ $"{{\"version\":1,\"requestId\":\"{requestId:D}\",\"messageId\":\"<12345@example.invalid>\",\"backbone\":\"Giganews\"}}";
+ var props = new BasicProperties
+ {
+ ContentType = ArticleWorkRequestParser.JsonContentType,
+ CorrelationId = correlationId,
+ ReplyTo = "silent-consumer-lab.unused-reply",
+ DeliveryMode = DeliveryModes.Persistent,
+ Headers = new Dictionary
+ {
+ ["RequestId"] = requestId.ToString("D"),
+ [StressOwnedWorkMessageCleanup.StressRunIdHeader] = stressRunId,
+ },
+ };
+ await channel.BasicPublishAsync(
+ exchange: GiganewsQueue,
+ routingKey: GiganewsQueue,
+ mandatory: false,
+ basicProperties: props,
+ body: Encoding.UTF8.GetBytes(json),
+ cancellationToken: cancellationToken)
+ .ConfigureAwait(false);
+ return correlationId;
+ }
+
+ /// Best-effort StressRunId-scoped cleanup (never purges foreign messages).
+ /// Broker options.
+ /// Owned run id.
+ /// Cancels cleanup.
+ /// Owned messages discarded.
+ internal static Task CleanupOwnedAsync(
+ StressHarnessOptions stress,
+ string stressRunId,
+ CancellationToken cancellationToken) =>
+ StressOwnedWorkMessageCleanup.RemoveOwnedAsync(
+ stress,
+ Backbone,
+ stressRunId,
+ cancellationToken);
+
+ /// Reads Management gauges for (no secrets logged).
+ /// Broker host/credentials.
+ /// Cancels HTTP.
+ /// Ready, unacked, and consumer counts.
+ internal static async Task<(long Ready, long Unacked, long Consumers)> SampleQueueAsync(
+ StressHarnessOptions stress,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(stress);
+ using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(10) };
+ var token = Convert.ToBase64String(
+ Encoding.ASCII.GetBytes($"{stress.RabbitMqUsername}:{stress.RabbitMqPassword}"));
+ client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", token);
+ var url =
+ $"http://{stress.RabbitMqHost}:15672/api/queues/%2F/{Uri.EscapeDataString(GiganewsQueue)}";
+ using var response = await client.GetAsync(url, cancellationToken).ConfigureAwait(false);
+ response.EnsureSuccessStatusCode();
+ await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
+ using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken)
+ .ConfigureAwait(false);
+ var root = doc.RootElement;
+ var ready = root.TryGetProperty("messages_ready", out var r) ? r.GetInt64() : -1;
+ var unacked = root.TryGetProperty("messages_unacknowledged", out var u) ? u.GetInt64() : -1;
+ var consumers = root.TryGetProperty("consumers", out var c) ? c.GetInt64() : -1;
+ return (ready, unacked, consumers);
+ }
+
+ ///
+ /// Projects sessions and advances the monitor clock through the open window for one condition evaluate pass.
+ ///
+ /// Monitor engine under test.
+ /// Live sessions.
+ /// Desired running session count.
+ /// Shared fake clock (must match session clocks for processing stall).
+ /// Open window to advance after the priming observe.
+ /// Results from the second observe (after open window).
+ internal static IReadOnlyList ObserveInstanceAfterOpenWindow(
+ SilentConsumerMonitorEngine engine,
+ IReadOnlyList sessions,
+ int desired,
+ FakeTimeProvider clock,
+ TimeSpan openWindow)
+ {
+ ArgumentNullException.ThrowIfNull(engine);
+ ArgumentNullException.ThrowIfNull(sessions);
+ ArgumentNullException.ThrowIfNull(clock);
+ var projected = sessions
+ .Select(
+ s => SilentConsumerSessionObservation.FromSession(
+ s,
+ ArticleWorkConsumerService.ProcessingStallThreshold))
+ .ToArray();
+ _ = engine.ObserveInstance(projected, desired);
+ clock.Advance(openWindow);
+ projected = sessions
+ .Select(
+ s => SilentConsumerSessionObservation.FromSession(
+ s,
+ ArticleWorkConsumerService.ProcessingStallThreshold))
+ .ToArray();
+ return engine.ObserveInstance(projected, desired);
+ }
+
+ /// Builds runtime options used only for monitor HostedService FQDN stamping in lab tests.
+ internal static BackFillerRuntimeOptions CreateMonitorRuntime() =>
+ RabbitMqServiceTests.CreateFastRuntime();
+ }
+}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabOptions.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabOptions.cs
new file mode 100644
index 00000000..fd00505a
--- /dev/null
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabOptions.cs
@@ -0,0 +1,87 @@
+using VectorNNTPD.BackFillerService.Tests.Stress;
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
+namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor
+{
+ ///
+ /// Opt-in controls for lab SilentConsumerMonitor runtime validation against the shared lab broker.
+ /// Disabled unless is exactly 1.
+ ///
+ internal static class SilentConsumerLabOptions
+ {
+ /// Master gate. Must be 1; any other value (including unset) keeps tests no-ops.
+ internal const string EnableEnvironmentVariable = "VECTORNNTPD_SILENT_CONSUMER_LAB";
+
+ /// Optional RabbitMQ host override (defaults to stress/lab host 198.18.0.3).
+ internal const string RabbitHostEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_RABBIT_HOST";
+
+ /// Returns whether lab runtime scenarios should execute.
+ internal static bool IsEnabled() =>
+ string.Equals(
+ Environment.GetEnvironmentVariable(EnableEnvironmentVariable),
+ "1",
+ StringComparison.Ordinal);
+
+ ///
+ /// Builds stress-compatible broker options from lab credentials without enabling the E2E stress master flag.
+ ///
+ /// Options suitable for and owned-message cleanup.
+ /// Thrown when RabbitMQ credentials are missing.
+ internal static StressHarnessOptions CreateStressBrokerOptions()
+ {
+ var username = Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__USERNAME");
+ var password = Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__PASSWORD");
+ if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password))
+ {
+ throw new InvalidOperationException(
+ "VECTOR__RABBITMQ__USERNAME and VECTOR__RABBITMQ__PASSWORD are required for SilentConsumer lab runs.");
+ }
+
+ var host = Environment.GetEnvironmentVariable(RabbitHostEnvironmentVariable);
+ if (string.IsNullOrWhiteSpace(host))
+ {
+ host = "198.18.0.3";
+ }
+
+ return new StressHarnessOptions
+ {
+ RabbitMqHost = host.Trim(),
+ RabbitMqPort = 5672,
+ RabbitMqVirtualHost = "/",
+ RabbitMqUsername = username,
+ RabbitMqPassword = password,
+ WorkReplyTimeout = TimeSpan.FromMinutes(5),
+ ReplyQueueDisposeDrainTimeout = TimeSpan.FromSeconds(30),
+ };
+ }
+
+ ///
+ /// Live for / Management client (secrets never logged).
+ ///
+ /// Broker identity and credentials.
+ /// Mutable options instance (BaseUrl can be cleared for Management-outage inject).
+ internal static RabbitMqOptions CreateRabbitMqOptions(StressHarnessOptions stress)
+ {
+ ArgumentNullException.ThrowIfNull(stress);
+ return new RabbitMqOptions
+ {
+ Hosts = [stress.RabbitMqHost],
+ Port = stress.RabbitMqPort,
+ VirtualHost = stress.RabbitMqVirtualHost,
+ Username = stress.RabbitMqUsername,
+ Password = stress.RabbitMqPassword,
+ EnableSsl = false,
+ MinConnections = 1,
+ MaxConnections = 4,
+ ChannelPoolSize = 64,
+ PoolReconnectBaseDelayMs = 50,
+ PoolReconnectMaxDelayMs = 250,
+ Management = new RabbitMqManagementOptions
+ {
+ BaseUrl = $"http://{stress.RabbitMqHost}:15672",
+ RequestTimeoutSeconds = 5,
+ },
+ };
+ }
+ }
+}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabRuntimeTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabRuntimeTests.cs
new file mode 100644
index 00000000..5ac6a8b9
--- /dev/null
+++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerLabRuntimeTests.cs
@@ -0,0 +1,464 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Time.Testing;
+using VectorNNTPD.BackFillerService.ArticleWork;
+using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor;
+using VectorNNTPD.BackFillerService.Configuration;
+using VectorNNTPD.BackFillerService.Nntp;
+using VectorNNTPD.BackFillerService.Tests.Fixtures;
+using VectorNNTPD.BackFillerService.Tests.TestDoubles;
+using VectorNNTPD.Common.Messaging.RabbitMq;
+
+namespace VectorNNTPD.BackFillerService.Tests.ArticleWork.SilentConsumerMonitor
+{
+ ///
+ /// Opt-in lab runtime validation for SilentConsumerMonitor against the shared lab RabbitMQ broker.
+ /// Disabled unless VECTORNNTPD_SILENT_CONSUMER_LAB=1. Does not start OS BackFiller processes;
+ /// uses production session/consumer/monitor types with /
+ /// seams (no production code changes).
+ ///
+ public sealed class SilentConsumerLabRuntimeTests
+ {
+ ///
+ /// Scenario C: DispatchAcquireHold opens instance DispatchStall; release resolves; Event 5328 path stays idle.
+ ///
+ [Fact]
+ public async Task ScenarioC_dispatch_stall_detects_and_resolves_on_lab_broker()
+ {
+ if (!SilentConsumerLabOptions.IsEnabled())
+ {
+ return;
+ }
+
+ var stress = SilentConsumerLabOptions.CreateStressBrokerOptions();
+ var runId = $"scm-lab-c-{Guid.NewGuid():N}";
+ var rabbitOptions = SilentConsumerLabOptions.CreateRabbitMqOptions(stress);
+ var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:00:00Z"));
+ var logger = new CollectingLogger();
+ var engine = new SilentConsumerMonitorEngine(() => "lab-bf-c.example", logger, clock);
+ await using var connections = await SilentConsumerLabHarness.StartLiveRabbitAsync(
+ rabbitOptions,
+ "VectorNNTPD.BackFillerService:silent-consumer-lab-c",
+ CancellationToken.None);
+ var handler = new ControllableArticleWorkHandler
+ {
+ Outcome = ArticleWorkOutcome.ArticleNotFound,
+ Error = "lab-dispatch",
+ Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn,
+ ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort,
+ Gate = NewSource(),
+ };
+ handler.Gate!.TrySetResult();
+ await using var consumer = SilentConsumerLabHarness.CreateConsumer(connections, handler, clock);
+ await consumer.StartAsync(CancellationToken.None);
+ var session = await SilentConsumerLabHarness.WaitForRunningSessionAsync(
+ consumer,
+ TimeSpan.FromSeconds(60));
+
+ var before = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ Console.WriteLine(
+ $"ScenarioC start utc={DateTimeOffset.UtcNow:o} ready={before.Ready} unacked={before.Unacked} consumers={before.Consumers}");
+
+ session.DispatchAcquired = NewSource();
+ session.DispatchAcquireHold = NewSource();
+ await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None);
+ await session.DispatchAcquired.Task.WaitAsync(TimeSpan.FromSeconds(30));
+ Assert.Equal(1, session.QueuedCount);
+ Assert.Equal(0, session.ActiveCount);
+ Assert.False(session.IsActiveProcessingStalled(ArticleWorkConsumerService.ProcessingStallThreshold));
+
+ var mid = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ Console.WriteLine(
+ $"ScenarioC held utc={DateTimeOffset.UtcNow:o} ready={mid.Ready} unacked={mid.Unacked} consumers={mid.Consumers} queued={session.QueuedCount} active={session.ActiveCount}");
+
+ var opened = SilentConsumerLabHarness
+ .ObserveInstanceAfterOpenWindow(
+ engine,
+ [session],
+ desired: 1,
+ clock,
+ SilentConsumerMonitorDefaults.DispatchStall.OpenWindow)
+ .Single(r => r.Key.Condition == SilentConsumerCondition.DispatchStall);
+ Assert.Equal(ObservationTransition.Opened, opened.Transition);
+ Assert.Equal(ObservationScope.Instance, opened.Key.Scope);
+ Assert.Contains("DispatchAcquireHold", opened.Detail, StringComparison.Ordinal);
+ Assert.Contains(
+ logger.Entries,
+ e => e.EventId.Id == 9110 && e.Message.Contains("DispatchStall", StringComparison.Ordinal));
+
+ session.DispatchAcquireHold.TrySetResult();
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => session.QueuedCount == 0 && session.ActiveCount == 0,
+ TimeSpan.FromSeconds(30));
+
+ var resolved = SilentConsumerLabHarness.ResolveInstanceCondition(
+ engine,
+ [session],
+ desired: 1,
+ clock,
+ SilentConsumerCondition.DispatchStall,
+ SilentConsumerMonitorDefaults.DispatchStall.ResolveWindow);
+ Assert.Equal(ObservationTransition.Resolved, resolved.Transition);
+ Assert.Contains(logger.Entries, e => e.EventId.Id == 9112);
+
+ var discarded = await SilentConsumerLabHarness.CleanupOwnedAsync(
+ stress,
+ runId,
+ CancellationToken.None);
+ var after = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ Console.WriteLine(
+ $"ScenarioC end utc={DateTimeOffset.UtcNow:o} discarded={discarded} ready={after.Ready} unacked={after.Unacked} consumers={after.Consumers}");
+ await consumer.StopAsync(CancellationToken.None);
+ }
+
+ ///
+ /// Scenario B: handler Gate opens ProcessingStall on one instance; healthy sibling stays clear; reconcile retires.
+ ///
+ [Fact]
+ public async Task ScenarioB_processing_stall_sibling_healthy_and_reconcile_recovers()
+ {
+ if (!SilentConsumerLabOptions.IsEnabled())
+ {
+ return;
+ }
+
+ var stress = SilentConsumerLabOptions.CreateStressBrokerOptions();
+ var runId = $"scm-lab-b-{Guid.NewGuid():N}";
+ var rabbitOptions = SilentConsumerLabOptions.CreateRabbitMqOptions(stress);
+ var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:10:00Z"));
+ var stalledLogger = new CollectingLogger();
+ var healthyLogger = new CollectingLogger();
+ var stalledEngine = new SilentConsumerMonitorEngine(() => "lab-bf-b-stalled.example", stalledLogger, clock);
+ var healthyEngine = new SilentConsumerMonitorEngine(() => "lab-bf-b-healthy.example", healthyLogger, clock);
+
+ await using var stalledConnections = await SilentConsumerLabHarness.StartLiveRabbitAsync(
+ rabbitOptions,
+ "VectorNNTPD.BackFillerService:silent-consumer-lab-b-stalled",
+ CancellationToken.None);
+ await using var healthyConnections = await SilentConsumerLabHarness.StartLiveRabbitAsync(
+ rabbitOptions,
+ "VectorNNTPD.BackFillerService:silent-consumer-lab-b-healthy",
+ CancellationToken.None);
+
+ var stalledHandler = new ControllableArticleWorkHandler
+ {
+ Outcome = ArticleWorkOutcome.ArticleNotFound,
+ Error = "lab-held",
+ Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn,
+ ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort,
+ Started = NewSource(),
+ Gate = NewSource(),
+ };
+ var healthyHandler = new ControllableArticleWorkHandler
+ {
+ Outcome = ArticleWorkOutcome.ArticleNotFound,
+ Error = "lab-ok",
+ Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn,
+ ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort,
+ Gate = NewSource(),
+ };
+ healthyHandler.Gate!.TrySetResult();
+
+ await using var stalledConsumer = SilentConsumerLabHarness.CreateConsumer(
+ stalledConnections,
+ stalledHandler,
+ clock);
+ await using var healthyConsumer = SilentConsumerLabHarness.CreateConsumer(
+ healthyConnections,
+ healthyHandler,
+ clock);
+ // Start stalled consumer first so the held delivery is routed to it (StartAsync is one-shot).
+ await stalledConsumer.StartAsync(CancellationToken.None);
+ var stalledSession = await SilentConsumerLabHarness.WaitForRunningSessionAsync(
+ stalledConsumer,
+ TimeSpan.FromSeconds(60));
+
+ var before = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ Console.WriteLine(
+ $"ScenarioB start utc={DateTimeOffset.UtcNow:o} ready={before.Ready} unacked={before.Unacked} consumers={before.Consumers}");
+
+ await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None);
+ await stalledHandler.Started!.Task.WaitAsync(TimeSpan.FromSeconds(30));
+ Assert.Equal(1, stalledSession.ActiveCount);
+ Assert.True(stalledSession.HasActiveProcessing);
+
+ await healthyConsumer.StartAsync(CancellationToken.None);
+ var healthySession = await SilentConsumerLabHarness.WaitForRunningSessionAsync(
+ healthyConsumer,
+ TimeSpan.FromSeconds(60));
+ await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None);
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => healthyHandler.HandleCount >= 1,
+ TimeSpan.FromSeconds(30));
+
+ clock.Advance(ArticleWorkConsumerService.ProcessingStallThreshold);
+ Assert.True(stalledSession.IsActiveProcessingStalled(ArticleWorkConsumerService.ProcessingStallThreshold));
+ Assert.False(healthySession.IsActiveProcessingStalled(ArticleWorkConsumerService.ProcessingStallThreshold));
+
+ var mid = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ Console.WriteLine(
+ $"ScenarioB stalled utc={DateTimeOffset.UtcNow:o} ready={mid.Ready} unacked={mid.Unacked} consumers={mid.Consumers}");
+
+ var stalledOpened = SilentConsumerLabHarness
+ .ObserveInstanceAfterOpenWindow(
+ stalledEngine,
+ [stalledSession],
+ desired: 1,
+ clock,
+ SilentConsumerMonitorDefaults.ProcessingStall.OpenWindow)
+ .Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall);
+ Assert.Equal(ObservationTransition.Opened, stalledOpened.Transition);
+ Assert.Contains("5328", stalledOpened.Detail, StringComparison.Ordinal);
+ Assert.Contains(stalledLogger.Entries, e => e.EventId.Id == 9110);
+
+ var healthyProcessing = healthyEngine
+ .ObserveInstance(
+ [
+ SilentConsumerSessionObservation.FromSession(
+ healthySession,
+ ArticleWorkConsumerService.ProcessingStallThreshold),
+ ],
+ desiredRunningSessions: 1)
+ .Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall);
+ Assert.Equal(ConditionSignal.Inactive, healthyProcessing.Signal);
+ Assert.DoesNotContain(healthyLogger.Entries, e => e.EventId.Id == 9110);
+
+ // Monitor remains observational: reconcile owns recovery.
+ await stalledConsumer.ReconcileAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(30));
+ Assert.Equal(ArticleWorkConsumerState.Stopped, stalledSession.State);
+ stalledHandler.Gate!.TrySetResult();
+
+ var discarded = await SilentConsumerLabHarness.CleanupOwnedAsync(
+ stress,
+ runId,
+ CancellationToken.None);
+ var after = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ Console.WriteLine(
+ $"ScenarioB end utc={DateTimeOffset.UtcNow:o} discarded={discarded} ready={after.Ready} unacked={after.Unacked} consumers={after.Consumers}");
+ await healthyConsumer.StopAsync(CancellationToken.None);
+ await stalledConsumer.StopAsync(CancellationToken.None);
+ }
+
+ ///
+ /// Scenario E: consumers drop with ready backlog → shared ConsumerLoss opens; restore consumers → resolves.
+ ///
+ [Fact]
+ public async Task ScenarioE_consumer_loss_shared_alert_and_restore()
+ {
+ if (!SilentConsumerLabOptions.IsEnabled())
+ {
+ return;
+ }
+
+ var stress = SilentConsumerLabOptions.CreateStressBrokerOptions();
+ var runId = $"scm-lab-e-{Guid.NewGuid():N}";
+ var rabbitOptions = SilentConsumerLabOptions.CreateRabbitMqOptions(stress);
+ var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:20:00Z"));
+ var logger = new CollectingLogger();
+ var engine = new SilentConsumerMonitorEngine(() => "lab-bf-e.example", logger, clock);
+
+ // Seed ready backlog with no consumer (StartAsync is one-shot — restore uses a fresh consumer).
+ var before = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ Console.WriteLine(
+ $"ScenarioE start utc={DateTimeOffset.UtcNow:o} ready={before.Ready} unacked={before.Unacked} consumers={before.Consumers}");
+ await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None);
+ await SilentConsumerLabHarness.PublishOwnedWorkAsync(stress, runId, CancellationToken.None);
+
+ var lossShape = await WaitForQueueAsync(
+ stress,
+ static s => s.Consumers == 0 && s.Ready >= 1,
+ TimeSpan.FromSeconds(30));
+ Console.WriteLine(
+ $"ScenarioE loss utc={DateTimeOffset.UtcNow:o} ready={lossShape.Ready} unacked={lossShape.Unacked} consumers={lossShape.Consumers}");
+
+ using var http = SilentConsumerLabHarness.CreateHttpServices();
+ var management = new ManagementQueueClient(
+ http.GetRequiredService(),
+ Options.Create(rabbitOptions),
+ clock);
+ var poll = await management.GetQueueAsync("/", SilentConsumerLabHarness.GiganewsQueue, CancellationToken.None);
+ Assert.Equal(ManagementPollKind.Success, poll.Kind);
+ Assert.NotNull(poll.Snapshot);
+
+ _ = engine.ObserveSharedQueue(
+ SilentConsumerLabHarness.GiganewsQueue,
+ poll,
+ expectedActive: true);
+ clock.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.OpenWindow);
+ poll = await management.GetQueueAsync("/", SilentConsumerLabHarness.GiganewsQueue, CancellationToken.None);
+ var opened = engine
+ .ObserveSharedQueue(SilentConsumerLabHarness.GiganewsQueue, poll, expectedActive: true)
+ .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss);
+ Assert.Equal(ObservationTransition.Opened, opened.Transition);
+ Assert.Equal(ObservationScope.SharedQueue, opened.Key.Scope);
+ Assert.Contains(logger.Entries, e => e.EventId.Id == 9110);
+
+ await using var connections = await SilentConsumerLabHarness.StartLiveRabbitAsync(
+ rabbitOptions,
+ "VectorNNTPD.BackFillerService:silent-consumer-lab-e",
+ CancellationToken.None);
+ var handler = new ControllableArticleWorkHandler
+ {
+ Outcome = ArticleWorkOutcome.ArticleNotFound,
+ Error = "lab-e",
+ Fqdn = ArticleWorkTestDeliveries.CanonicalFqdn,
+ ListenerPort = ArticleWorkTestDeliveries.CanonicalListenerPort,
+ Gate = NewSource(),
+ };
+ handler.Gate!.TrySetResult();
+ await using var consumer = SilentConsumerLabHarness.CreateConsumer(connections, handler, clock);
+ await consumer.StartAsync(CancellationToken.None);
+ _ = await SilentConsumerLabHarness.WaitForRunningSessionAsync(consumer, TimeSpan.FromSeconds(60));
+ var restored = await WaitForQueueAsync(
+ stress,
+ static s => s.Consumers >= 1 && s.Ready == 0 && s.Unacked == 0,
+ TimeSpan.FromSeconds(60));
+ Console.WriteLine(
+ $"ScenarioE restored utc={DateTimeOffset.UtcNow:o} ready={restored.Ready} unacked={restored.Unacked} consumers={restored.Consumers}");
+
+ poll = await management.GetQueueAsync("/", SilentConsumerLabHarness.GiganewsQueue, CancellationToken.None);
+ var resolveStart = engine
+ .ObserveSharedQueue(SilentConsumerLabHarness.GiganewsQueue, poll, expectedActive: true)
+ .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss);
+ Assert.Equal(ConditionSignal.Inactive, resolveStart.Signal);
+ clock.Advance(SilentConsumerMonitorDefaults.ConsumerLoss.ResolveWindow);
+ poll = await management.GetQueueAsync("/", SilentConsumerLabHarness.GiganewsQueue, CancellationToken.None);
+ var resolved = engine
+ .ObserveSharedQueue(SilentConsumerLabHarness.GiganewsQueue, poll, expectedActive: true)
+ .Single(r => r.Key.Condition == SilentConsumerCondition.ConsumerLoss);
+ Assert.Equal(ObservationTransition.Resolved, resolved.Transition);
+ Assert.Contains(logger.Entries, e => e.EventId.Id == 9112);
+
+ var discarded = await SilentConsumerLabHarness.CleanupOwnedAsync(
+ stress,
+ runId,
+ CancellationToken.None);
+ Console.WriteLine($"ScenarioE cleanup discarded={discarded}");
+ await consumer.StopAsync(CancellationToken.None);
+ }
+
+ ///
+ /// Scenario D: clear Management BaseUrl on the options instance (process-local) → Event 9123; no shared Management poll.
+ /// Does not take down the lab Management HTTP service.
+ ///
+ [Fact]
+ public async Task ScenarioD_management_baseurl_cleared_emits_9123_without_shared_outage()
+ {
+ if (!SilentConsumerLabOptions.IsEnabled())
+ {
+ return;
+ }
+
+ var stress = SilentConsumerLabOptions.CreateStressBrokerOptions();
+ var rabbitOptions = SilentConsumerLabOptions.CreateRabbitMqOptions(stress);
+ var clock = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-26T15:30:00Z"));
+ var engineLogger = new CollectingLogger();
+ var hostedLogger = new CollectingLogger();
+ var engine = new SilentConsumerMonitorEngine(() => "lab-bf-d.example", engineLogger, clock);
+
+ await using var connections = await SilentConsumerLabHarness.StartLiveRabbitAsync(
+ rabbitOptions,
+ "VectorNNTPD.BackFillerService:silent-consumer-lab-d",
+ CancellationToken.None);
+ var handler = new ControllableArticleWorkHandler
+ {
+ Outcome = ArticleWorkOutcome.ArticleNotFound,
+ Error = "lab-d",
+ Gate = NewSource(),
+ };
+ handler.Gate!.TrySetResult();
+ await using var consumer = SilentConsumerLabHarness.CreateConsumer(connections, handler, clock);
+ await consumer.StartAsync(CancellationToken.None);
+ _ = await SilentConsumerLabHarness.WaitForRunningSessionAsync(consumer, TimeSpan.FromSeconds(60));
+
+ using var http = SilentConsumerLabHarness.CreateHttpServices();
+ var rabbitOpts = Options.Create(rabbitOptions);
+ var management = new ManagementQueueClient(
+ http.GetRequiredService(),
+ rabbitOpts,
+ clock);
+ var catalog = new StaticBackFillerProviderCatalog(
+ [
+ new BackFillerProviderDefinition(
+ SilentConsumerLabHarness.Backbone,
+ "127.0.0.1",
+ 119,
+ false,
+ "u",
+ "p",
+ MinSessions: 0,
+ MaxSessions: 1),
+ ]);
+ var capacity = new BackboneUsableCapacityState();
+ capacity.PublishSnapshot(
+ new Dictionary(StringComparer.OrdinalIgnoreCase)
+ {
+ [SilentConsumerLabHarness.Backbone] = 1,
+ });
+ var runtime = SilentConsumerLabHarness.CreateMonitorRuntime();
+ var hosted = new SilentConsumerMonitorHostedService(
+ engine,
+ management,
+ consumer,
+ capacity,
+ catalog,
+ Options.Create(new SilentConsumerMonitorOptions { Enabled = true }),
+ rabbitOpts,
+ runtime,
+ hostedLogger,
+ clock);
+
+ // Isolated inject: clear BaseUrl on the shared options object (not broker Management).
+ rabbitOptions.Management!.BaseUrl = string.Empty;
+ using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(3));
+ await hosted.StartAsync(cts.Token);
+ try
+ {
+ await ArticleWorkTestDeliveries.WaitUntilAsync(
+ () => hostedLogger.Entries.Any(e => e.EventId.Id == 9123),
+ TimeSpan.FromSeconds(5));
+ }
+ finally
+ {
+ await hosted.StopAsync(CancellationToken.None);
+ }
+
+ Assert.Contains(hostedLogger.Entries, e => e.EventId.Id == 9123);
+ Console.WriteLine($"ScenarioD 9123 utc={DateTimeOffset.UtcNow:o} detail={hostedLogger.Entries.First(e => e.EventId.Id == 9123).Message}");
+
+ // Lab Management remains reachable for operators / NNTPD.
+ rabbitOptions.Management.BaseUrl = $"http://{stress.RabbitMqHost}:15672";
+ var sample = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ Assert.True(sample.Consumers >= 0);
+ await consumer.StopAsync(CancellationToken.None);
+ }
+
+ /// Creates an unset TCS with async continuations.
+ private static TaskCompletionSource NewSource() =>
+ new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ /// Polls Management until holds or timeout.
+ private static async Task<(long Ready, long Unacked, long Consumers)> WaitForQueueAsync(
+ Stress.StressHarnessOptions stress,
+ Func<(long Ready, long Unacked, long Consumers), bool> predicate,
+ TimeSpan timeout)
+ {
+ var deadline = DateTime.UtcNow + timeout;
+ (long Ready, long Unacked, long Consumers) last = default;
+ while (DateTime.UtcNow < deadline)
+ {
+ last = await SilentConsumerLabHarness.SampleQueueAsync(stress, CancellationToken.None);
+ if (predicate(last))
+ {
+ return last;
+ }
+
+ await Task.Delay(250);
+ }
+
+ throw new TimeoutException(
+ $"Queue predicate not met (ready={last.Ready} unacked={last.Unacked} consumers={last.Consumers}).");
+ }
+ }
+}