From 2e6ebe696853125b2cc0684a4cd12c4027312ff9 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Mon, 26 Oct 2026 08:08:13 +0200 Subject: [PATCH] Fix orphaned E2E stress work messages - Clean up work messages by StressRunId during harness teardown - Preserve foreign messages while scanning shared queues - Bound post-test drain waits - Add tests for owned-message matching and header handling --- .../Stress/BackFillerE2EStressHarness.cs | 37 ++- .../Stress/StressOwnedWorkMessageCleanup.cs | 219 ++++++++++++++++++ .../StressOwnedWorkMessageCleanupTests.cs | 98 ++++++++ 3 files changed, 349 insertions(+), 5 deletions(-) create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs index 59d91f8a..82f62a96 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs @@ -582,21 +582,34 @@ or NntpResponseKind.NotAvailable var drainStarted = Stopwatch.GetTimestamp(); var drainDeadline = drainStarted + (long)(_options.ReplyQueueDisposeDrainTimeout.TotalSeconds * Stopwatch.Frequency); + // Wait for caller-visible outstanding first (waiters + delivery workers). while ((metrics.Outstanding > 0 || metrics.DeliveryOutstanding > 0 - || workClient.WaitingCount > 0 - || workClient.BrokerOutstandingCount > 0) + || workClient.WaitingCount > 0) && Stopwatch.GetTimestamp() < drainDeadline) { await Task.Delay(100, CancellationToken.None).ConfigureAwait(false); } + // Short grace for late ReplyTo arrivals. Do not burn the full dispose drain waiting on + // broker-outstanding work that will never complete when consumers=0 (silent / absent). + // Owned ready messages are removed after dispose via StressRunId-scoped cleanup. + var lateReplyGrace = TimeSpan.FromSeconds(5); + var lateReplyDeadline = Stopwatch.GetTimestamp() + + (long)(lateReplyGrace.TotalSeconds * Stopwatch.Frequency); + var brokerWaitDeadline = Math.Min(drainDeadline, lateReplyDeadline); + while (workClient.BrokerOutstandingCount > 0 + && Stopwatch.GetTimestamp() < brokerWaitDeadline) + { + await Task.Delay(100, CancellationToken.None).ConfigureAwait(false); + } + metrics.SetDrainMilliseconds((long)Stopwatch.GetElapsedTime(drainStarted).TotalMilliseconds); var brokerOutstandingAtDrainEnd = workClient.BrokerOutstandingCount; if (brokerOutstandingAtDrainEnd > 0) { Console.Error.WriteLine( - $"Harness drain ended with brokerOutstanding={brokerOutstandingAtDrainEnd} waiting={workClient.WaitingCount}; dispose will apply bounded ReplyTo drain."); + $"Harness drain ended with brokerOutstanding={brokerOutstandingAtDrainEnd} waiting={workClient.WaitingCount}; dispose will apply bounded ReplyTo drain; owned work-queue cleanup follows."); } if (pool is not null) @@ -623,9 +636,23 @@ await File.AppendAllTextAsync( } finally { - if (disposeReplyClientOnFailure) + try + { + if (disposeReplyClientOnFailure) + { + await workClient.DisposeAsync().ConfigureAwait(false); + } + } + finally { - await workClient.DisposeAsync().ConfigureAwait(false); + // Always remove this run's ready work messages (success and failure paths). + // Does not purge foreign traffic: only StressRunId-tagged payloads are removed. + _ = await StressOwnedWorkMessageCleanup.RemoveOwnedAsync( + _options, + backbone, + runId, + CancellationToken.None) + .ConfigureAwait(false); } } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs new file mode 100644 index 00000000..4628e8ea --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs @@ -0,0 +1,219 @@ +using System.Text; +using RabbitMQ.Client; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Removes only work-queue messages published by a specific stress run (StressRunId header). + /// Non-matching messages are parked and republished; shared/production traffic is never purged wholesale. + /// + /// + /// Foreign messages cannot be basic.reject(requeue=true)'d while scanning: on classic queues that + /// often resurfaces the same head message immediately, so owned payloads behind foreign backlog would + /// never be reached. Park+ack then republish avoids that spin. Unavoidable limitations: brief absence + /// of parked messages until republish; relative order vs messages published during cleanup may change; + /// a process crash between ack and republish can lose a parked foreign message. + /// + public static class StressOwnedWorkMessageCleanup + { + /// AMQP header name written by . + public const string StressRunIdHeader = "StressRunId"; + + /// + /// Upper bound on basic.get visits per cleanup (owned discards + foreign parks). + /// Prevents unbounded scans when the queue holds unrelated backlog. + /// + internal const int DefaultMaxVisits = 512; + + /// + /// Reads a StressRunId header value from AMQP headers (string or UTF-8 bytes). + /// + /// Delivery headers; may be null. + /// The run id when present and non-blank; otherwise null. + public static string? TryGetStressRunId(IDictionary? headers) + { + if (headers is null + || !headers.TryGetValue(StressRunIdHeader, out var raw) + || raw is null) + { + return null; + } + + switch (raw) + { + case string s when !string.IsNullOrWhiteSpace(s): + return s.Trim(); + case byte[] bytes when bytes.Length > 0: + return Encoding.UTF8.GetString(bytes).Trim(); + case ReadOnlyMemory memory when memory.Length > 0: + return Encoding.UTF8.GetString(memory.Span).Trim(); + case ArraySegment segment when segment.Count > 0: + return Encoding.UTF8.GetString(segment.AsSpan()).Trim(); + case Memory memory when memory.Length > 0: + return Encoding.UTF8.GetString(memory.Span).Trim(); + default: + return null; + } + } + + /// + /// Returns whether a delivery's StressRunId equals . + /// + /// Delivery headers. + /// Run id that owns messages eligible for discard. + /// when the header matches the owned run id. + public static bool IsOwnedByRun(IDictionary? headers, string ownedRunId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(ownedRunId); + var observed = TryGetStressRunId(headers); + return observed is not null + && string.Equals(observed, ownedRunId, StringComparison.Ordinal); + } + + /// + /// Connects briefly and removes ready messages on backfiller.{backbone} whose + /// StressRunId equals . Foreign messages are parked + /// (acked) and republished after the scan so owned messages behind them can be reached. + /// Best-effort: failures are written to stderr and do not throw (harness teardown must proceed). + /// + /// Broker connection settings (credentials never logged). + /// Provider backbone used to compose the work queue name. + /// Run id that owns messages to delete. + /// Cancels connect/get/settle/republish. + /// Maximum basic.get operations this call may perform. + /// Count of owned messages discarded. + public static async Task RemoveOwnedAsync( + StressHarnessOptions options, + string backbone, + string stressRunId, + CancellationToken cancellationToken, + int maxVisits = DefaultMaxVisits) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(backbone); + ArgumentException.ThrowIfNullOrWhiteSpace(stressRunId); + if (maxVisits <= 0) + { + return 0; + } + + var queueName = ArticleWorkStressClient.ComposeProviderEntity(backbone); + try + { + var factory = new ConnectionFactory + { + HostName = options.RabbitMqHost, + Port = options.RabbitMqPort, + VirtualHost = options.RabbitMqVirtualHost, + UserName = options.RabbitMqUsername, + Password = options.RabbitMqPassword, + RequestedHeartbeat = TimeSpan.FromSeconds(30), + AutomaticRecoveryEnabled = false, + ClientProvidedName = "VectorNNTPD.BackFillerService.Tests:stress-owned-cleanup", + }; + + await using var connection = await factory.CreateConnectionAsync(cancellationToken) + .ConfigureAwait(false); + await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var removed = 0; + var visits = 0; + var parkedForeign = new List(); + while (visits < maxVisits) + { + cancellationToken.ThrowIfCancellationRequested(); + var got = await channel.BasicGetAsync(queueName, autoAck: false, cancellationToken) + .ConfigureAwait(false); + if (got is null) + { + break; + } + + visits++; + if (IsOwnedByRun(got.BasicProperties.Headers, stressRunId)) + { + await channel.BasicNackAsync( + got.DeliveryTag, + multiple: false, + requeue: false, + cancellationToken) + .ConfigureAwait(false); + removed++; + continue; + } + + // Park foreign so the scan can continue past them (reject+requeue spins on classic head). + parkedForeign.Add( + new ParkedForeignMessage( + CopyProperties(got.BasicProperties), + got.Body.ToArray())); + await channel.BasicAckAsync(got.DeliveryTag, multiple: false, cancellationToken) + .ConfigureAwait(false); + } + + foreach (var foreign in parkedForeign) + { + cancellationToken.ThrowIfCancellationRequested(); + await channel.BasicPublishAsync( + exchange: queueName, + routingKey: queueName, + mandatory: false, + basicProperties: foreign.Properties, + body: foreign.Body, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + if (removed > 0 || parkedForeign.Count > 0) + { + Console.WriteLine( + $"stress-owned-cleanup queue={queueName} runId={stressRunId} removed={removed} parkedForeign={parkedForeign.Count} visits={visits}"); + } + + return removed; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + Console.Error.WriteLine( + $"stress-owned-cleanup failed for runId={stressRunId}: {ex.GetType().Name}"); + return 0; + } + } + + /// Copies settlement-relevant AMQP properties for a temporary foreign park/republish. + /// Properties from the gotten delivery. + /// A new suitable for republish. + internal static BasicProperties CopyProperties(IReadOnlyBasicProperties source) + { + ArgumentNullException.ThrowIfNull(source); + IDictionary? headers = null; + if (source.Headers is { Count: > 0 }) + { + headers = new Dictionary(source.Headers, StringComparer.Ordinal); + } + + return new BasicProperties + { + ContentType = source.ContentType, + ContentEncoding = source.ContentEncoding, + Headers = headers, + DeliveryMode = source.DeliveryMode, + Priority = source.Priority, + CorrelationId = source.CorrelationId, + ReplyTo = source.ReplyTo, + Expiration = source.Expiration, + MessageId = source.MessageId, + Timestamp = source.Timestamp, + Type = source.Type, + UserId = source.UserId, + AppId = source.AppId, + }; + } + + /// One foreign work message held in memory until the owned scan finishes. + /// AMQP properties to republish. + /// Payload octets to republish. + private readonly record struct ParkedForeignMessage(BasicProperties Properties, byte[] Body); + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs new file mode 100644 index 00000000..a48508b1 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs @@ -0,0 +1,98 @@ +using System.Text; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// Deterministic coverage for StressRunId header matching used by owned work-queue cleanup. + public sealed class StressOwnedWorkMessageCleanupTests + { + /// String header values match the owned run id. + [Fact] + public void IsOwnedByRun_matches_string_header() + { + var headers = new Dictionary + { + [StressOwnedWorkMessageCleanup.StressRunIdHeader] = "run-a", + }; + Assert.True(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-a")); + Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-b")); + } + + /// UTF-8 byte[] headers (RabbitMQ.Client default) match the owned run id. + [Fact] + public void IsOwnedByRun_matches_utf8_byte_header() + { + var headers = new Dictionary + { + [StressOwnedWorkMessageCleanup.StressRunIdHeader] = Encoding.UTF8.GetBytes("run-bytes"), + }; + Assert.True(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-bytes")); + Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "other")); + } + + /// Missing, blank, empty-byte, and non-string header values are never treated as owned. + [Fact] + public void IsOwnedByRun_rejects_missing_blank_empty_bytes_and_unknown_types() + { + Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(null, "run-a")); + Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(new Dictionary(), "run-a")); + Assert.False( + StressOwnedWorkMessageCleanup.IsOwnedByRun( + new Dictionary + { + [StressOwnedWorkMessageCleanup.StressRunIdHeader] = " ", + }, + "run-a")); + Assert.False( + StressOwnedWorkMessageCleanup.IsOwnedByRun( + new Dictionary + { + [StressOwnedWorkMessageCleanup.StressRunIdHeader] = Array.Empty(), + }, + "run-a")); + Assert.False( + StressOwnedWorkMessageCleanup.IsOwnedByRun( + new Dictionary + { + [StressOwnedWorkMessageCleanup.StressRunIdHeader] = 42, + }, + "run-a")); + Assert.Null( + StressOwnedWorkMessageCleanup.TryGetStressRunId( + new Dictionary + { + [StressOwnedWorkMessageCleanup.StressRunIdHeader] = 42, + })); + } + + /// TryGetStressRunId trims string values. + [Fact] + public void TryGetStressRunId_trims_string() + { + var headers = new Dictionary + { + [StressOwnedWorkMessageCleanup.StressRunIdHeader] = " padded-id ", + }; + Assert.Equal("padded-id", StressOwnedWorkMessageCleanup.TryGetStressRunId(headers)); + } + + /// CopyProperties preserves correlation, reply-to, and StressRunId for foreign republish. + [Fact] + public void CopyProperties_preserves_identity_and_stress_run_header() + { + var source = ArticleWorkStressClient.CreateRequestProperties( + correlationId: "corr-1", + replyTo: "reply-q", + requestId: Guid.Parse("11111111-1111-1111-1111-111111111111"), + stressRunId: "foreign-run"); + var copy = StressOwnedWorkMessageCleanup.CopyProperties(source); + Assert.Equal("corr-1", copy.CorrelationId); + Assert.Equal("reply-q", copy.ReplyTo); + Assert.Equal(source.ContentType, copy.ContentType); + Assert.Equal(source.DeliveryMode, copy.DeliveryMode); + Assert.Equal( + "foreign-run", + StressOwnedWorkMessageCleanup.TryGetStressRunId(copy.Headers)); + Assert.NotSame(source.Headers, copy.Headers); + } + } +}