diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs
index 59d91f8a..82f62a96 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs
@@ -582,21 +582,34 @@ or NntpResponseKind.NotAvailable
var drainStarted = Stopwatch.GetTimestamp();
var drainDeadline = drainStarted
+ (long)(_options.ReplyQueueDisposeDrainTimeout.TotalSeconds * Stopwatch.Frequency);
+ // Wait for caller-visible outstanding first (waiters + delivery workers).
while ((metrics.Outstanding > 0
|| metrics.DeliveryOutstanding > 0
- || workClient.WaitingCount > 0
- || workClient.BrokerOutstandingCount > 0)
+ || workClient.WaitingCount > 0)
&& Stopwatch.GetTimestamp() < drainDeadline)
{
await Task.Delay(100, CancellationToken.None).ConfigureAwait(false);
}
+ // Short grace for late ReplyTo arrivals. Do not burn the full dispose drain waiting on
+ // broker-outstanding work that will never complete when consumers=0 (silent / absent).
+ // Owned ready messages are removed after dispose via StressRunId-scoped cleanup.
+ var lateReplyGrace = TimeSpan.FromSeconds(5);
+ var lateReplyDeadline = Stopwatch.GetTimestamp()
+ + (long)(lateReplyGrace.TotalSeconds * Stopwatch.Frequency);
+ var brokerWaitDeadline = Math.Min(drainDeadline, lateReplyDeadline);
+ while (workClient.BrokerOutstandingCount > 0
+ && Stopwatch.GetTimestamp() < brokerWaitDeadline)
+ {
+ await Task.Delay(100, CancellationToken.None).ConfigureAwait(false);
+ }
+
metrics.SetDrainMilliseconds((long)Stopwatch.GetElapsedTime(drainStarted).TotalMilliseconds);
var brokerOutstandingAtDrainEnd = workClient.BrokerOutstandingCount;
if (brokerOutstandingAtDrainEnd > 0)
{
Console.Error.WriteLine(
- $"Harness drain ended with brokerOutstanding={brokerOutstandingAtDrainEnd} waiting={workClient.WaitingCount}; dispose will apply bounded ReplyTo drain.");
+ $"Harness drain ended with brokerOutstanding={brokerOutstandingAtDrainEnd} waiting={workClient.WaitingCount}; dispose will apply bounded ReplyTo drain; owned work-queue cleanup follows.");
}
if (pool is not null)
@@ -623,9 +636,23 @@ await File.AppendAllTextAsync(
}
finally
{
- if (disposeReplyClientOnFailure)
+ try
+ {
+ if (disposeReplyClientOnFailure)
+ {
+ await workClient.DisposeAsync().ConfigureAwait(false);
+ }
+ }
+ finally
{
- await workClient.DisposeAsync().ConfigureAwait(false);
+ // Always remove this run's ready work messages (success and failure paths).
+ // Does not purge foreign traffic: only StressRunId-tagged payloads are removed.
+ _ = await StressOwnedWorkMessageCleanup.RemoveOwnedAsync(
+ _options,
+ backbone,
+ runId,
+ CancellationToken.None)
+ .ConfigureAwait(false);
}
}
}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs
new file mode 100644
index 00000000..4628e8ea
--- /dev/null
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs
@@ -0,0 +1,219 @@
+using System.Text;
+using RabbitMQ.Client;
+
+namespace VectorNNTPD.BackFillerService.Tests.Stress
+{
+ ///
+ /// Removes only work-queue messages published by a specific stress run (StressRunId header).
+ /// Non-matching messages are parked and republished; shared/production traffic is never purged wholesale.
+ ///
+ ///
+ /// Foreign messages cannot be basic.reject(requeue=true)'d while scanning: on classic queues that
+ /// often resurfaces the same head message immediately, so owned payloads behind foreign backlog would
+ /// never be reached. Park+ack then republish avoids that spin. Unavoidable limitations: brief absence
+ /// of parked messages until republish; relative order vs messages published during cleanup may change;
+ /// a process crash between ack and republish can lose a parked foreign message.
+ ///
+ public static class StressOwnedWorkMessageCleanup
+ {
+ /// AMQP header name written by .
+ public const string StressRunIdHeader = "StressRunId";
+
+ ///
+ /// Upper bound on basic.get visits per cleanup (owned discards + foreign parks).
+ /// Prevents unbounded scans when the queue holds unrelated backlog.
+ ///
+ internal const int DefaultMaxVisits = 512;
+
+ ///
+ /// Reads a StressRunId header value from AMQP headers (string or UTF-8 bytes).
+ ///
+ /// Delivery headers; may be null.
+ /// The run id when present and non-blank; otherwise null.
+ public static string? TryGetStressRunId(IDictionary? headers)
+ {
+ if (headers is null
+ || !headers.TryGetValue(StressRunIdHeader, out var raw)
+ || raw is null)
+ {
+ return null;
+ }
+
+ switch (raw)
+ {
+ case string s when !string.IsNullOrWhiteSpace(s):
+ return s.Trim();
+ case byte[] bytes when bytes.Length > 0:
+ return Encoding.UTF8.GetString(bytes).Trim();
+ case ReadOnlyMemory memory when memory.Length > 0:
+ return Encoding.UTF8.GetString(memory.Span).Trim();
+ case ArraySegment segment when segment.Count > 0:
+ return Encoding.UTF8.GetString(segment.AsSpan()).Trim();
+ case Memory memory when memory.Length > 0:
+ return Encoding.UTF8.GetString(memory.Span).Trim();
+ default:
+ return null;
+ }
+ }
+
+ ///
+ /// Returns whether a delivery's StressRunId equals .
+ ///
+ /// Delivery headers.
+ /// Run id that owns messages eligible for discard.
+ /// when the header matches the owned run id.
+ public static bool IsOwnedByRun(IDictionary? headers, string ownedRunId)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(ownedRunId);
+ var observed = TryGetStressRunId(headers);
+ return observed is not null
+ && string.Equals(observed, ownedRunId, StringComparison.Ordinal);
+ }
+
+ ///
+ /// Connects briefly and removes ready messages on backfiller.{backbone} whose
+ /// StressRunId equals . Foreign messages are parked
+ /// (acked) and republished after the scan so owned messages behind them can be reached.
+ /// Best-effort: failures are written to stderr and do not throw (harness teardown must proceed).
+ ///
+ /// Broker connection settings (credentials never logged).
+ /// Provider backbone used to compose the work queue name.
+ /// Run id that owns messages to delete.
+ /// Cancels connect/get/settle/republish.
+ /// Maximum basic.get operations this call may perform.
+ /// Count of owned messages discarded.
+ public static async Task RemoveOwnedAsync(
+ StressHarnessOptions options,
+ string backbone,
+ string stressRunId,
+ CancellationToken cancellationToken,
+ int maxVisits = DefaultMaxVisits)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentException.ThrowIfNullOrWhiteSpace(backbone);
+ ArgumentException.ThrowIfNullOrWhiteSpace(stressRunId);
+ if (maxVisits <= 0)
+ {
+ return 0;
+ }
+
+ var queueName = ArticleWorkStressClient.ComposeProviderEntity(backbone);
+ try
+ {
+ var factory = new ConnectionFactory
+ {
+ HostName = options.RabbitMqHost,
+ Port = options.RabbitMqPort,
+ VirtualHost = options.RabbitMqVirtualHost,
+ UserName = options.RabbitMqUsername,
+ Password = options.RabbitMqPassword,
+ RequestedHeartbeat = TimeSpan.FromSeconds(30),
+ AutomaticRecoveryEnabled = false,
+ ClientProvidedName = "VectorNNTPD.BackFillerService.Tests:stress-owned-cleanup",
+ };
+
+ await using var connection = await factory.CreateConnectionAsync(cancellationToken)
+ .ConfigureAwait(false);
+ await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken)
+ .ConfigureAwait(false);
+
+ var removed = 0;
+ var visits = 0;
+ var parkedForeign = new List();
+ while (visits < maxVisits)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var got = await channel.BasicGetAsync(queueName, autoAck: false, cancellationToken)
+ .ConfigureAwait(false);
+ if (got is null)
+ {
+ break;
+ }
+
+ visits++;
+ if (IsOwnedByRun(got.BasicProperties.Headers, stressRunId))
+ {
+ await channel.BasicNackAsync(
+ got.DeliveryTag,
+ multiple: false,
+ requeue: false,
+ cancellationToken)
+ .ConfigureAwait(false);
+ removed++;
+ continue;
+ }
+
+ // Park foreign so the scan can continue past them (reject+requeue spins on classic head).
+ parkedForeign.Add(
+ new ParkedForeignMessage(
+ CopyProperties(got.BasicProperties),
+ got.Body.ToArray()));
+ await channel.BasicAckAsync(got.DeliveryTag, multiple: false, cancellationToken)
+ .ConfigureAwait(false);
+ }
+
+ foreach (var foreign in parkedForeign)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ await channel.BasicPublishAsync(
+ exchange: queueName,
+ routingKey: queueName,
+ mandatory: false,
+ basicProperties: foreign.Properties,
+ body: foreign.Body,
+ cancellationToken: cancellationToken)
+ .ConfigureAwait(false);
+ }
+
+ if (removed > 0 || parkedForeign.Count > 0)
+ {
+ Console.WriteLine(
+ $"stress-owned-cleanup queue={queueName} runId={stressRunId} removed={removed} parkedForeign={parkedForeign.Count} visits={visits}");
+ }
+
+ return removed;
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ Console.Error.WriteLine(
+ $"stress-owned-cleanup failed for runId={stressRunId}: {ex.GetType().Name}");
+ return 0;
+ }
+ }
+
+ /// Copies settlement-relevant AMQP properties for a temporary foreign park/republish.
+ /// Properties from the gotten delivery.
+ /// A new suitable for republish.
+ internal static BasicProperties CopyProperties(IReadOnlyBasicProperties source)
+ {
+ ArgumentNullException.ThrowIfNull(source);
+ IDictionary? headers = null;
+ if (source.Headers is { Count: > 0 })
+ {
+ headers = new Dictionary(source.Headers, StringComparer.Ordinal);
+ }
+
+ return new BasicProperties
+ {
+ ContentType = source.ContentType,
+ ContentEncoding = source.ContentEncoding,
+ Headers = headers,
+ DeliveryMode = source.DeliveryMode,
+ Priority = source.Priority,
+ CorrelationId = source.CorrelationId,
+ ReplyTo = source.ReplyTo,
+ Expiration = source.Expiration,
+ MessageId = source.MessageId,
+ Timestamp = source.Timestamp,
+ Type = source.Type,
+ UserId = source.UserId,
+ AppId = source.AppId,
+ };
+ }
+
+ /// One foreign work message held in memory until the owned scan finishes.
+ /// AMQP properties to republish.
+ /// Payload octets to republish.
+ private readonly record struct ParkedForeignMessage(BasicProperties Properties, byte[] Body);
+ }
+}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs
new file mode 100644
index 00000000..a48508b1
--- /dev/null
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs
@@ -0,0 +1,98 @@
+using System.Text;
+
+namespace VectorNNTPD.BackFillerService.Tests.Stress
+{
+ /// Deterministic coverage for StressRunId header matching used by owned work-queue cleanup.
+ public sealed class StressOwnedWorkMessageCleanupTests
+ {
+ /// String header values match the owned run id.
+ [Fact]
+ public void IsOwnedByRun_matches_string_header()
+ {
+ var headers = new Dictionary
+ {
+ [StressOwnedWorkMessageCleanup.StressRunIdHeader] = "run-a",
+ };
+ Assert.True(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-a"));
+ Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-b"));
+ }
+
+ /// UTF-8 byte[] headers (RabbitMQ.Client default) match the owned run id.
+ [Fact]
+ public void IsOwnedByRun_matches_utf8_byte_header()
+ {
+ var headers = new Dictionary
+ {
+ [StressOwnedWorkMessageCleanup.StressRunIdHeader] = Encoding.UTF8.GetBytes("run-bytes"),
+ };
+ Assert.True(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-bytes"));
+ Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "other"));
+ }
+
+ /// Missing, blank, empty-byte, and non-string header values are never treated as owned.
+ [Fact]
+ public void IsOwnedByRun_rejects_missing_blank_empty_bytes_and_unknown_types()
+ {
+ Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(null, "run-a"));
+ Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(new Dictionary(), "run-a"));
+ Assert.False(
+ StressOwnedWorkMessageCleanup.IsOwnedByRun(
+ new Dictionary
+ {
+ [StressOwnedWorkMessageCleanup.StressRunIdHeader] = " ",
+ },
+ "run-a"));
+ Assert.False(
+ StressOwnedWorkMessageCleanup.IsOwnedByRun(
+ new Dictionary
+ {
+ [StressOwnedWorkMessageCleanup.StressRunIdHeader] = Array.Empty(),
+ },
+ "run-a"));
+ Assert.False(
+ StressOwnedWorkMessageCleanup.IsOwnedByRun(
+ new Dictionary
+ {
+ [StressOwnedWorkMessageCleanup.StressRunIdHeader] = 42,
+ },
+ "run-a"));
+ Assert.Null(
+ StressOwnedWorkMessageCleanup.TryGetStressRunId(
+ new Dictionary
+ {
+ [StressOwnedWorkMessageCleanup.StressRunIdHeader] = 42,
+ }));
+ }
+
+ /// TryGetStressRunId trims string values.
+ [Fact]
+ public void TryGetStressRunId_trims_string()
+ {
+ var headers = new Dictionary
+ {
+ [StressOwnedWorkMessageCleanup.StressRunIdHeader] = " padded-id ",
+ };
+ Assert.Equal("padded-id", StressOwnedWorkMessageCleanup.TryGetStressRunId(headers));
+ }
+
+ /// CopyProperties preserves correlation, reply-to, and StressRunId for foreign republish.
+ [Fact]
+ public void CopyProperties_preserves_identity_and_stress_run_header()
+ {
+ var source = ArticleWorkStressClient.CreateRequestProperties(
+ correlationId: "corr-1",
+ replyTo: "reply-q",
+ requestId: Guid.Parse("11111111-1111-1111-1111-111111111111"),
+ stressRunId: "foreign-run");
+ var copy = StressOwnedWorkMessageCleanup.CopyProperties(source);
+ Assert.Equal("corr-1", copy.CorrelationId);
+ Assert.Equal("reply-q", copy.ReplyTo);
+ Assert.Equal(source.ContentType, copy.ContentType);
+ Assert.Equal(source.DeliveryMode, copy.DeliveryMode);
+ Assert.Equal(
+ "foreign-run",
+ StressOwnedWorkMessageCleanup.TryGetStressRunId(copy.Headers));
+ Assert.NotSame(source.Headers, copy.Headers);
+ }
+ }
+}