diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs
index 4628e8ea..27a4ea71 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs
@@ -5,14 +5,15 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress
{
///
/// Removes only work-queue messages published by a specific stress run (StressRunId header).
- /// Non-matching messages are parked and republished; shared/production traffic is never purged wholesale.
+ /// Foreign deliveries stay unacknowledged and are requeued when the cleanup channel is disposed.
///
///
- /// Foreign messages cannot be basic.reject(requeue=true)'d while scanning: on classic queues that
- /// often resurfaces the same head message immediately, so owned payloads behind foreign backlog would
- /// never be reached. Park+ack then republish avoids that spin. Unavoidable limitations: brief absence
- /// of parked messages until republish; relative order vs messages published during cleanup may change;
- /// a process crash between ack and republish can lose a parked foreign message.
+ /// Foreign messages must not be basic.reject(requeue=true)'d during the scan: on classic queues
+ /// that often resurfaces the same head message. They also must not be acked and republished: that
+ /// creates a loss window on crash. Holding foreign deliveries unacked lets basic.get advance
+ /// through ready messages; channel/connection dispose returns unacked foreign work to the queue.
+ /// Concurrent publishers may still change relative order when foreign messages are requeued.
+ /// Brief unacked occupancy is far below typical broker consumer-timeout defaults (minutes).
///
public static class StressOwnedWorkMessageCleanup
{
@@ -20,7 +21,7 @@ public static class StressOwnedWorkMessageCleanup
public const string StressRunIdHeader = "StressRunId";
///
- /// Upper bound on basic.get visits per cleanup (owned discards + foreign parks).
+ /// Upper bound on basic.get visits per cleanup (owned discards + foreign holds).
/// Prevents unbounded scans when the queue holds unrelated backlog.
///
internal const int DefaultMaxVisits = 512;
@@ -71,15 +72,15 @@ public static bool IsOwnedByRun(IDictionary? headers, string ow
}
///
- /// Connects briefly and removes ready messages on backfiller.{backbone} whose
- /// StressRunId equals . Foreign messages are parked
- /// (acked) and republished after the scan so owned messages behind them can be reached.
+ /// Connects briefly and discards ready messages on backfiller.{backbone} whose
+ /// StressRunId equals . Foreign deliveries remain
+ /// unacknowledged and are requeued when the cleanup channel is disposed.
/// Best-effort: failures are written to stderr and do not throw (harness teardown must proceed).
///
/// Broker connection settings (credentials never logged).
/// Provider backbone used to compose the work queue name.
/// Run id that owns messages to delete.
- /// Cancels connect/get/settle/republish.
+ /// Cancels connect/get/settle.
/// Maximum basic.get operations this call may perform.
/// Count of owned messages discarded.
public static async Task RemoveOwnedAsync(
@@ -117,58 +118,17 @@ public static async Task RemoveOwnedAsync(
await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
- var removed = 0;
- var visits = 0;
- var parkedForeign = new List();
- while (visits < maxVisits)
- {
- cancellationToken.ThrowIfCancellationRequested();
- var got = await channel.BasicGetAsync(queueName, autoAck: false, cancellationToken)
- .ConfigureAwait(false);
- if (got is null)
- {
- break;
- }
-
- visits++;
- if (IsOwnedByRun(got.BasicProperties.Headers, stressRunId))
- {
- await channel.BasicNackAsync(
- got.DeliveryTag,
- multiple: false,
- requeue: false,
- cancellationToken)
- .ConfigureAwait(false);
- removed++;
- continue;
- }
-
- // Park foreign so the scan can continue past them (reject+requeue spins on classic head).
- parkedForeign.Add(
- new ParkedForeignMessage(
- CopyProperties(got.BasicProperties),
- got.Body.ToArray()));
- await channel.BasicAckAsync(got.DeliveryTag, multiple: false, cancellationToken)
- .ConfigureAwait(false);
- }
-
- foreach (var foreign in parkedForeign)
- {
- cancellationToken.ThrowIfCancellationRequested();
- await channel.BasicPublishAsync(
- exchange: queueName,
- routingKey: queueName,
- mandatory: false,
- basicProperties: foreign.Properties,
- body: foreign.Body,
- cancellationToken: cancellationToken)
- .ConfigureAwait(false);
- }
+ var removed = await DiscardOwnedHoldingForeignAsync(
+ new LiveCleanupChannel(channel, queueName),
+ stressRunId,
+ maxVisits,
+ cancellationToken)
+ .ConfigureAwait(false);
- if (removed > 0 || parkedForeign.Count > 0)
+ if (removed > 0)
{
Console.WriteLine(
- $"stress-owned-cleanup queue={queueName} runId={stressRunId} removed={removed} parkedForeign={parkedForeign.Count} visits={visits}");
+ $"stress-owned-cleanup queue={queueName} runId={stressRunId} removed={removed}");
}
return removed;
@@ -181,39 +141,95 @@ await channel.BasicPublishAsync(
}
}
- /// Copies settlement-relevant AMQP properties for a temporary foreign park/republish.
- /// Properties from the gotten delivery.
- /// A new suitable for republish.
- internal static BasicProperties CopyProperties(IReadOnlyBasicProperties source)
+ ///
+ /// Scans ready deliveries: nack-discards owned messages; leaves foreign unacked.
+ /// Caller must dispose the channel so foreign deliveries are requeued by the broker.
+ ///
+ /// Cleanup channel (get + discard only; no foreign ack).
+ /// Owned run id.
+ /// Visit bound.
+ /// Cancels get/discard.
+ /// Owned messages discarded.
+ internal static async Task DiscardOwnedHoldingForeignAsync(
+ IStressCleanupChannel channel,
+ string stressRunId,
+ int maxVisits,
+ CancellationToken cancellationToken)
{
- ArgumentNullException.ThrowIfNull(source);
- IDictionary? headers = null;
- if (source.Headers is { Count: > 0 })
+ ArgumentNullException.ThrowIfNull(channel);
+ ArgumentException.ThrowIfNullOrWhiteSpace(stressRunId);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxVisits);
+
+ var removed = 0;
+ var visits = 0;
+ while (visits < maxVisits)
{
- headers = new Dictionary(source.Headers, StringComparer.Ordinal);
+ cancellationToken.ThrowIfCancellationRequested();
+ var got = await channel.BasicGetAsync(cancellationToken).ConfigureAwait(false);
+ if (got is not { } delivery)
+ {
+ break;
+ }
+
+ visits++;
+ if (IsOwnedByRun(delivery.Headers, stressRunId))
+ {
+ await channel.BasicNackDiscardAsync(delivery.DeliveryTag, cancellationToken)
+ .ConfigureAwait(false);
+ removed++;
+ }
+ // Foreign / missing / malformed StressRunId: leave unacked for channel dispose requeue.
}
- return new BasicProperties
- {
- ContentType = source.ContentType,
- ContentEncoding = source.ContentEncoding,
- Headers = headers,
- DeliveryMode = source.DeliveryMode,
- Priority = source.Priority,
- CorrelationId = source.CorrelationId,
- ReplyTo = source.ReplyTo,
- Expiration = source.Expiration,
- MessageId = source.MessageId,
- Timestamp = source.Timestamp,
- Type = source.Type,
- UserId = source.UserId,
- AppId = source.AppId,
- };
+ return removed;
+ }
+
+ /// One gotten delivery used by the cleanup scan (headers + delivery tag only).
+ /// Broker delivery tag for nack/discard.
+ /// AMQP headers; may be null.
+ internal readonly record struct StressCleanupDelivery(
+ ulong DeliveryTag,
+ IDictionary? Headers);
+
+ ///
+ /// Minimal channel surface for owned discard while holding foreign unacked.
+ /// Implementations must not ack foreign deliveries.
+ ///
+ internal interface IStressCleanupChannel : IAsyncDisposable
+ {
+ /// Gets the next ready message, or null when the ready set is empty.
+ /// Cancels the get.
+ Task BasicGetAsync(CancellationToken cancellationToken);
+
+ /// Discards one owned delivery without requeue.
+ /// Tag from a prior get.
+ /// Cancels the nack.
+ Task BasicNackDiscardAsync(ulong deliveryTag, CancellationToken cancellationToken);
}
- /// One foreign work message held in memory until the owned scan finishes.
- /// AMQP properties to republish.
- /// Payload octets to republish.
- private readonly record struct ParkedForeignMessage(BasicProperties Properties, byte[] Body);
+ /// Live RabbitMQ.Client adapter for .
+ private sealed class LiveCleanupChannel(IChannel channel, string queueName) : IStressCleanupChannel
+ {
+ ///
+ public async Task BasicGetAsync(CancellationToken cancellationToken)
+ {
+ var got = await channel.BasicGetAsync(queueName, autoAck: false, cancellationToken)
+ .ConfigureAwait(false);
+ if (got is null)
+ {
+ return null;
+ }
+
+ return new StressCleanupDelivery(got.DeliveryTag, got.BasicProperties.Headers);
+ }
+
+ ///
+ public Task BasicNackDiscardAsync(ulong deliveryTag, CancellationToken cancellationToken) =>
+ channel.BasicNackAsync(deliveryTag, multiple: false, requeue: false, cancellationToken)
+ .AsTask();
+
+ ///
+ public ValueTask DisposeAsync() => channel.DisposeAsync();
+ }
}
}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs
index a48508b1..d36465c4 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs
@@ -1,20 +1,26 @@
using System.Text;
+using static VectorNNTPD.BackFillerService.Tests.Stress.StressOwnedWorkMessageCleanup;
namespace VectorNNTPD.BackFillerService.Tests.Stress
{
- /// Deterministic coverage for StressRunId header matching used by owned work-queue cleanup.
+ ///
+ /// Deterministic coverage for StressRunId matching and the hold-unacked foreign scan
+ /// ().
+ ///
public sealed class StressOwnedWorkMessageCleanupTests
{
+ private const string OwnedRunId = "owned-run";
+
/// String header values match the owned run id.
[Fact]
public void IsOwnedByRun_matches_string_header()
{
var headers = new Dictionary
{
- [StressOwnedWorkMessageCleanup.StressRunIdHeader] = "run-a",
+ [StressRunIdHeader] = "run-a",
};
- Assert.True(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-a"));
- Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-b"));
+ Assert.True(IsOwnedByRun(headers, "run-a"));
+ Assert.False(IsOwnedByRun(headers, "run-b"));
}
/// UTF-8 byte[] headers (RabbitMQ.Client default) match the owned run id.
@@ -23,45 +29,32 @@ public void IsOwnedByRun_matches_utf8_byte_header()
{
var headers = new Dictionary
{
- [StressOwnedWorkMessageCleanup.StressRunIdHeader] = Encoding.UTF8.GetBytes("run-bytes"),
+ [StressRunIdHeader] = Encoding.UTF8.GetBytes("run-bytes"),
};
- Assert.True(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-bytes"));
- Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "other"));
+ Assert.True(IsOwnedByRun(headers, "run-bytes"));
+ Assert.False(IsOwnedByRun(headers, "other"));
}
/// Missing, blank, empty-byte, and non-string header values are never treated as owned.
[Fact]
public void IsOwnedByRun_rejects_missing_blank_empty_bytes_and_unknown_types()
{
- Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(null, "run-a"));
- Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(new Dictionary(), "run-a"));
+ Assert.False(IsOwnedByRun(null, "run-a"));
+ Assert.False(IsOwnedByRun(new Dictionary(), "run-a"));
Assert.False(
- StressOwnedWorkMessageCleanup.IsOwnedByRun(
- new Dictionary
- {
- [StressOwnedWorkMessageCleanup.StressRunIdHeader] = " ",
- },
+ IsOwnedByRun(
+ new Dictionary { [StressRunIdHeader] = " " },
"run-a"));
Assert.False(
- StressOwnedWorkMessageCleanup.IsOwnedByRun(
- new Dictionary
- {
- [StressOwnedWorkMessageCleanup.StressRunIdHeader] = Array.Empty(),
- },
+ IsOwnedByRun(
+ new Dictionary { [StressRunIdHeader] = Array.Empty() },
"run-a"));
Assert.False(
- StressOwnedWorkMessageCleanup.IsOwnedByRun(
- new Dictionary
- {
- [StressOwnedWorkMessageCleanup.StressRunIdHeader] = 42,
- },
+ IsOwnedByRun(
+ new Dictionary { [StressRunIdHeader] = 42 },
"run-a"));
Assert.Null(
- StressOwnedWorkMessageCleanup.TryGetStressRunId(
- new Dictionary
- {
- [StressOwnedWorkMessageCleanup.StressRunIdHeader] = 42,
- }));
+ TryGetStressRunId(new Dictionary { [StressRunIdHeader] = 42 }));
}
/// TryGetStressRunId trims string values.
@@ -70,29 +63,225 @@ public void TryGetStressRunId_trims_string()
{
var headers = new Dictionary
{
- [StressOwnedWorkMessageCleanup.StressRunIdHeader] = " padded-id ",
+ [StressRunIdHeader] = " padded-id ",
};
- Assert.Equal("padded-id", StressOwnedWorkMessageCleanup.TryGetStressRunId(headers));
+ Assert.Equal("padded-id", TryGetStressRunId(headers));
+ }
+
+ /// Owned deliveries are nack-discarded; foreign deliveries are never settled.
+ [Fact]
+ public async Task Scan_discards_owned_and_never_acks_foreign()
+ {
+ var channel = new FakeCleanupChannel(
+ Delivery(1, OwnedRunId),
+ Delivery(2, "foreign-run"),
+ Delivery(3, OwnedRunId));
+
+ var removed = await DiscardOwnedHoldingForeignAsync(
+ channel,
+ OwnedRunId,
+ maxVisits: 32,
+ CancellationToken.None);
+
+ Assert.Equal(2, removed);
+ Assert.Equal(new ulong[] { 1, 3 }, channel.DiscardedTags);
+ Assert.Empty(channel.AckedTags);
+ Assert.Equal(new ulong[] { 2 }, channel.UnackedForeignTags);
+ Assert.Equal(0, channel.DisposeCount);
}
- /// CopyProperties preserves correlation, reply-to, and StressRunId for foreign republish.
+ /// Missing, blank, empty-byte, and unsupported headers are held as foreign (never discarded).
[Fact]
- public void CopyProperties_preserves_identity_and_stress_run_header()
+ public async Task Scan_treats_missing_blank_and_malformed_headers_as_foreign()
{
- var source = ArticleWorkStressClient.CreateRequestProperties(
- correlationId: "corr-1",
- replyTo: "reply-q",
- requestId: Guid.Parse("11111111-1111-1111-1111-111111111111"),
- stressRunId: "foreign-run");
- var copy = StressOwnedWorkMessageCleanup.CopyProperties(source);
- Assert.Equal("corr-1", copy.CorrelationId);
- Assert.Equal("reply-q", copy.ReplyTo);
- Assert.Equal(source.ContentType, copy.ContentType);
- Assert.Equal(source.DeliveryMode, copy.DeliveryMode);
- Assert.Equal(
- "foreign-run",
- StressOwnedWorkMessageCleanup.TryGetStressRunId(copy.Headers));
- Assert.NotSame(source.Headers, copy.Headers);
+ var channel = new FakeCleanupChannel(
+ new StressCleanupDelivery(1, null),
+ new StressCleanupDelivery(2, new Dictionary()),
+ new StressCleanupDelivery(3, new Dictionary { [StressRunIdHeader] = " " }),
+ new StressCleanupDelivery(4, new Dictionary { [StressRunIdHeader] = Array.Empty() }),
+ new StressCleanupDelivery(5, new Dictionary { [StressRunIdHeader] = 99 }),
+ Delivery(6, OwnedRunId));
+
+ var removed = await DiscardOwnedHoldingForeignAsync(
+ channel,
+ OwnedRunId,
+ maxVisits: 32,
+ CancellationToken.None);
+
+ Assert.Equal(1, removed);
+ Assert.Equal(new ulong[] { 6 }, channel.DiscardedTags);
+ Assert.Empty(channel.AckedTags);
+ Assert.Equal(new ulong[] { 1, 2, 3, 4, 5 }, channel.UnackedForeignTags);
+ }
+
+ /// Owned messages behind foreign ready messages are still reached (no head-spin).
+ [Fact]
+ public async Task Scan_reaches_owned_messages_behind_foreign()
+ {
+ var channel = new FakeCleanupChannel(
+ Delivery(10, "foreign-a"),
+ Delivery(11, "foreign-b"),
+ Delivery(12, OwnedRunId));
+
+ var removed = await DiscardOwnedHoldingForeignAsync(
+ channel,
+ OwnedRunId,
+ maxVisits: 32,
+ CancellationToken.None);
+
+ Assert.Equal(1, removed);
+ Assert.Equal(new ulong[] { 12 }, channel.DiscardedTags);
+ Assert.Equal(new ulong[] { 10, 11 }, channel.UnackedForeignTags);
+ Assert.Equal(3, channel.GetCount);
+ }
+
+ /// Scan stops at maxVisits even when more deliveries remain.
+ [Fact]
+ public async Task Scan_stops_at_visit_limit()
+ {
+ var channel = new FakeCleanupChannel(
+ Delivery(1, "foreign"),
+ Delivery(2, OwnedRunId),
+ Delivery(3, OwnedRunId),
+ Delivery(4, OwnedRunId));
+
+ var removed = await DiscardOwnedHoldingForeignAsync(
+ channel,
+ OwnedRunId,
+ maxVisits: 2,
+ CancellationToken.None);
+
+ Assert.Equal(1, removed);
+ Assert.Equal(2, channel.GetCount);
+ Assert.Equal(new ulong[] { 2 }, channel.DiscardedTags);
+ Assert.Equal(new ulong[] { 1 }, channel.UnackedForeignTags);
+ Assert.Equal(2, channel.RemainingReadyCount);
+ }
+
+ ///
+ /// Callers that use await using on the cleanup channel dispose it after the scan,
+ /// including when the scan throws — modelling live channel dispose that requeues foreign.
+ ///
+ [Fact]
+ public async Task Channel_dispose_is_guaranteed_when_scan_throws()
+ {
+ var channel = new FakeCleanupChannel(Delivery(1, "foreign"))
+ {
+ ThrowOnGetAfter = 0,
+ };
+
+ await Assert.ThrowsAsync(async () =>
+ {
+ await using (channel)
+ {
+ _ = await DiscardOwnedHoldingForeignAsync(
+ channel,
+ OwnedRunId,
+ maxVisits: 8,
+ CancellationToken.None);
+ }
+ });
+
+ Assert.Equal(1, channel.DisposeCount);
+ Assert.Empty(channel.DiscardedTags);
+ Assert.Empty(channel.AckedTags);
+ }
+
+ /// Builds an owned/foreign delivery with a string StressRunId header.
+ private static StressCleanupDelivery Delivery(ulong tag, string stressRunId) =>
+ new(
+ tag,
+ new Dictionary
+ {
+ [StressRunIdHeader] = stressRunId,
+ });
+
+ ///
+ /// In-memory cleanup channel: gets advance past unacked foreign holds; never invents acks for foreign.
+ ///
+ private sealed class FakeCleanupChannel : IStressCleanupChannel
+ {
+ private readonly Queue _ready;
+ private readonly List _unackedForeign = [];
+ private readonly List _discarded = [];
+ private readonly List _acked = [];
+ private int _gets;
+
+ /// Creates a fake with the given ready deliveries in FIFO order.
+ /// Initial ready set.
+ public FakeCleanupChannel(params StressCleanupDelivery[] ready)
+ {
+ _ready = new Queue(ready);
+ }
+
+ /// When non-negative, the get after this many successful gets throws.
+ public int ThrowOnGetAfter { get; init; } = -1;
+
+ /// Successful get count.
+ public int GetCount => _gets;
+
+ /// Ready messages not yet gotten.
+ public int RemainingReadyCount => _ready.Count;
+
+ /// Delivery tags left unacked (foreign holds).
+ public IReadOnlyList UnackedForeignTags => _unackedForeign;
+
+ /// Delivery tags discarded via nack requeue=false.
+ public IReadOnlyList DiscardedTags => _discarded;
+
+ /// Delivery tags that were incorrectly acked (must stay empty in correct scans).
+ public IReadOnlyList AckedTags => _acked;
+
+ /// Dispose invocations.
+ public int DisposeCount { get; private set; }
+
+ ///
+ public Task BasicGetAsync(CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (ThrowOnGetAfter >= 0 && _gets >= ThrowOnGetAfter)
+ {
+ throw new InvalidOperationException("simulated get failure");
+ }
+
+ if (_ready.Count == 0)
+ {
+ return Task.FromResult(null);
+ }
+
+ var next = _ready.Dequeue();
+ _gets++;
+ // Model broker behaviour: gotten foreign stays out of ready until channel dispose requeues.
+ _unackedForeign.Add(next.DeliveryTag);
+ return Task.FromResult(next);
+ }
+
+ ///
+ public Task BasicNackDiscardAsync(ulong deliveryTag, CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (!_unackedForeign.Remove(deliveryTag))
+ {
+ throw new InvalidOperationException($"delivery {deliveryTag} was not unacked");
+ }
+
+ _discarded.Add(deliveryTag);
+ return Task.CompletedTask;
+ }
+
+ ///
+ public ValueTask DisposeAsync()
+ {
+ DisposeCount++;
+ // Model broker requeue of remaining unacked foreign on channel close.
+ foreach (var tag in _unackedForeign)
+ {
+ _ready.Enqueue(new StressCleanupDelivery(tag, null));
+ }
+
+ _unackedForeign.Clear();
+ return ValueTask.CompletedTask;
+ }
}
}
}