From 4fe64fc71ffd39cd16340eb4df423c083e41d4c4 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Mon, 26 Oct 2026 08:23:35 +0200 Subject: [PATCH] Make stress queue cleanup safe for foreign messages - Keep foreign deliveries unacknowledged during cleanup - Discard only messages matching the current StressRunId - Rely on channel disposal to requeue foreign deliveries - Add deterministic tests for mixed messages and scan limits --- .../Stress/StressOwnedWorkMessageCleanup.cs | 192 ++++++------ .../StressOwnedWorkMessageCleanupTests.cs | 283 +++++++++++++++--- 2 files changed, 340 insertions(+), 135 deletions(-) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs index 4628e8ea..27a4ea71 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanup.cs @@ -5,14 +5,15 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress { /// /// Removes only work-queue messages published by a specific stress run (StressRunId header). - /// Non-matching messages are parked and republished; shared/production traffic is never purged wholesale. + /// Foreign deliveries stay unacknowledged and are requeued when the cleanup channel is disposed. /// /// - /// Foreign messages cannot be basic.reject(requeue=true)'d while scanning: on classic queues that - /// often resurfaces the same head message immediately, so owned payloads behind foreign backlog would - /// never be reached. Park+ack then republish avoids that spin. Unavoidable limitations: brief absence - /// of parked messages until republish; relative order vs messages published during cleanup may change; - /// a process crash between ack and republish can lose a parked foreign message. + /// Foreign messages must not be basic.reject(requeue=true)'d during the scan: on classic queues + /// that often resurfaces the same head message. They also must not be acked and republished: that + /// creates a loss window on crash. Holding foreign deliveries unacked lets basic.get advance + /// through ready messages; channel/connection dispose returns unacked foreign work to the queue. + /// Concurrent publishers may still change relative order when foreign messages are requeued. + /// Brief unacked occupancy is far below typical broker consumer-timeout defaults (minutes). /// public static class StressOwnedWorkMessageCleanup { @@ -20,7 +21,7 @@ public static class StressOwnedWorkMessageCleanup public const string StressRunIdHeader = "StressRunId"; /// - /// Upper bound on basic.get visits per cleanup (owned discards + foreign parks). + /// Upper bound on basic.get visits per cleanup (owned discards + foreign holds). /// Prevents unbounded scans when the queue holds unrelated backlog. /// internal const int DefaultMaxVisits = 512; @@ -71,15 +72,15 @@ public static bool IsOwnedByRun(IDictionary? headers, string ow } /// - /// Connects briefly and removes ready messages on backfiller.{backbone} whose - /// StressRunId equals . Foreign messages are parked - /// (acked) and republished after the scan so owned messages behind them can be reached. + /// Connects briefly and discards ready messages on backfiller.{backbone} whose + /// StressRunId equals . Foreign deliveries remain + /// unacknowledged and are requeued when the cleanup channel is disposed. /// Best-effort: failures are written to stderr and do not throw (harness teardown must proceed). /// /// Broker connection settings (credentials never logged). /// Provider backbone used to compose the work queue name. /// Run id that owns messages to delete. - /// Cancels connect/get/settle/republish. + /// Cancels connect/get/settle. /// Maximum basic.get operations this call may perform. /// Count of owned messages discarded. public static async Task RemoveOwnedAsync( @@ -117,58 +118,17 @@ public static async Task RemoveOwnedAsync( await using var channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); - var removed = 0; - var visits = 0; - var parkedForeign = new List(); - while (visits < maxVisits) - { - cancellationToken.ThrowIfCancellationRequested(); - var got = await channel.BasicGetAsync(queueName, autoAck: false, cancellationToken) - .ConfigureAwait(false); - if (got is null) - { - break; - } - - visits++; - if (IsOwnedByRun(got.BasicProperties.Headers, stressRunId)) - { - await channel.BasicNackAsync( - got.DeliveryTag, - multiple: false, - requeue: false, - cancellationToken) - .ConfigureAwait(false); - removed++; - continue; - } - - // Park foreign so the scan can continue past them (reject+requeue spins on classic head). - parkedForeign.Add( - new ParkedForeignMessage( - CopyProperties(got.BasicProperties), - got.Body.ToArray())); - await channel.BasicAckAsync(got.DeliveryTag, multiple: false, cancellationToken) - .ConfigureAwait(false); - } - - foreach (var foreign in parkedForeign) - { - cancellationToken.ThrowIfCancellationRequested(); - await channel.BasicPublishAsync( - exchange: queueName, - routingKey: queueName, - mandatory: false, - basicProperties: foreign.Properties, - body: foreign.Body, - cancellationToken: cancellationToken) - .ConfigureAwait(false); - } + var removed = await DiscardOwnedHoldingForeignAsync( + new LiveCleanupChannel(channel, queueName), + stressRunId, + maxVisits, + cancellationToken) + .ConfigureAwait(false); - if (removed > 0 || parkedForeign.Count > 0) + if (removed > 0) { Console.WriteLine( - $"stress-owned-cleanup queue={queueName} runId={stressRunId} removed={removed} parkedForeign={parkedForeign.Count} visits={visits}"); + $"stress-owned-cleanup queue={queueName} runId={stressRunId} removed={removed}"); } return removed; @@ -181,39 +141,95 @@ await channel.BasicPublishAsync( } } - /// Copies settlement-relevant AMQP properties for a temporary foreign park/republish. - /// Properties from the gotten delivery. - /// A new suitable for republish. - internal static BasicProperties CopyProperties(IReadOnlyBasicProperties source) + /// + /// Scans ready deliveries: nack-discards owned messages; leaves foreign unacked. + /// Caller must dispose the channel so foreign deliveries are requeued by the broker. + /// + /// Cleanup channel (get + discard only; no foreign ack). + /// Owned run id. + /// Visit bound. + /// Cancels get/discard. + /// Owned messages discarded. + internal static async Task DiscardOwnedHoldingForeignAsync( + IStressCleanupChannel channel, + string stressRunId, + int maxVisits, + CancellationToken cancellationToken) { - ArgumentNullException.ThrowIfNull(source); - IDictionary? headers = null; - if (source.Headers is { Count: > 0 }) + ArgumentNullException.ThrowIfNull(channel); + ArgumentException.ThrowIfNullOrWhiteSpace(stressRunId); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxVisits); + + var removed = 0; + var visits = 0; + while (visits < maxVisits) { - headers = new Dictionary(source.Headers, StringComparer.Ordinal); + cancellationToken.ThrowIfCancellationRequested(); + var got = await channel.BasicGetAsync(cancellationToken).ConfigureAwait(false); + if (got is not { } delivery) + { + break; + } + + visits++; + if (IsOwnedByRun(delivery.Headers, stressRunId)) + { + await channel.BasicNackDiscardAsync(delivery.DeliveryTag, cancellationToken) + .ConfigureAwait(false); + removed++; + } + // Foreign / missing / malformed StressRunId: leave unacked for channel dispose requeue. } - return new BasicProperties - { - ContentType = source.ContentType, - ContentEncoding = source.ContentEncoding, - Headers = headers, - DeliveryMode = source.DeliveryMode, - Priority = source.Priority, - CorrelationId = source.CorrelationId, - ReplyTo = source.ReplyTo, - Expiration = source.Expiration, - MessageId = source.MessageId, - Timestamp = source.Timestamp, - Type = source.Type, - UserId = source.UserId, - AppId = source.AppId, - }; + return removed; + } + + /// One gotten delivery used by the cleanup scan (headers + delivery tag only). + /// Broker delivery tag for nack/discard. + /// AMQP headers; may be null. + internal readonly record struct StressCleanupDelivery( + ulong DeliveryTag, + IDictionary? Headers); + + /// + /// Minimal channel surface for owned discard while holding foreign unacked. + /// Implementations must not ack foreign deliveries. + /// + internal interface IStressCleanupChannel : IAsyncDisposable + { + /// Gets the next ready message, or null when the ready set is empty. + /// Cancels the get. + Task BasicGetAsync(CancellationToken cancellationToken); + + /// Discards one owned delivery without requeue. + /// Tag from a prior get. + /// Cancels the nack. + Task BasicNackDiscardAsync(ulong deliveryTag, CancellationToken cancellationToken); } - /// One foreign work message held in memory until the owned scan finishes. - /// AMQP properties to republish. - /// Payload octets to republish. - private readonly record struct ParkedForeignMessage(BasicProperties Properties, byte[] Body); + /// Live RabbitMQ.Client adapter for . + private sealed class LiveCleanupChannel(IChannel channel, string queueName) : IStressCleanupChannel + { + /// + public async Task BasicGetAsync(CancellationToken cancellationToken) + { + var got = await channel.BasicGetAsync(queueName, autoAck: false, cancellationToken) + .ConfigureAwait(false); + if (got is null) + { + return null; + } + + return new StressCleanupDelivery(got.DeliveryTag, got.BasicProperties.Headers); + } + + /// + public Task BasicNackDiscardAsync(ulong deliveryTag, CancellationToken cancellationToken) => + channel.BasicNackAsync(deliveryTag, multiple: false, requeue: false, cancellationToken) + .AsTask(); + + /// + public ValueTask DisposeAsync() => channel.DisposeAsync(); + } } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs index a48508b1..d36465c4 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressOwnedWorkMessageCleanupTests.cs @@ -1,20 +1,26 @@ using System.Text; +using static VectorNNTPD.BackFillerService.Tests.Stress.StressOwnedWorkMessageCleanup; namespace VectorNNTPD.BackFillerService.Tests.Stress { - /// Deterministic coverage for StressRunId header matching used by owned work-queue cleanup. + /// + /// Deterministic coverage for StressRunId matching and the hold-unacked foreign scan + /// (). + /// public sealed class StressOwnedWorkMessageCleanupTests { + private const string OwnedRunId = "owned-run"; + /// String header values match the owned run id. [Fact] public void IsOwnedByRun_matches_string_header() { var headers = new Dictionary { - [StressOwnedWorkMessageCleanup.StressRunIdHeader] = "run-a", + [StressRunIdHeader] = "run-a", }; - Assert.True(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-a")); - Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-b")); + Assert.True(IsOwnedByRun(headers, "run-a")); + Assert.False(IsOwnedByRun(headers, "run-b")); } /// UTF-8 byte[] headers (RabbitMQ.Client default) match the owned run id. @@ -23,45 +29,32 @@ public void IsOwnedByRun_matches_utf8_byte_header() { var headers = new Dictionary { - [StressOwnedWorkMessageCleanup.StressRunIdHeader] = Encoding.UTF8.GetBytes("run-bytes"), + [StressRunIdHeader] = Encoding.UTF8.GetBytes("run-bytes"), }; - Assert.True(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "run-bytes")); - Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(headers, "other")); + Assert.True(IsOwnedByRun(headers, "run-bytes")); + Assert.False(IsOwnedByRun(headers, "other")); } /// Missing, blank, empty-byte, and non-string header values are never treated as owned. [Fact] public void IsOwnedByRun_rejects_missing_blank_empty_bytes_and_unknown_types() { - Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(null, "run-a")); - Assert.False(StressOwnedWorkMessageCleanup.IsOwnedByRun(new Dictionary(), "run-a")); + Assert.False(IsOwnedByRun(null, "run-a")); + Assert.False(IsOwnedByRun(new Dictionary(), "run-a")); Assert.False( - StressOwnedWorkMessageCleanup.IsOwnedByRun( - new Dictionary - { - [StressOwnedWorkMessageCleanup.StressRunIdHeader] = " ", - }, + IsOwnedByRun( + new Dictionary { [StressRunIdHeader] = " " }, "run-a")); Assert.False( - StressOwnedWorkMessageCleanup.IsOwnedByRun( - new Dictionary - { - [StressOwnedWorkMessageCleanup.StressRunIdHeader] = Array.Empty(), - }, + IsOwnedByRun( + new Dictionary { [StressRunIdHeader] = Array.Empty() }, "run-a")); Assert.False( - StressOwnedWorkMessageCleanup.IsOwnedByRun( - new Dictionary - { - [StressOwnedWorkMessageCleanup.StressRunIdHeader] = 42, - }, + IsOwnedByRun( + new Dictionary { [StressRunIdHeader] = 42 }, "run-a")); Assert.Null( - StressOwnedWorkMessageCleanup.TryGetStressRunId( - new Dictionary - { - [StressOwnedWorkMessageCleanup.StressRunIdHeader] = 42, - })); + TryGetStressRunId(new Dictionary { [StressRunIdHeader] = 42 })); } /// TryGetStressRunId trims string values. @@ -70,29 +63,225 @@ public void TryGetStressRunId_trims_string() { var headers = new Dictionary { - [StressOwnedWorkMessageCleanup.StressRunIdHeader] = " padded-id ", + [StressRunIdHeader] = " padded-id ", }; - Assert.Equal("padded-id", StressOwnedWorkMessageCleanup.TryGetStressRunId(headers)); + Assert.Equal("padded-id", TryGetStressRunId(headers)); + } + + /// Owned deliveries are nack-discarded; foreign deliveries are never settled. + [Fact] + public async Task Scan_discards_owned_and_never_acks_foreign() + { + var channel = new FakeCleanupChannel( + Delivery(1, OwnedRunId), + Delivery(2, "foreign-run"), + Delivery(3, OwnedRunId)); + + var removed = await DiscardOwnedHoldingForeignAsync( + channel, + OwnedRunId, + maxVisits: 32, + CancellationToken.None); + + Assert.Equal(2, removed); + Assert.Equal(new ulong[] { 1, 3 }, channel.DiscardedTags); + Assert.Empty(channel.AckedTags); + Assert.Equal(new ulong[] { 2 }, channel.UnackedForeignTags); + Assert.Equal(0, channel.DisposeCount); } - /// CopyProperties preserves correlation, reply-to, and StressRunId for foreign republish. + /// Missing, blank, empty-byte, and unsupported headers are held as foreign (never discarded). [Fact] - public void CopyProperties_preserves_identity_and_stress_run_header() + public async Task Scan_treats_missing_blank_and_malformed_headers_as_foreign() { - var source = ArticleWorkStressClient.CreateRequestProperties( - correlationId: "corr-1", - replyTo: "reply-q", - requestId: Guid.Parse("11111111-1111-1111-1111-111111111111"), - stressRunId: "foreign-run"); - var copy = StressOwnedWorkMessageCleanup.CopyProperties(source); - Assert.Equal("corr-1", copy.CorrelationId); - Assert.Equal("reply-q", copy.ReplyTo); - Assert.Equal(source.ContentType, copy.ContentType); - Assert.Equal(source.DeliveryMode, copy.DeliveryMode); - Assert.Equal( - "foreign-run", - StressOwnedWorkMessageCleanup.TryGetStressRunId(copy.Headers)); - Assert.NotSame(source.Headers, copy.Headers); + var channel = new FakeCleanupChannel( + new StressCleanupDelivery(1, null), + new StressCleanupDelivery(2, new Dictionary()), + new StressCleanupDelivery(3, new Dictionary { [StressRunIdHeader] = " " }), + new StressCleanupDelivery(4, new Dictionary { [StressRunIdHeader] = Array.Empty() }), + new StressCleanupDelivery(5, new Dictionary { [StressRunIdHeader] = 99 }), + Delivery(6, OwnedRunId)); + + var removed = await DiscardOwnedHoldingForeignAsync( + channel, + OwnedRunId, + maxVisits: 32, + CancellationToken.None); + + Assert.Equal(1, removed); + Assert.Equal(new ulong[] { 6 }, channel.DiscardedTags); + Assert.Empty(channel.AckedTags); + Assert.Equal(new ulong[] { 1, 2, 3, 4, 5 }, channel.UnackedForeignTags); + } + + /// Owned messages behind foreign ready messages are still reached (no head-spin). + [Fact] + public async Task Scan_reaches_owned_messages_behind_foreign() + { + var channel = new FakeCleanupChannel( + Delivery(10, "foreign-a"), + Delivery(11, "foreign-b"), + Delivery(12, OwnedRunId)); + + var removed = await DiscardOwnedHoldingForeignAsync( + channel, + OwnedRunId, + maxVisits: 32, + CancellationToken.None); + + Assert.Equal(1, removed); + Assert.Equal(new ulong[] { 12 }, channel.DiscardedTags); + Assert.Equal(new ulong[] { 10, 11 }, channel.UnackedForeignTags); + Assert.Equal(3, channel.GetCount); + } + + /// Scan stops at maxVisits even when more deliveries remain. + [Fact] + public async Task Scan_stops_at_visit_limit() + { + var channel = new FakeCleanupChannel( + Delivery(1, "foreign"), + Delivery(2, OwnedRunId), + Delivery(3, OwnedRunId), + Delivery(4, OwnedRunId)); + + var removed = await DiscardOwnedHoldingForeignAsync( + channel, + OwnedRunId, + maxVisits: 2, + CancellationToken.None); + + Assert.Equal(1, removed); + Assert.Equal(2, channel.GetCount); + Assert.Equal(new ulong[] { 2 }, channel.DiscardedTags); + Assert.Equal(new ulong[] { 1 }, channel.UnackedForeignTags); + Assert.Equal(2, channel.RemainingReadyCount); + } + + /// + /// Callers that use await using on the cleanup channel dispose it after the scan, + /// including when the scan throws — modelling live channel dispose that requeues foreign. + /// + [Fact] + public async Task Channel_dispose_is_guaranteed_when_scan_throws() + { + var channel = new FakeCleanupChannel(Delivery(1, "foreign")) + { + ThrowOnGetAfter = 0, + }; + + await Assert.ThrowsAsync(async () => + { + await using (channel) + { + _ = await DiscardOwnedHoldingForeignAsync( + channel, + OwnedRunId, + maxVisits: 8, + CancellationToken.None); + } + }); + + Assert.Equal(1, channel.DisposeCount); + Assert.Empty(channel.DiscardedTags); + Assert.Empty(channel.AckedTags); + } + + /// Builds an owned/foreign delivery with a string StressRunId header. + private static StressCleanupDelivery Delivery(ulong tag, string stressRunId) => + new( + tag, + new Dictionary + { + [StressRunIdHeader] = stressRunId, + }); + + /// + /// In-memory cleanup channel: gets advance past unacked foreign holds; never invents acks for foreign. + /// + private sealed class FakeCleanupChannel : IStressCleanupChannel + { + private readonly Queue _ready; + private readonly List _unackedForeign = []; + private readonly List _discarded = []; + private readonly List _acked = []; + private int _gets; + + /// Creates a fake with the given ready deliveries in FIFO order. + /// Initial ready set. + public FakeCleanupChannel(params StressCleanupDelivery[] ready) + { + _ready = new Queue(ready); + } + + /// When non-negative, the get after this many successful gets throws. + public int ThrowOnGetAfter { get; init; } = -1; + + /// Successful get count. + public int GetCount => _gets; + + /// Ready messages not yet gotten. + public int RemainingReadyCount => _ready.Count; + + /// Delivery tags left unacked (foreign holds). + public IReadOnlyList UnackedForeignTags => _unackedForeign; + + /// Delivery tags discarded via nack requeue=false. + public IReadOnlyList DiscardedTags => _discarded; + + /// Delivery tags that were incorrectly acked (must stay empty in correct scans). + public IReadOnlyList AckedTags => _acked; + + /// Dispose invocations. + public int DisposeCount { get; private set; } + + /// + public Task BasicGetAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (ThrowOnGetAfter >= 0 && _gets >= ThrowOnGetAfter) + { + throw new InvalidOperationException("simulated get failure"); + } + + if (_ready.Count == 0) + { + return Task.FromResult(null); + } + + var next = _ready.Dequeue(); + _gets++; + // Model broker behaviour: gotten foreign stays out of ready until channel dispose requeues. + _unackedForeign.Add(next.DeliveryTag); + return Task.FromResult(next); + } + + /// + public Task BasicNackDiscardAsync(ulong deliveryTag, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!_unackedForeign.Remove(deliveryTag)) + { + throw new InvalidOperationException($"delivery {deliveryTag} was not unacked"); + } + + _discarded.Add(deliveryTag); + return Task.CompletedTask; + } + + /// + public ValueTask DisposeAsync() + { + DisposeCount++; + // Model broker requeue of remaining unacked foreign on channel close. + foreach (var tag in _unackedForeign) + { + _ready.Enqueue(new StressCleanupDelivery(tag, null)); + } + + _unackedForeign.Clear(); + return ValueTask.CompletedTask; + } } } }