From 34ac56cd67603e3a68d0cde5cd7d0bc37edeb23f Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 28 Oct 2026 08:32:50 +0200 Subject: [PATCH 1/8] Pipeline confirmed Article Work replies and add opt-in lab isolation controls Raise concurrent confirmed publications to 128 with aligned client throttling, preserve mandatory non-persistent replies, harden scratch release and SCM prune, point BackFiller RabbitMQ JSON at 198.18.0.3, and ship lab isolation/profiling disabled by default. --- docs/configuration.md | 2 + .../LabUpstreamDestinationRewriter.cs | 73 +++++ .../ProviderAccountConfigurationService.cs | 32 +- .../ArticleWorkDeliveryPipeline.cs | 278 ++++++++++-------- .../ArticleWorkResponsePublisher.cs | 55 +++- .../ArticleWork/ProviderArticleWorkHandler.cs | 5 + .../SilentConsumerMonitorEngine.cs | 10 +- .../Configuration/LabStageProfilingOptions.cs | 25 ++ .../LabStageProfilingOptionsValidator.cs | 42 +++ .../LabUpstreamIsolationOptions.cs | 52 ++++ .../LabUpstreamIsolationOptionsValidator.cs | 70 +++++ .../Diagnostics/ArticleWorkStageId.cs | 94 ++++++ .../Diagnostics/ArticleWorkStageProfiler.cs | 252 ++++++++++++++++ .../ArticleWorkStageProfilerJsonContext.cs | 17 ++ .../ArticleWorkStageProfilingHostedService.cs | 133 +++++++++ .../Diagnostics/StageLatencyHistogram.cs | 164 +++++++++++ .../BackFillerServiceCollectionExtensions.cs | 40 ++- ...abUpstreamAllowlistNntpTransportFactory.cs | 117 ++++++++ .../Nntp/NntpArticleRetriever.cs | 9 + .../Nntp/NntpProtocolIo.cs | 48 ++- .../Nntp/NntpProviderSession.cs | 28 +- .../Retention/ArticleRetentionMetrics.cs | 92 ++++++ .../VectorNNTPD.RabbitMQ.json | 6 +- .../RabbitMq/RabbitMqClientConnection.cs | 6 +- .../RabbitMqPublisherConfirmLimits.cs | 20 ++ .../LabUpstreamDestinationRewriterTests.cs | 137 +++++++++ ...roviderAccountConfigurationServiceTests.cs | 107 ++++++- .../ArticleWorkResponsePublicationTests.cs | 89 ++++++ .../SilentConsumerMultiInstanceTests.cs | 43 +++ .../Diagnostics/StageLatencyHistogramTests.cs | 180 ++++++++++++ .../Hosting/BackFillerHostCompositionTests.cs | 57 +++- .../LabUpstreamIsolationCompositionTests.cs | 241 +++++++++++++++ ...treamAllowlistNntpTransportFactoryTests.cs | 266 +++++++++++++++++ .../Nntp/NntpPayloadScratchPoolTests.cs | 99 ++++--- .../Retention/ArticleRetentionMetricsTests.cs | 241 +++++++++++++++ .../TestDoubles/FakeBackFillerRabbitMq.cs | 90 ++++-- .../Articles/CanonicalArticleFramingTests.cs | 49 +++ 37 files changed, 3009 insertions(+), 260 deletions(-) create mode 100644 src/VectorNNTPD.BackFillerService/Accounts/LabUpstreamDestinationRewriter.cs create mode 100644 src/VectorNNTPD.BackFillerService/Configuration/LabStageProfilingOptions.cs create mode 100644 src/VectorNNTPD.BackFillerService/Configuration/LabStageProfilingOptionsValidator.cs create mode 100644 src/VectorNNTPD.BackFillerService/Configuration/LabUpstreamIsolationOptions.cs create mode 100644 src/VectorNNTPD.BackFillerService/Configuration/LabUpstreamIsolationOptionsValidator.cs create mode 100644 src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageId.cs create mode 100644 src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfiler.cs create mode 100644 src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilerJsonContext.cs create mode 100644 src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilingHostedService.cs create mode 100644 src/VectorNNTPD.BackFillerService/Diagnostics/StageLatencyHistogram.cs create mode 100644 src/VectorNNTPD.BackFillerService/Nntp/LabUpstreamAllowlistNntpTransportFactory.cs create mode 100644 src/VectorNNTPD.BackFillerService/Retention/ArticleRetentionMetrics.cs create mode 100644 src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqPublisherConfirmLimits.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Accounts/LabUpstreamDestinationRewriterTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Hosting/LabUpstreamIsolationCompositionTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Nntp/LabUpstreamAllowlistNntpTransportFactoryTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Retention/ArticleRetentionMetricsTests.cs diff --git a/docs/configuration.md b/docs/configuration.md index 95109e24..c56e2b6b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -59,6 +59,8 @@ Nested groups: - `BackFiller:TcpKeepAlive` — TCP `SO_KEEPALIVE` for every upstream NNTP socket and every accepted cache-listener socket. `TimeSeconds` default 10 (range 1–7200) is idle time before the first probe. `IntervalSeconds` default 5 (range 1–300) is the gap between probes. `RetryCount` default 3 (range 1–20) is unanswered probes before the stack drops the connection. At the defaults, dead-path detection completes in about 25 s, before the default NNTP command timeout (30 s) and well before the listener I/O-progress timeout (60 s). Independent of per-provider MySQL `keepalive` (NNTP `DATE`). Nagle and buffer settings are unchanged. Platforms that cannot set a probe option still enable `SO_KEEPALIVE` and keep OS defaults for that option; enabling keepalive itself must succeed or the socket is rejected. - `BackFiller:Systemd` — notify and watchdog enablement - `BackFiller:SilentConsumerMonitor` — optional. Only `Enabled` (default true) is public. Do not place VirtualHost, Hosts, Management URL, or queue lists here. AMQP hosts/vhost/port and Management BaseUrl bind from the `RabbitMQ` section; username/password are projected from `nntpsharedconfig` onto live `RabbitMqOptions` on account refresh (Management uses them on the next HTTP call; AMQP uses them on the next reconnect dial). Article Work queues come from the MySQL provider catalog as `backfiller.{backbone}`. Detection thresholds are internal code defaults (`ProcessingStall` / `DispatchStall` / `ConsumerLoss`). Management poll failures yield `Unknown` and do not resolve open shared alerts; instance-local monitoring continues. See `docs/architecture.md` § Silent-consumer monitoring (including Management outages and `Unknown`) for the traced data flow and multi-instance semantics. Monitor failures never disable consumption. +- `BackFiller:LabStageProfiling` — lab-only Article Work stage latency histograms. Default `Enabled` false (no ambient profiler installed; production path unchanged). When `Enabled` is true, installs bounded per-stage histograms (pipeline, handler, session acquire, busy wait, ARTICLE command/status, payload receive, parse, retain, reply publish, plus reply serialize / publish-gate wait / gate-held confirm) and optionally writes `article-work-stage-profile.json` under `ExportDirectory` every `ExportIntervalSeconds` (default 2). Nested stages overlap — do not sum totals as exclusive wall time. Env: `BackFiller__LabStageProfiling__Enabled`, `BackFiller__LabStageProfiling__ExportDirectory`, `BackFiller__LabStageProfiling__ExportIntervalSeconds`. +- `BackFiller:LabUpstreamIsolation` — lab-only. Default `Enabled` false (ordinary deployments unchanged). When `Enabled` is true, every upstream NNTP dial through `INntpTransportFactory` must target `AllowedHost`:`AllowedPort` before a socket opens; otherwise connect fails closed with no unguarded fallback. `AllowedHost` must be exactly `127.0.0.1` (no `localhost`, DNS names, IPv6, or non-loopback). `AllowedPort` is required (1–65535) when enabled. `RejectTls` default true rejects provider `UseTls=true`. `RewritePublishedDestinations` default false; when true (requires `Enabled` and `RejectTls`), each account refresh rewrites **published** provider dial fields to `AllowedHost`:`AllowedPort` with `UseTls=false` and clears username/password so AUTHINFO is not sent — MySQL rows are **not** updated, and production dial behaviour is unchanged when rewrite is off. Incomplete or inconsistent enabled/rewrite configuration fails `ValidateOnStart`. Env examples: `BackFiller__LabUpstreamIsolation__Enabled`, `BackFiller__LabUpstreamIsolation__AllowedPort`, `BackFiller__LabUpstreamIsolation__RewritePublishedDestinations`. ## RabbitMQ section diff --git a/src/VectorNNTPD.BackFillerService/Accounts/LabUpstreamDestinationRewriter.cs b/src/VectorNNTPD.BackFillerService/Accounts/LabUpstreamDestinationRewriter.cs new file mode 100644 index 00000000..cdac1ed1 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Accounts/LabUpstreamDestinationRewriter.cs @@ -0,0 +1,73 @@ +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Nntp; + +namespace VectorNNTPD.BackFillerService.Accounts +{ + /// + /// Lab-only rewriter that replaces published provider dial targets with the isolation allowlist + /// endpoint and clears credentials so AUTHINFO is not attempted against the local peer. + /// + /// + /// Does not mutate MySQL rows. Applied after on each refresh + /// when is true. + /// Production behaviour is unchanged when that flag is false. + /// + internal static class LabUpstreamDestinationRewriter + { + /// + /// Returns unchanged when rewrite is disabled; otherwise every + /// definition is retargeted to the allowlisted cleartext loopback endpoint without credentials. + /// + /// Mapped providers from the account source. + /// Lab isolation options already validated at startup when rewrite is enabled. + /// Effective definitions used for pool publication. + /// An argument is null. + /// + /// Rewrite is enabled but isolation is incomplete (should have failed ValidateOnStart). + /// + internal static IReadOnlyList Apply( + IReadOnlyList providers, + LabUpstreamIsolationOptions options) + { + ArgumentNullException.ThrowIfNull(providers); + ArgumentNullException.ThrowIfNull(options); + + if (!options.RewritePublishedDestinations) + { + return providers; + } + + if (!options.Enabled) + { + throw new InvalidOperationException( + "LabUpstreamIsolation RewritePublishedDestinations requires Enabled=true."); + } + + if (options.AllowedPort is null or < 1 or > 65535) + { + throw new InvalidOperationException( + "LabUpstreamIsolation AllowedPort must be in 1..65535 when RewritePublishedDestinations is true."); + } + + var host = LabUpstreamIsolationOptions.RequiredLiteralHost; + var port = options.AllowedPort.Value; + var rewritten = new BackFillerProviderDefinition[providers.Count]; + for (var i = 0; i < providers.Count; i++) + { + var source = providers[i]; + rewritten[i] = new BackFillerProviderDefinition( + source.Backbone, + host, + port, + UseTls: false, + Username: null, + Password: null, + source.MinSessions, + source.MaxSessions, + source.KeepAliveSeconds); + } + + return rewritten; + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Accounts/ProviderAccountConfigurationService.cs b/src/VectorNNTPD.BackFillerService/Accounts/ProviderAccountConfigurationService.cs index 8b758c79..b4a40bc9 100644 --- a/src/VectorNNTPD.BackFillerService/Accounts/ProviderAccountConfigurationService.cs +++ b/src/VectorNNTPD.BackFillerService/Accounts/ProviderAccountConfigurationService.cs @@ -43,6 +43,11 @@ internal sealed class ProviderAccountConfigurationService : IHostedService, INnt /// Live RabbitMQ options. Username and password are replaced from each published snapshot. private readonly IOptions _rabbitMq; + /// + /// Lab isolation options. When rewrite is enabled, published dial targets are replaced after mapping. + /// + private readonly IOptions _labUpstreamIsolation; + /// Control-plane logger. Events are written through . private readonly ILogger _logger; @@ -91,6 +96,9 @@ internal sealed class ProviderAccountConfigurationService : IHostedService, INnt /// Live ACME options updated from each shared snapshot. /// Logger passed to . /// Live RabbitMQ options. Username and password are replaced from each shared snapshot. + /// + /// Lab isolation options. Optional in tests; defaults to disabled rewrite when omitted. + /// /// Any argument is null. /// /// The live is owned by . @@ -104,7 +112,8 @@ internal ProviderAccountConfigurationService( IOptions backFillerOptions, IOptions acmeOptions, ILogger logger, - IOptions? rabbitMqOptions = null) + IOptions? rabbitMqOptions = null, + IOptions? labUpstreamIsolation = null) { ArgumentNullException.ThrowIfNull(source); ArgumentNullException.ThrowIfNull(sharedConfiguration); @@ -118,6 +127,7 @@ internal ProviderAccountConfigurationService( _registry = registry; _runtime = runtime; _backFillerOptions = backFillerOptions; + _labUpstreamIsolation = labUpstreamIsolation ?? Options.Create(new LabUpstreamIsolationOptions()); _acmeOptions = acmeOptions; _rabbitMq = rabbitMqOptions ?? Options.Create(new RabbitMqOptions()); _logger = logger; @@ -278,13 +288,19 @@ private async Task RefreshAsync(bool required, CancellationToken cancellat ProviderAccountLogMessages.RowRejected(_logger, rejected.Backbone, rejected.Reason); } + // Lab rewrite runs after mapping so pools never publish WAN dial targets while enabled. + // MySQL rows are not updated; credentials are cleared so AUTHINFO is not sent to the peer. + var effectiveProviders = LabUpstreamDestinationRewriter.Apply( + mapped.Providers, + _labUpstreamIsolation.Value); + IReadOnlyList previous; lock (_gate) { previous = _published; } - if (SnapshotsEqual(previous, mapped.Providers)) + if (SnapshotsEqual(previous, effectiveProviders)) { lock (_gate) { @@ -292,10 +308,10 @@ private async Task RefreshAsync(bool required, CancellationToken cancellat _shared = shared; } - ProviderAccountLogMessages.SnapshotUnchanged(_logger, mapped.Providers.Count); + ProviderAccountLogMessages.SnapshotUnchanged(_logger, effectiveProviders.Count); if (_lastRefreshFailed) { - ProviderAccountLogMessages.RefreshRecovered(_logger, mapped.Providers.Count); + ProviderAccountLogMessages.RefreshRecovered(_logger, effectiveProviders.Count); } _lastRefreshFailed = false; @@ -303,19 +319,19 @@ private async Task RefreshAsync(bool required, CancellationToken cancellat return true; } - await _registry.ApplySnapshotAsync(mapped.Providers, cancellationToken).ConfigureAwait(false); + await _registry.ApplySnapshotAsync(effectiveProviders, cancellationToken).ConfigureAwait(false); lock (_gate) { Project(shared, fqdn); - _published = mapped.Providers; + _published = effectiveProviders; _shared = shared; } - LogDelta(previous, mapped.Providers); + LogDelta(previous, effectiveProviders); ProviderAccountLogMessages.SnapshotLoaded( _logger, _runtime.ServerId, - mapped.Providers.Count, + effectiveProviders.Count, mapped.Rejected.Count); if (_lastRefreshFailed) { diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs index 1da7fbb9..22232767 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs @@ -1,4 +1,5 @@ - +using System.Diagnostics; +using VectorNNTPD.BackFillerService.Diagnostics; using VectorNNTPD.BackFillerService.Nntp; using VectorNNTPD.Common.Messaging.RabbitMq; using VectorNNTPD.Common.Core; @@ -145,153 +146,172 @@ internal async Task ProcessAsync( LastEscapedHandlerFailureError = null; ArgumentNullException.ThrowIfNull(channelStillCurrent); - var parsed = preParsed ?? ArticleWorkRequestParser.Parse(delivery, consumingBackbone, _maxPayloadBytes); - var lease = new ArticleWorkSettlementLease(channel, delivery.DeliveryTag, delivery.Generation); - var replyable = !string.IsNullOrWhiteSpace(delivery.CorrelationId) - && !string.IsNullOrWhiteSpace(delivery.ReplyTo); - - if (!parsed.IsValid) + var pipelineStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; + var profiler = ArticleWorkStageProfiler.Installed; + profiler?.BeginActive(); + var pipelineFailed = false; + try { - var failure = parsed.Failure!; - var invalid = ArticleWorkDispositionPlanner.Create( - ArticleWorkOutcome.InvalidRequest, - replyable, - cancellationRequested: false, - delivery.DeliveryCount, - delivery.Redelivered, - _deliveryLimit, - _treatMissingRedeliveryCountAsExhausted); - if (!channelStillCurrent() || !lease.IsOriginalChannel(channel)) - { - return ArticleWorkOutcome.InvalidRequest; - } + var parsed = preParsed ?? ArticleWorkRequestParser.Parse(delivery, consumingBackbone, _maxPayloadBytes); + var lease = new ArticleWorkSettlementLease(channel, delivery.DeliveryTag, delivery.Generation); + var replyable = !string.IsNullOrWhiteSpace(delivery.CorrelationId) + && !string.IsNullOrWhiteSpace(delivery.ReplyTo); - var invalidPublished = await TryPublishIfRequiredAsync( - invalid, - ArticleWorkOutcome.InvalidRequest, - failure.Identities.RequestId, - failure.Identities.MessageId, - failure.Identities.Backbone, - delivery.CorrelationId, - delivery.ReplyTo, - failure.Reason, - fqdn: null, - listenerPort: null, - articleIdHex: null, - cancellationToken) - .ConfigureAwait(false); - var publishOutcome = await CompleteAfterPublishAttemptAsync( - invalidPublished, - invalid, + if (!parsed.IsValid) + { + pipelineFailed = true; + var failure = parsed.Failure!; + var invalid = ArticleWorkDispositionPlanner.Create( ArticleWorkOutcome.InvalidRequest, - lease, - channel, - channelStillCurrent, + replyable, + cancellationRequested: false, delivery.DeliveryCount, - delivery.Redelivered) - .ConfigureAwait(false); - return publishOutcome; - } + delivery.Redelivered, + _deliveryLimit, + _treatMissingRedeliveryCountAsExhausted); + if (!channelStillCurrent() || !lease.IsOriginalChannel(channel)) + { + return ArticleWorkOutcome.InvalidRequest; + } + + var replyStartedInvalid = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; + var invalidPublished = await TryPublishIfRequiredAsync( + invalid, + ArticleWorkOutcome.InvalidRequest, + failure.Identities.RequestId, + failure.Identities.MessageId, + failure.Identities.Backbone, + delivery.CorrelationId, + delivery.ReplyTo, + failure.Reason, + fqdn: null, + listenerPort: null, + articleIdHex: null, + cancellationToken) + .ConfigureAwait(false); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.ReplyPublish, replyStartedInvalid); + var publishOutcome = await CompleteAfterPublishAttemptAsync( + invalidPublished, + invalid, + ArticleWorkOutcome.InvalidRequest, + lease, + channel, + channelStillCurrent, + delivery.DeliveryCount, + delivery.Redelivered) + .ConfigureAwait(false); + return publishOutcome; + } - var item = new ArticleWorkItem( - parsed.Request!, - delivery.CorrelationId!, - delivery.ReplyTo!, - lease); - - ArticleWorkOutcome outcome; - string? error; - string? fqdn = null; - int? listenerPort = null; - string? articleIdHex = null; - try - { - if (cancellationToken.IsCancellationRequested) + var item = new ArticleWorkItem( + parsed.Request!, + delivery.CorrelationId!, + delivery.ReplyTo!, + lease); + + ArticleWorkOutcome outcome; + string? error; + string? fqdn = null; + int? listenerPort = null; + string? articleIdHex = null; + try + { + if (cancellationToken.IsCancellationRequested) + { + outcome = ArticleWorkOutcome.Cancelled; + error = null; + } + else + { + var result = await _handler.HandleAsync(item, cancellationToken).ConfigureAwait(false); + outcome = result.Outcome == ArticleWorkOutcome.InvalidRequest + ? ArticleWorkOutcome.UnexpectedFailure + : result.Outcome; + error = result.Error; + fqdn = result.Fqdn; + listenerPort = result.ListenerPort; + articleIdHex = result.ArticleId?.ToLowerHexString(); + result.Article?.Dispose(); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { outcome = ArticleWorkOutcome.Cancelled; error = null; } - else + // codeql[cs/catch-of-all-exceptions] A delivery failure must be settled or logged by this pipeline boundary. + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { - var result = await _handler.HandleAsync(item, cancellationToken).ConfigureAwait(false); - outcome = result.Outcome == ArticleWorkOutcome.InvalidRequest - ? ArticleWorkOutcome.UnexpectedFailure - : result.Outcome; - error = result.Error; - fqdn = result.Fqdn; - listenerPort = result.ListenerPort; - articleIdHex = result.ArticleId?.ToLowerHexString(); - result.Article?.Dispose(); + outcome = ArticleWorkOutcome.UnexpectedFailure; + error = NntpExceptionDiagnostics.FormatReason(ex); + LastEscapedHandlerFailureError = error; } - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - outcome = ArticleWorkOutcome.Cancelled; - error = null; - } - // codeql[cs/catch-of-all-exceptions] A delivery failure must be settled or logged by this pipeline boundary. - catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) - { - outcome = ArticleWorkOutcome.UnexpectedFailure; - error = NntpExceptionDiagnostics.FormatReason(ex); - LastEscapedHandlerFailureError = error; - } - var disposition = ArticleWorkDispositionPlanner.Create( - outcome, - replyable: true, - cancellationToken.IsCancellationRequested || outcome == ArticleWorkOutcome.Cancelled, - delivery.DeliveryCount, - delivery.Redelivered, - _deliveryLimit, - _treatMissingRedeliveryCountAsExhausted); + pipelineFailed = outcome is not ArticleWorkOutcome.Success and not ArticleWorkOutcome.Cancelled; - if (!channelStillCurrent() || !lease.IsOriginalChannel(channel)) - { - return outcome; - } + var disposition = ArticleWorkDispositionPlanner.Create( + outcome, + replyable: true, + cancellationToken.IsCancellationRequested || outcome == ArticleWorkOutcome.Cancelled, + delivery.DeliveryCount, + delivery.Redelivered, + _deliveryLimit, + _treatMissingRedeliveryCountAsExhausted); - if (outcome == ArticleWorkOutcome.Success && !_publisher.CompletesSuccessPublication) - { - var pending = CreateRetryDisposition(delivery.DeliveryCount, delivery.Redelivered); - await lease.TrySettleAsync( - pending, - channelStillCurrent() && lease.IsOriginalChannel(channel), - CancellationToken.None) - .ConfigureAwait(false); - return outcome; - } + if (!channelStillCurrent() || !lease.IsOriginalChannel(channel)) + { + return outcome; + } - var published = ArticleWorkPublishAttempt.Confirmed; - if (disposition.PublishResponse) - { - published = await TryPublishIfRequiredAsync( + if (outcome == ArticleWorkOutcome.Success && !_publisher.CompletesSuccessPublication) + { + var pending = CreateRetryDisposition(delivery.DeliveryCount, delivery.Redelivered); + await lease.TrySettleAsync( + pending, + channelStillCurrent() && lease.IsOriginalChannel(channel), + CancellationToken.None) + .ConfigureAwait(false); + return outcome; + } + + var published = ArticleWorkPublishAttempt.Confirmed; + if (disposition.PublishResponse) + { + var replyStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; + published = await TryPublishIfRequiredAsync( + disposition, + outcome, + item.Request.RequestId, + item.Request.MessageId, + item.Request.Backbone, + item.CorrelationId, + item.ReplyTo, + error, + fqdn, + listenerPort, + articleIdHex, + cancellationToken) + .ConfigureAwait(false); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.ReplyPublish, replyStarted); + } + + ArticleWorkOutcome finalOutcome = await CompleteAfterPublishAttemptAsync( + published, disposition, outcome, - item.Request.RequestId, - item.Request.MessageId, - item.Request.Backbone, - item.CorrelationId, - item.ReplyTo, - error, - fqdn, - listenerPort, - articleIdHex, - cancellationToken) + lease, + channel, + channelStillCurrent, + delivery.DeliveryCount, + delivery.Redelivered) .ConfigureAwait(false); + return finalOutcome; + } + finally + { + profiler?.Record(ArticleWorkStageId.PipelineTotal, pipelineStarted); + profiler?.EndActive(pipelineFailed); } - - ArticleWorkOutcome finalOutcome = await CompleteAfterPublishAttemptAsync( - published, - disposition, - outcome, - lease, - channel, - channelStillCurrent, - delivery.DeliveryCount, - delivery.Redelivered) - .ConfigureAwait(false); - return finalOutcome; } /// NACK with requeue while retries remain; NACK without requeue when the delivery limit is exhausted. diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs index d42d79bb..91b37a7e 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs @@ -1,4 +1,6 @@ +using System.Diagnostics; using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Diagnostics; using VectorNNTPD.BackFillerService.Logging; using VectorNNTPD.Common.Core; using VectorNNTPD.Common.Messaging.RabbitMq; @@ -15,10 +17,22 @@ namespace VectorNNTPD.BackFillerService.ArticleWork /// and never ACK/NACKs deliveries. /// Completing means the broker confirmed and the publication generation was still current afterward. /// Confirmation is not permission to ACK the original delivery. - /// One publication holds . The confirmation wait is cancelled by the caller token, disposal, or the configured confirmation timeout. + /// Up to publications may hold while overlapping + /// broker confirmation waits on the shared tracking-enabled channel. The confirmation wait is cancelled by the caller token, + /// disposal, or the configured confirmation timeout. /// internal sealed class ArticleWorkResponsePublisher : IArticleWorkResponsePublisher, IHostedService, IAsyncDisposable { + /// + /// Maximum concurrent confirmed publications on the shared channel. + /// Matches used when opening + /// the confirm-tracked publish channel. The client serializes publish-sequence assignment and frame send, + /// then overlaps per-message confirm waits; this bound prevents unbounded app-level in-flight confirms + /// and enables dispose drain. + /// + internal const int MaxConcurrentConfirmedPublications = + RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations; + /// Process RabbitMQ connection owner. This publisher opens channels on it and does not open another connection. private readonly IRabbitMqService _connections; @@ -31,8 +45,13 @@ internal sealed class ArticleWorkResponsePublisher : IArticleWorkResponsePublish /// Guards the publication channel, lifecycle state, the stopping flag, and the replacement task. private readonly Lock _gate = new(); - /// Serializes confirmed publications. Disposal waits until the current holder releases it. - private readonly SemaphoreSlim _publishGate = new(1, 1); + /// + /// Bounds concurrent confirmed publications to . + /// Disposal acquires every permit before disposing the semaphore so in-flight publishes have finished or cancelled. + /// + private readonly SemaphoreSlim _publishGate = new( + MaxConcurrentConfirmedPublications, + MaxConcurrentConfirmedPublications); /// Serializes startup, channel replacement, and disposal of the publication channel. private readonly SemaphoreSlim _replaceGate = new(1, 1); @@ -197,7 +216,7 @@ public async Task StopAsync(CancellationToken cancellationToken) /// /// Waits for the tracked replacement task and logs its failure without rethrowing. /// The replacement-gate wait is not cancellable. After the channel is disposed, disposal waits up to the confirmation timeout - /// for and then releases it. A timeout is ignored, so disposal still finishes. + /// to acquire every permit (drain). A timeout is ignored, so disposal still finishes. /// The stopped log reads after the channel is cleared, so that value is zero. /// public async ValueTask DisposeAsync() @@ -284,7 +303,9 @@ private void DisposeShutdownHandles() /// That exception is not written as a publication-failed log. /// /// - /// Publications on this instance are serialized by . + /// Publications on this instance share one confirm-tracked channel. Up to + /// may await confirmation concurrently; + /// bounds that depth (it does not serialize confirm waits to one). /// The AMQP message id is a new GUID, not the request id. App id is null. Delivery is non-persistent and mandatory. /// The routing key is and the exchange is empty. /// Content type, expiration, and body come from . @@ -300,12 +321,19 @@ public async Task PublishAsync(ArticleWorkResponseIntent intent, CancellationTok throw new InvalidOperationException("Response publication requires CorrelationId and ReplyTo."); } + var serializeStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; var body = ArticleWorkResponseWireProtocol.SerializeV1(intent); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.ReplySerialize, serializeStarted); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _runCts.Token); linked.CancelAfter(TimeSpan.FromSeconds(_runtime.RabbitMq.PublishConfirmTimeoutSeconds)); + var gateWaitStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; await _publishGate.WaitAsync(linked.Token).ConfigureAwait(false); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.ReplyPublishGateWait, gateWaitStarted); + var generation = 0L; + var confirmStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; try { ThrowIfNotRunning(); @@ -326,6 +354,8 @@ public async Task PublishAsync(ArticleWorkResponseIntent intent, CancellationTok ArticleWorkResponseWireProtocol.ExpirationMilliseconds, body); + // RabbitMQ.Client 7.x with publisherConfirmationTrackingEnabled: BasicPublishAsync + // completes only after broker confirm (or return). Confirm wait is not separable. await channel.PublishConfirmedAsync( new RabbitMqConfirmedPublication( Exchange: string.Empty, @@ -388,6 +418,7 @@ await channel.PublishConfirmedAsync( } finally { + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.ReplyPublishConfirm, confirmStarted); _publishGate.Release(); } } @@ -678,20 +709,24 @@ private void DetachPublishChannelFaultHandler() } /// - /// Waits until can be acquired, then releases it so disposal can dispose of the semaphore. + /// Acquires every permit so no confirmed publish still holds a slot, then leaves them held + /// until disposes the semaphore. /// /// - /// A task that completes when the gate has been observed free. The wait is bounded by the confirmation timeout, with a minimum of one second. - /// from that timeout is swallowed. + /// A task that completes when all permits are held or the confirmation-timeout bound elapses. + /// from that timeout is swallowed; any permits acquired are not released. /// private async Task WaitForPublishGateReleaseAsync() { using var waitCts = new CancellationTokenSource( TimeSpan.FromSeconds(Math.Max(1, _runtime.RabbitMq.PublishConfirmTimeoutSeconds))); + var acquired = 0; try { - await _publishGate.WaitAsync(waitCts.Token).ConfigureAwait(false); - _publishGate.Release(); + for (; acquired < MaxConcurrentConfirmedPublications; acquired++) + { + await _publishGate.WaitAsync(waitCts.Token).ConfigureAwait(false); + } } catch (OperationCanceledException) { diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ProviderArticleWorkHandler.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ProviderArticleWorkHandler.cs index 969fbfa9..67a26333 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ProviderArticleWorkHandler.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ProviderArticleWorkHandler.cs @@ -3,6 +3,7 @@ using System.Globalization; using Microsoft.Extensions.Logging.Abstractions; using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Diagnostics; using VectorNNTPD.BackFillerService.Nntp; using VectorNNTPD.Common.NntpDb; using VectorNNTPD.BackFillerService.Retention; @@ -303,6 +304,7 @@ public async ValueTask HandleAsync( endpointFqdn = prepared.EndpointFqdn; } + var handlerStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; try { try @@ -446,12 +448,14 @@ public async ValueTask HandleAsync( "Canonical ArtData is not an owned contiguous buffer."); } + var retainStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; var retained = _retention.RetainCanonical( item.Request.MessageId, item.Request.RequestId, record, created.SelectedDateHeaderName, endpointFqdn); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.Retain, retainStarted); LastRetentionKind = retained.Kind; LastFqdn = retained.Fqdn; LastListenerPort = retained.ListenerPort; @@ -477,6 +481,7 @@ public async ValueTask HandleAsync( } finally { + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.HandlerTotal, handlerStarted); if (created.IsAccepted) { created.Record.ReleaseUnusedCanonicalBuffer(); diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs index f24805ac..cde8ebdf 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/SilentConsumerMonitor/SilentConsumerMonitorEngine.cs @@ -132,6 +132,8 @@ public IReadOnlyList ObserveInstance( } // Drive Inactive for session keys that left the live session set (sibling keys untouched). + // After resolve (or when already Clear), drop the machine so catalog/MaxSessions churn cannot + // retain ObservationStateMachine entries for the process lifetime. foreach (var key in _machines.Keys.ToArray()) { if (key.Scope != ObservationScope.Instance @@ -146,7 +148,13 @@ public IReadOnlyList ObserveInstance( continue; } - results.Add(Apply(key, ConditionSignal.Inactive, "session removed", now)); + var removed = Apply(key, ConditionSignal.Inactive, "session removed", now); + results.Add(removed); + if (removed.Transition == ObservationTransition.Resolved + || removed.Phase == ObservationPhase.Clear) + { + RemoveMachine(key); + } } results.Add( diff --git a/src/VectorNNTPD.BackFillerService/Configuration/LabStageProfilingOptions.cs b/src/VectorNNTPD.BackFillerService/Configuration/LabStageProfilingOptions.cs new file mode 100644 index 00000000..11fc117e --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Configuration/LabStageProfilingOptions.cs @@ -0,0 +1,25 @@ +namespace VectorNNTPD.BackFillerService.Configuration +{ + /// + /// Lab-only Article Work stage profiler under BackFiller:LabStageProfiling. + /// Default is false — production behaviour unchanged. + /// + internal sealed class LabStageProfilingOptions + { + /// Relative section name under . + internal const string SectionRelativeName = "LabStageProfiling"; + + /// + /// When true, installs and records stage histograms. + /// + public bool Enabled { get; set; } + + /// + /// Optional directory for periodic JSON snapshots. Empty disables file export (profiler still records in-memory). + /// + public string? ExportDirectory { get; set; } + + /// Snapshot period when is set. Default 2 seconds. + public int ExportIntervalSeconds { get; set; } = 2; + } +} diff --git a/src/VectorNNTPD.BackFillerService/Configuration/LabStageProfilingOptionsValidator.cs b/src/VectorNNTPD.BackFillerService/Configuration/LabStageProfilingOptionsValidator.cs new file mode 100644 index 00000000..a7938898 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Configuration/LabStageProfilingOptionsValidator.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.BackFillerService.Configuration +{ + /// Validates at host start. + internal sealed class LabStageProfilingOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, LabStageProfilingOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (!options.Enabled) + { + return ValidateOptionsResult.Success; + } + + if (options.ExportIntervalSeconds is < 1 or > 3600) + { + return ValidateOptionsResult.Fail( + "BackFiller:LabStageProfiling:ExportIntervalSeconds must be between 1 and 3600 when Enabled."); + } + + if (!string.IsNullOrWhiteSpace(options.ExportDirectory)) + { + try + { + _ = Path.GetFullPath(options.ExportDirectory); + } + catch (Exception ex) when (ex is ArgumentException + or NotSupportedException + or PathTooLongException + or System.Security.SecurityException) + { + return ValidateOptionsResult.Fail( + "BackFiller:LabStageProfiling:ExportDirectory is not a usable path: " + ex.Message); + } + } + + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Configuration/LabUpstreamIsolationOptions.cs b/src/VectorNNTPD.BackFillerService/Configuration/LabUpstreamIsolationOptions.cs new file mode 100644 index 00000000..6646fbc8 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Configuration/LabUpstreamIsolationOptions.cs @@ -0,0 +1,52 @@ +namespace VectorNNTPD.BackFillerService.Configuration +{ + /// + /// Lab-only fail-closed upstream dial allowlist under BackFiller:LabUpstreamIsolation. + /// + /// + /// Default is so ordinary deployments are unchanged. + /// When enabled, every must target + /// : before a socket is opened. + /// When is also true, each refresh rewrites published + /// provider dial fields to that allowlist endpoint and clears credentials (no MySQL row updates). + /// + internal sealed class LabUpstreamIsolationOptions + { + /// Relative section name under . + internal const string SectionRelativeName = "LabUpstreamIsolation"; + + /// Only accepted host when isolation is enabled (literal IPv4; no DNS). + internal const string RequiredLiteralHost = "127.0.0.1"; + + /// + /// Gets or sets whether the lab upstream allowlist is active. + /// Default — production stays unguarded. + /// + public bool Enabled { get; set; } + + /// + /// Gets or sets the only permitted upstream host when is true. + /// Must be exactly . + /// + public string AllowedHost { get; set; } = RequiredLiteralHost; + + /// + /// Gets or sets the only permitted upstream TCP port when is true. + /// Required (1–65535) when enabled; ignored when disabled. + /// + public int? AllowedPort { get; set; } + + /// + /// Gets or sets whether providers with UseTls=true are rejected when enabled. + /// Default — the local ARTICLE peer is cleartext. + /// + public bool RejectTls { get; set; } = true; + + /// + /// Gets or sets whether each account refresh rewrites published provider destinations to + /// : with cleartext and no credentials. + /// Default . Requires true. Does not mutate MySQL. + /// + public bool RewritePublishedDestinations { get; set; } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Configuration/LabUpstreamIsolationOptionsValidator.cs b/src/VectorNNTPD.BackFillerService/Configuration/LabUpstreamIsolationOptionsValidator.cs new file mode 100644 index 00000000..e23b855d --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Configuration/LabUpstreamIsolationOptionsValidator.cs @@ -0,0 +1,70 @@ +using Microsoft.Extensions.Options; + +namespace VectorNNTPD.BackFillerService.Configuration +{ + /// + /// Startup validation for . + /// When disabled, always succeeds. When enabled, incomplete or non-loopback config fails closed. + /// + internal sealed class LabUpstreamIsolationOptionsValidator : IValidateOptions + { + /// + public ValidateOptionsResult Validate(string? name, LabUpstreamIsolationOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + if (!options.Enabled) + { + if (options.RewritePublishedDestinations) + { + return ValidateOptionsResult.Fail( + "BackFiller:LabUpstreamIsolation:RewritePublishedDestinations requires Enabled=true " + + "(lab destination rewrite cannot run without the dial allowlist)."); + } + + return ValidateOptionsResult.Success; + } + + var failures = new List(); + if (string.IsNullOrWhiteSpace(options.AllowedHost)) + { + failures.Add( + "BackFiller:LabUpstreamIsolation:AllowedHost is required when Enabled is true " + + $"(must be {LabUpstreamIsolationOptions.RequiredLiteralHost})."); + } + else if (!string.Equals( + options.AllowedHost.Trim(), + LabUpstreamIsolationOptions.RequiredLiteralHost, + StringComparison.Ordinal)) + { + failures.Add( + "BackFiller:LabUpstreamIsolation:AllowedHost must be exactly " + + $"{LabUpstreamIsolationOptions.RequiredLiteralHost} when Enabled is true " + + "(localhost, DNS names, IPv6, and non-loopback addresses are rejected). " + + $"Configured value is not accepted."); + } + + if (options.AllowedPort is null) + { + failures.Add( + "BackFiller:LabUpstreamIsolation:AllowedPort is required when Enabled is true."); + } + else if (options.AllowedPort is < 1 or > 65535) + { + failures.Add( + "BackFiller:LabUpstreamIsolation:AllowedPort must be an integer in 1..65535 when Enabled is true."); + } + + if (options.RewritePublishedDestinations && !options.RejectTls) + { + failures.Add( + "BackFiller:LabUpstreamIsolation:RejectTls must be true when RewritePublishedDestinations is true " + + "(local ARTICLE peer is cleartext; rewritten destinations use UseTls=false)."); + } + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageId.cs b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageId.cs new file mode 100644 index 00000000..afa5bf2a --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageId.cs @@ -0,0 +1,94 @@ +namespace VectorNNTPD.BackFillerService.Diagnostics +{ + /// + /// Named stages recorded by . + /// Nested stages are documented on each member; totals must not be summed blindly. + /// + internal enum ArticleWorkStageId + { + /// + /// wall from entry through reply + /// publish attempt (settlement excluded). Inclusive of handler and reply publish. + /// + PipelineTotal = 0, + + /// + /// wall. Nested inside + /// . Inclusive of retrieve, parse-in-callback, and retain. + /// + HandlerTotal = 1, + + /// + /// wait for a session lease. Nested inside + /// . Exclusive of ARTICLE exchange. + /// + SessionAcquire = 2, + + /// + /// Wait for busy gate before ARTICLE. Nested inside + /// . + /// + SessionBusyWait = 3, + + /// + /// ARTICLE command write through status-line read. Nested inside . + /// Exclusive of payload receive. + /// + ArticleCommandToStatus = 4, + + /// + /// Multiline ARTICLE body receive including the synchronous payload consumer callback + /// (canonical parse). Nested inside . Inclusive of + /// . + /// + ArticlePayloadReceive = 5, + + /// + /// Canonical parse / materialize inside the payload callback. Nested inside + /// (do not add to parent). + /// + ParseCanonical = 6, + + /// + /// wall after the lease is held. + /// Nested inside . Inclusive of busy wait, command, payload, parse. + /// Exclusive of . + /// + NntpDownload = 7, + + /// + /// wall. Nested inside + /// . Exclusive of NNTP. + /// + Retain = 8, + + /// + /// Confirmed reply publication attempt ( + /// → ). Nested inside + /// . Inclusive of serialize, publish-gate wait, and confirm. + /// Exclusive of handler. + /// + ReplyPublish = 9, + + /// + /// before the publish gate. + /// Nested inside . Exclusive of gate wait and confirm. + /// + ReplySerialize = 10, + + /// + /// Wait for a _publishGate permit + /// (capacity ). + /// Nested inside . Exclusive of serialize and of work performed while holding a permit. + /// + ReplyPublishGateWait = 11, + + /// + /// Work while holding a _publishGate permit: channel ensure + confirmed publication + /// (RabbitMQ.Client BasicPublishAsync with publisher confirmation tracking — confirm + /// completion is not separately observable). Nested inside . + /// Exclusive of gate wait. Multiple operations may hold this interval concurrently up to the gate capacity. + /// + ReplyPublishConfirm = 12, + } +} diff --git a/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfiler.cs b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfiler.cs new file mode 100644 index 00000000..0173b1de --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfiler.cs @@ -0,0 +1,252 @@ +using System.Diagnostics; +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Diagnostics +{ + /// + /// Opt-in, bounded Article Work stage latency profiler for lab diagnosis. + /// When not installed or disabled, is false and hooks are no-ops. + /// + /// + /// Installed by only when lab options enable it. + /// Production hosts leave this uninstalled. Nested stages overlap; do not sum exclusive wall time. + /// + internal sealed class ArticleWorkStageProfiler + { + private static ArticleWorkStageProfiler? _installed; + + private readonly StageLatencyHistogram[] _stages; + private readonly bool _enabled; + private int _active; + private int _peakActive; + private long _completed; + private long _failed; + + /// Creates a profiler. gates recording. + /// When false, is false and record methods return immediately. + internal ArticleWorkStageProfiler(bool enabled) + { + _enabled = enabled; + var count = Enum.GetValues().Length; + _stages = new StageLatencyHistogram[count]; + for (var i = 0; i < count; i++) + { + _stages[i] = new StageLatencyHistogram(); + } + } + + /// Currently installed profiler for ambient hot-path hooks; null when disabled. + internal static ArticleWorkStageProfiler? Installed => Volatile.Read(ref _installed); + + /// True when an enabled profiler is installed. + internal static bool IsEnabled + { + get + { + var current = Volatile.Read(ref _installed); + return current is { _enabled: true }; + } + } + + /// Publishes for ambient hooks (or clears when null). + /// Profiler to install, or null to uninstall. + internal static void Install(ArticleWorkStageProfiler? profiler) => + Volatile.Write(ref _installed, profiler); + + /// Instance enabled flag (independent of installation). + internal bool Enabled => _enabled; + + /// In-flight handler/pipeline operations currently between begin and end. + internal int Active => Volatile.Read(ref _active); + + /// Peak concurrent active operations since install/reset. + internal int PeakActive => Volatile.Read(ref _peakActive); + + /// Completed pipeline operations (success or failure) that called . + internal long Completed => Volatile.Read(ref _completed); + + /// Pipeline operations marked failed via . + internal long Failed => Volatile.Read(ref _failed); + + /// Histogram for . + internal StageLatencyHistogram this[ArticleWorkStageId stage] => _stages[(int)stage]; + + /// Marks the start of one concurrent active unit of work. + internal void BeginActive() + { + if (!_enabled) + { + return; + } + + var n = Interlocked.Increment(ref _active); + UpdatePeak(ref _peakActive, n); + } + + /// Marks the end of one concurrent active unit of work. + /// When true, increments . + internal void EndActive(bool failed = false) + { + if (!_enabled) + { + return; + } + + Interlocked.Decrement(ref _active); + Interlocked.Increment(ref _completed); + if (failed) + { + Interlocked.Increment(ref _failed); + } + } + + /// Records elapsed time for when enabled. + /// Stage identity. + /// at stage start. + internal void Record(ArticleWorkStageId stage, long startedTimestamp) + { + if (!_enabled || startedTimestamp == 0) + { + return; + } + + _stages[(int)stage].Record(startedTimestamp); + } + + /// Records a failure on without a duration. + internal void RecordFailure(ArticleWorkStageId stage) + { + if (!_enabled) + { + return; + } + + _stages[(int)stage].RecordFailure(); + } + + /// Records a timeout on without a duration. + internal void RecordTimeout(ArticleWorkStageId stage) + { + if (!_enabled) + { + return; + } + + _stages[(int)stage].RecordTimeout(); + } + + /// Builds a JSON-serializable snapshot for lab evidence export. + internal ArticleWorkStageProfilerSnapshot CaptureSnapshot() + { + var stages = new List(_stages.Length); + foreach (ArticleWorkStageId id in Enum.GetValues()) + { + var h = _stages[(int)id]; + stages.Add( + new ArticleWorkStageSnapshot( + id.ToString(), + NestedNote(id), + h.Count, + h.MeanMilliseconds, + h.ApproximatePercentileMilliseconds(0.50), + h.ApproximatePercentileMilliseconds(0.95), + h.ApproximatePercentileMilliseconds(0.99), + h.MaxMicroseconds / 1000.0, + h.SumMicroseconds / 1000.0, + h.Failures, + h.Timeouts)); + } + + return new ArticleWorkStageProfilerSnapshot( + DateTimeOffset.UtcNow, + _enabled, + Active, + PeakActive, + Completed, + Failed, + stages); + } + + /// Serializes as indented JSON (AOT-safe source generation). + internal string ToJson() => + JsonSerializer.Serialize( + CaptureSnapshot(), + ArticleWorkStageProfilerJsonContext.Default.ArticleWorkStageProfilerSnapshot); + + private static string NestedNote(ArticleWorkStageId id) => + id switch + { + ArticleWorkStageId.PipelineTotal => "Inclusive of HandlerTotal and ReplyPublish; settlement excluded.", + ArticleWorkStageId.HandlerTotal => "Nested in PipelineTotal. Inclusive of NntpDownload, ParseCanonical (via payload), Retain.", + ArticleWorkStageId.SessionAcquire => "Nested in NntpDownload/HandlerTotal. Exclusive of ARTICLE bytes.", + ArticleWorkStageId.SessionBusyWait => "Nested in NntpDownload.", + ArticleWorkStageId.ArticleCommandToStatus => "Nested in NntpDownload. Exclusive of payload.", + ArticleWorkStageId.ArticlePayloadReceive => "Nested in NntpDownload. Inclusive of ParseCanonical.", + ArticleWorkStageId.ParseCanonical => "Nested in ArticlePayloadReceive — do not add to parent.", + ArticleWorkStageId.NntpDownload => "Nested in HandlerTotal. Inclusive of busy wait, command, payload, parse. Exclusive of SessionAcquire.", + ArticleWorkStageId.Retain => "Nested in HandlerTotal. Exclusive of NNTP.", + ArticleWorkStageId.ReplyPublish => "Nested in PipelineTotal. Inclusive of ReplySerialize, ReplyPublishGateWait, ReplyPublishConfirm.", + ArticleWorkStageId.ReplySerialize => "Nested in ReplyPublish. Before publish gate. Exclusive of gate wait/confirm.", + ArticleWorkStageId.ReplyPublishGateWait => "Nested in ReplyPublish. Wait for _publishGate permit (capacity MaxConcurrentConfirmedPublications). Exclusive of serialize/confirm.", + ArticleWorkStageId.ReplyPublishConfirm => "Nested in ReplyPublish. Gate-held: channel ensure + BasicPublishAsync with confirm tracking (opaque confirm). Exclusive of gate wait. Concurrent up to gate capacity.", + _ => string.Empty, + }; + + private static void UpdatePeak(ref int location, int candidate) + { + var observed = Volatile.Read(ref location); + while (candidate > observed) + { + var prev = Interlocked.CompareExchange(ref location, candidate, observed); + if (prev == observed) + { + return; + } + + observed = prev; + } + } + } + + /// Point-in-time profiler export for lab evidence. + /// Snapshot time. + /// Whether recording was enabled. + /// Current in-flight active count. + /// Peak in-flight active count. + /// Completed active units. + /// Failed active units. + /// Per-stage histograms. + internal sealed record ArticleWorkStageProfilerSnapshot( + DateTimeOffset CapturedUtc, + bool Enabled, + int Active, + int PeakActive, + long Completed, + long Failed, + IReadOnlyList Stages); + + /// One stage histogram summary. + /// Stage name. + /// Inclusive/exclusive nesting note. + /// Observation count. + /// Arithmetic mean milliseconds. + /// Approximate p50 milliseconds. + /// Approximate p95 milliseconds. + /// Approximate p99 milliseconds. + /// Maximum milliseconds. + /// Sum of durations in milliseconds (parallelism may exceed wall clock). + /// Attributed failures. + /// Attributed timeouts. + internal sealed record ArticleWorkStageSnapshot( + string Stage, + string Nesting, + long Count, + double MeanMs, + double P50Ms, + double P95Ms, + double P99Ms, + double MaxMs, + double TotalMs, + long Failures, + long Timeouts); +} diff --git a/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilerJsonContext.cs b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilerJsonContext.cs new file mode 100644 index 00000000..bd182d2f --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilerJsonContext.cs @@ -0,0 +1,17 @@ +using System.Text.Json.Serialization; + +namespace VectorNNTPD.BackFillerService.Diagnostics +{ + /// + /// Source-generated JSON metadata for lab stage-profiler snapshots. + /// + /// + /// Required for Native AOT publish (IL2026/IL3050). Lab-only at runtime + /// (profiler disabled by default), but the serializer still ships in the production binary. + /// + [JsonSourceGenerationOptions(WriteIndented = true)] + [JsonSerializable(typeof(ArticleWorkStageProfilerSnapshot))] + [JsonSerializable(typeof(ArticleWorkStageSnapshot))] + [JsonSerializable(typeof(List))] + internal sealed partial class ArticleWorkStageProfilerJsonContext : JsonSerializerContext; +} diff --git a/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilingHostedService.cs b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilingHostedService.cs new file mode 100644 index 00000000..30ec628e --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilingHostedService.cs @@ -0,0 +1,133 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.BackFillerService.Configuration; + +namespace VectorNNTPD.BackFillerService.Diagnostics +{ + /// + /// Installs the lab stage profiler when enabled and optionally exports JSON snapshots. + /// + internal sealed class ArticleWorkStageProfilingHostedService : IHostedService, IAsyncDisposable + { + private readonly LabStageProfilingOptions _options; + private readonly ArticleWorkStageProfiler _profiler; + private readonly ILogger _logger; + private CancellationTokenSource? _exportCts; + private Task? _exportLoop; + + /// Creates the hosted service bound to validated options. + public ArticleWorkStageProfilingHostedService( + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(logger); + _options = options.Value; + _logger = logger; + _profiler = new ArticleWorkStageProfiler(_options.Enabled); + } + + /// The profiler instance owned by this hosted service. + internal ArticleWorkStageProfiler Profiler => _profiler; + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (!_options.Enabled) + { + ArticleWorkStageProfiler.Install(null); + _logger.LogInformation("Lab Article Work stage profiling is disabled."); + return Task.CompletedTask; + } + + ArticleWorkStageProfiler.Install(_profiler); + _logger.LogInformation( + "Lab Article Work stage profiling enabled. ExportDirectory={ExportDirectory} IntervalSeconds={IntervalSeconds}", + string.IsNullOrWhiteSpace(_options.ExportDirectory) ? "(none)" : _options.ExportDirectory, + _options.ExportIntervalSeconds); + + if (!string.IsNullOrWhiteSpace(_options.ExportDirectory)) + { + Directory.CreateDirectory(_options.ExportDirectory); + _exportCts = new CancellationTokenSource(); + _exportLoop = ExportLoopAsync(_exportCts.Token); + } + + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_exportCts is not null) + { + await _exportCts.CancelAsync().ConfigureAwait(false); + } + + if (_exportLoop is not null) + { + try + { + await _exportLoop.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + _logger.LogTrace("Stage profiling export loop wait cancelled during stop."); + } + // codeql[cs/catch-of-all-exceptions] Stop must observe export-loop failure without failing host shutdown. + catch (Exception ex) + { + _logger.LogDebug(ex, "Stage profiling export loop ended with error."); + } + } + + if (_options.Enabled && !string.IsNullOrWhiteSpace(_options.ExportDirectory)) + { + try + { + await WriteSnapshotAsync(CancellationToken.None).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Final snapshot is best-effort; shutdown must continue. + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to write final stage profiling snapshot."); + } + } + + ArticleWorkStageProfiler.Install(null); + } + + /// + public async ValueTask DisposeAsync() + { + if (_exportCts is not null) + { + await _exportCts.CancelAsync().ConfigureAwait(false); + _exportCts.Dispose(); + } + + ArticleWorkStageProfiler.Install(null); + } + + private async Task ExportLoopAsync(CancellationToken cancellationToken) + { + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(_options.ExportIntervalSeconds)); + try + { + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + await WriteSnapshotAsync(cancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _logger.LogTrace("Stage profiling export loop cancelled."); + } + } + + private Task WriteSnapshotAsync(CancellationToken cancellationToken) + { + var path = Path.Join(_options.ExportDirectory!, "article-work-stage-profile.json"); + return File.WriteAllTextAsync(path, _profiler.ToJson(), cancellationToken); + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Diagnostics/StageLatencyHistogram.cs b/src/VectorNNTPD.BackFillerService/Diagnostics/StageLatencyHistogram.cs new file mode 100644 index 00000000..e23af321 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Diagnostics/StageLatencyHistogram.cs @@ -0,0 +1,164 @@ +using System.Diagnostics; + +namespace VectorNNTPD.BackFillerService.Diagnostics +{ + /// + /// Lock-free fixed-bucket latency histogram in microseconds. + /// Buckets are powers of two from 1 µs through 2^20 µs (~1.05 s), plus an overflow bucket. + /// + /// + /// Percentiles are approximate (upper edge of the cumulative bucket). Memory is bounded: + /// counters plus sum/count/max/failure fields. + /// + internal sealed class StageLatencyHistogram + { + /// Number of power-of-two buckets including overflow (indices 0..20 + 21). + internal const int BucketCount = 22; + + /// Upper inclusive microsecond edge for bucket (overflow uses ). + internal static long BucketUpperUs(int index) => + index >= BucketCount - 1 ? long.MaxValue : 1L << index; + + private readonly long[] _buckets = new long[BucketCount]; + private long _count; + private long _sumUs; + private long _maxUs; + private long _failures; + private long _timeouts; + + /// Successful and failed observations that recorded a duration. + internal long Count => Volatile.Read(ref _count); + + /// Sum of recorded durations in microseconds. + internal long SumMicroseconds => Volatile.Read(ref _sumUs); + + /// Maximum recorded duration in microseconds. + internal long MaxMicroseconds => Volatile.Read(ref _maxUs); + + /// Failure outcomes attributed to this stage (not every observation). + internal long Failures => Volatile.Read(ref _failures); + + /// Timeout outcomes attributed to this stage. + internal long Timeouts => Volatile.Read(ref _timeouts); + + /// Records one elapsed interval from a prior . + /// at stage start. + internal void Record(long startedTimestamp) + { + var us = (long)(Stopwatch.GetElapsedTime(startedTimestamp).TotalMicroseconds); + if (us < 0) + { + us = 0; + } + + RecordMicroseconds(us); + } + + /// Records a duration already expressed in microseconds (tests). + /// Non-negative elapsed microseconds. + internal void RecordMicroseconds(long microseconds) + { + if (microseconds < 0) + { + microseconds = 0; + } + + Interlocked.Increment(ref _count); + Interlocked.Add(ref _sumUs, microseconds); + UpdateMax(ref _maxUs, microseconds); + Interlocked.Increment(ref _buckets[BucketIndex(microseconds)]); + } + + /// Increments the failure counter without recording a duration. + internal void RecordFailure() => Interlocked.Increment(ref _failures); + + /// Increments the timeout counter without recording a duration. + internal void RecordTimeout() => Interlocked.Increment(ref _timeouts); + + /// Copies bucket counts for snapshotting. + /// Must have length ≥ . + internal void CopyBuckets(Span destination) + { + for (var i = 0; i < BucketCount; i++) + { + destination[i] = Volatile.Read(ref _buckets[i]); + } + } + + /// Approximate percentile in milliseconds from the current bucket snapshot. + /// Inclusive (0, 1]. + /// Milliseconds at the upper edge of the cumulative bucket; 0 when empty. + internal double ApproximatePercentileMilliseconds(double quantile) + { + var count = Volatile.Read(ref _count); + if (count == 0 || quantile <= 0) + { + return 0; + } + + Span buckets = stackalloc long[BucketCount]; + CopyBuckets(buckets); + var target = (long)Math.Ceiling(Math.Clamp(quantile, 0, 1) * count); + if (target < 1) + { + target = 1; + } + + long cumulative = 0; + for (var i = 0; i < BucketCount; i++) + { + cumulative += buckets[i]; + if (cumulative >= target) + { + var upperUs = BucketUpperUs(i); + if (upperUs == long.MaxValue) + { + return Volatile.Read(ref _maxUs) / 1000.0; + } + + return upperUs / 1000.0; + } + } + + return Volatile.Read(ref _maxUs) / 1000.0; + } + + /// Mean duration in milliseconds; 0 when empty. + internal double MeanMilliseconds + { + get + { + var count = Volatile.Read(ref _count); + return count == 0 ? 0 : (Volatile.Read(ref _sumUs) / (double)count) / 1000.0; + } + } + + /// Maps microseconds to a power-of-two bucket index. + internal static int BucketIndex(long microseconds) + { + if (microseconds <= 1) + { + return 0; + } + + // ceil(log2(us)): bucket i covers (2^(i-1), 2^i] for i>=1, with i=0 covering [0,1]. + var logCeil = 64 - (int)ulong.LeadingZeroCount((ulong)(microseconds - 1)); + return Math.Min(logCeil, BucketCount - 1); + } + + private static void UpdateMax(ref long location, long candidate) + { + var observed = Volatile.Read(ref location); + while (candidate > observed) + { + var prev = Interlocked.CompareExchange(ref location, candidate, observed); + if (prev == observed) + { + return; + } + + observed = prev; + } + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs index f9bcc811..a02be779 100644 --- a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs +++ b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using VectorNNTPD.BackFillerService.ArticleWork; using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Diagnostics; using VectorNNTPD.Common.Core; using VectorNNTPD.BackFillerService.Hosting.Systemd; using VectorNNTPD.BackFillerService.Listener; @@ -203,10 +204,38 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication provider.GetRequiredService>(), provider.GetRequiredService>(), provider.GetRequiredService>(), - provider.GetRequiredService>())); + provider.GetRequiredService>(), + provider.GetRequiredService>())); builder.Services.AddSingleton(static provider => provider.GetRequiredService()); - builder.Services.TryAddSingleton(); + builder.Services + .AddOptions() + .BindConfiguration( + BackFillerOptions.SectionName + ":" + LabUpstreamIsolationOptions.SectionRelativeName) + .ValidateOnStart(); + builder.Services.AddSingleton, LabUpstreamIsolationOptionsValidator>(); + + builder.Services + .AddOptions() + .BindConfiguration( + BackFillerOptions.SectionName + ":" + LabStageProfilingOptions.SectionRelativeName) + .ValidateOnStart(); + builder.Services.AddSingleton, LabStageProfilingOptionsValidator>(); + + // Concrete TCP factory is always registered; INntpTransportFactory wraps it with the lab + // allowlist only when BackFiller:LabUpstreamIsolation:Enabled is true (default false). + builder.Services.TryAddSingleton(); + builder.Services.AddSingleton(static provider => + { + var inner = provider.GetRequiredService(); + var lab = provider.GetRequiredService>().Value; + if (!lab.Enabled) + { + return inner; + } + + return new LabUpstreamAllowlistNntpTransportFactory(inner, lab); + }); builder.Services.AddSingleton(); builder.Services.AddSingleton(static provider => provider.GetRequiredService()); @@ -245,6 +274,8 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication builder.Services.AddSingleton(static provider => new ArticleRetentionSweepService( provider.GetRequiredService(), provider.GetRequiredService>())); + builder.Services.AddSingleton(static provider => new ArticleRetentionMetrics( + provider.GetRequiredService())); builder.Services.AddSingleton(static provider => new CacheListenerService( provider.GetRequiredService(), provider.GetRequiredService(), @@ -261,6 +292,11 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication builder.Services.AddSingleton(); builder.Services.AddSingleton(static provider => provider.GetRequiredService()); + builder.Services.AddSingleton(static provider => + provider.GetRequiredService()); + builder.Services.AddSingleton(); + builder.Services.AddHostedService(static provider => + provider.GetRequiredService()); // Rebuild when the runtime FQDN changes after nntpsharedconfig projection. A plain // singleton NntpArticleParser would keep the config-file identity forever. builder.Services.AddSingleton(static provider => diff --git a/src/VectorNNTPD.BackFillerService/Nntp/LabUpstreamAllowlistNntpTransportFactory.cs b/src/VectorNNTPD.BackFillerService/Nntp/LabUpstreamAllowlistNntpTransportFactory.cs new file mode 100644 index 00000000..b470b2c7 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Nntp/LabUpstreamAllowlistNntpTransportFactory.cs @@ -0,0 +1,117 @@ +using VectorNNTPD.BackFillerService.Configuration; + +namespace VectorNNTPD.BackFillerService.Nntp +{ + /// + /// Lab-only decorator that rejects upstream dials that are not the configured loopback peer + /// before the inner factory opens a socket. + /// + /// + /// Constructed only when is true. + /// Does not rewrite fields. + /// Reconnects and new pooled sessions call again and reapply the gate. + /// There is no fallback to an unguarded factory from this type. + /// + internal sealed class LabUpstreamAllowlistNntpTransportFactory : INntpTransportFactory + { + private readonly INntpTransportFactory _inner; + private readonly string _allowedHost; + private readonly int _allowedPort; + private readonly bool _rejectTls; + + /// + /// Wraps with fail-closed allowlist checks from . + /// + /// Concrete transport that opens sockets (typically ). + /// Enabled lab isolation options already validated at startup. + /// An argument is null. + /// is disabled or incomplete. + internal LabUpstreamAllowlistNntpTransportFactory( + INntpTransportFactory inner, + LabUpstreamIsolationOptions options) + { + ArgumentNullException.ThrowIfNull(inner); + ArgumentNullException.ThrowIfNull(options); + if (!options.Enabled) + { + throw new InvalidOperationException( + "LabUpstreamAllowlistNntpTransportFactory must not be constructed when LabUpstreamIsolation is disabled."); + } + + if (string.IsNullOrWhiteSpace(options.AllowedHost) + || !string.Equals( + options.AllowedHost.Trim(), + LabUpstreamIsolationOptions.RequiredLiteralHost, + StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "LabUpstreamIsolation AllowedHost must be " + + LabUpstreamIsolationOptions.RequiredLiteralHost + + " when constructing the allowlist transport."); + } + + if (options.AllowedPort is null or < 1 or > 65535) + { + throw new InvalidOperationException( + "LabUpstreamIsolation AllowedPort must be in 1..65535 when constructing the allowlist transport."); + } + + _inner = inner; + _allowedHost = LabUpstreamIsolationOptions.RequiredLiteralHost; + _allowedPort = options.AllowedPort.Value; + _rejectTls = options.RejectTls; + } + + /// + /// + /// Validates host/port/(optional) TLS before calling the inner factory. + /// Invalid destinations never reach socket creation. + /// + public Task ConnectAsync( + BackFillerProviderDefinition provider, + NntpSessionOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(provider); + ArgumentNullException.ThrowIfNull(options); + EnsureAllowedOrThrow(provider); + return _inner.ConnectAsync(provider, options, cancellationToken); + } + + /// Throws when is outside the lab allowlist. + internal void EnsureAllowedOrThrow(BackFillerProviderDefinition provider) + { + ArgumentNullException.ThrowIfNull(provider); + + if (string.IsNullOrWhiteSpace(provider.Host)) + { + throw new InvalidOperationException( + $"Lab upstream isolation refused backbone={provider.Backbone}: host is missing. " + + $"Allowed={_allowedHost}:{_allowedPort}."); + } + + var host = provider.Host.Trim(); + if (!string.Equals(host, _allowedHost, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Lab upstream isolation refused backbone={provider.Backbone} host='{host}' port={provider.Port}. " + + $"Allowed host is literal {_allowedHost} only (no DNS, localhost, IPv6, or non-loopback). " + + $"Allowed={_allowedHost}:{_allowedPort}."); + } + + if (provider.Port != _allowedPort) + { + throw new InvalidOperationException( + $"Lab upstream isolation refused backbone={provider.Backbone} port={provider.Port}; " + + $"allowed port is {_allowedPort}."); + } + + if (_rejectTls && provider.UseTls) + { + throw new InvalidOperationException( + $"Lab upstream isolation refused backbone={provider.Backbone}: UseTls=true is incompatible " + + "with the cleartext local ARTICLE peer (configure usessl=n). Destination was not rewritten."); + } + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpArticleRetriever.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpArticleRetriever.cs index 19c97d0e..50c69171 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpArticleRetriever.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpArticleRetriever.cs @@ -1,4 +1,6 @@ +using System.Diagnostics; using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Diagnostics; using VectorNNTPD.Common.Articles; using VectorNNTPD.Common.Core; @@ -95,12 +97,14 @@ public async Task RetrieveAsync( } NntpSessionLease lease; + var acquireStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; try { lease = await pool.AcquireAsync(cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + ArticleWorkStageProfiler.Installed?.RecordFailure(ArticleWorkStageId.SessionAcquire); return ArticleRetrievalResult.Failed( ArticleRetrievalKind.Cancelled, null, @@ -109,13 +113,17 @@ public async Task RetrieveAsync( } catch (NntpProviderConnectException ex) { + ArticleWorkStageProfiler.Installed?.RecordFailure(ArticleWorkStageId.SessionAcquire); return ArticleRetrievalResult.Failed(ex.Kind, ex.StatusCode, ex.Message, sessionReusable: false); } + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.SessionAcquire, acquireStarted); + await using (lease) { try { + var downloadStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; var result = await lease.Session .DownloadArticleAsync( item.Request.MessageId, @@ -123,6 +131,7 @@ public async Task RetrieveAsync( consumePayload, maxArticleBytes) .ConfigureAwait(false); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.NntpDownload, downloadStarted); if (!result.SessionReusable) { lease.Retire(); diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpProtocolIo.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpProtocolIo.cs index 4a5274c5..71538918 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpProtocolIo.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpProtocolIo.cs @@ -242,6 +242,8 @@ internal sealed class NntpStreamReader : IDisposable /// . 8 MiB is the next power of two above that limit. /// Eight arrays per bucket reuse a warm buffer after reconnects. A returned array larger than the pool /// maximum is left for the collector; does not store it. + /// Lab prototype binaries may use a larger maxArraysPerBucket; see + /// .artifacts/.../policy-b-scratch-reuse-design/prototype-b64/. /// private static readonly ArrayPool PayloadScratchPool = ArrayPool.Create( maxArrayLength: 8 * 1024 * 1024, @@ -256,14 +258,13 @@ internal sealed class NntpStreamReader : IDisposable /// when maxBytes allows it. A common article then needs no growth copy. Larger articles double /// until they fit, and growth stops at the caller's max instead of renting that max up front. /// The rented array may be longer than the size requested. is that - /// request, and the array length is the usable capacity. After a payload read, a request above 1 MiB - /// is returned and a 1 MiB buffer is rented again, so one large article does not stay on the reader. - /// The exact-array read copies the written prefix out. The callback read exposes that prefix only - /// until the callback returns, then the next article may overwrite it. + /// request, and the array length is the usable capacity. After a payload read finishes, the scratch is + /// returned to so idle readers do not retain a ~1 MiB lease; the next article + /// rents again. The exact-array read copies the written prefix out. The callback read exposes that prefix + /// only until the callback returns; callers must not store it. /// Returned arrays are not cleared. The scratch holds article bytes, not credentials. /// private byte[] _payloadScratch = []; - /// Size last requested from . Zero when no buffer is rented. private int _payloadRentMinimum; @@ -499,9 +500,10 @@ internal Task ReadArticlePayloadAsync( /// Destuffing matches . /// This overload does not copy the prefix. The scratch length is cleared after returns, /// including when throws. A later read on this reader may overwrite the same bytes. - /// When the scratch was rented above 1 MiB, that array is returned after the callback and a 1 MiB buffer is rented - /// for the next article. is not called when the read fails before a terminator. - /// Caller cancellation and the timeout both surface as . + /// After returns (or throws), any rented scratch is returned to the pool unless + /// disposal already owns cleanup. The next article rents again. is not called when + /// the read fails before a terminator. Caller cancellation and the timeout both surface as + /// . /// internal async Task ReadArticlePayloadAsync( int maxBytes, @@ -587,7 +589,7 @@ internal async Task ReadArticlePayloadAsync( { if (Volatile.Read(ref _payloadRelease) == 0) { - TrimPayloadScratch(); + ReleasePayloadScratchAfterArticle(); } } finally @@ -596,7 +598,6 @@ internal async Task ReadArticlePayloadAsync( } } } - /// /// Reads one more socket chunk after a trailing CR was left as the only unread byte. /// @@ -764,30 +765,23 @@ private void EnsurePayloadCapacity(int needed, int maxBytes) } /// - /// Returns an oversized scratch and rents the 1 MiB working buffer. - /// Does nothing when the last request was already at or below that size, or when no buffer is rented. + /// Returns any rented payload scratch after an ARTICLE payload read finishes and consume has returned. /// /// - /// Called only after is cleared, so there is no prefix to preserve. - /// The replacement is published before the previous array is returned. A rent failure leaves the oversized array in place. + /// Called from finally only when disposal has not taken ownership + /// ( is still zero). Prefix length is already cleared. The exchange publishes an + /// empty non-pooled array before so a later Dispose cannot return the same + /// buffer twice. Called while is still elevated; runs after. /// - private void TrimPayloadScratch() + private void ReleasePayloadScratchAfterArticle() { - if (_payloadRentMinimum <= PayloadScratchInitialBytes) - { - return; - } - - var replacement = _payloadPool.Rent(PayloadScratchInitialBytes); - var previous = _payloadScratch; - _payloadScratch = replacement; - _payloadRentMinimum = PayloadScratchInitialBytes; - if (previous.Length != 0) + var scratch = Interlocked.Exchange(ref _payloadScratch, []); + _payloadRentMinimum = 0; + if (scratch.Length != 0) { - _payloadPool.Return(previous); + _payloadPool.Return(scratch); } } - /// Marks one payload read so cannot return the scratch underneath it. private void EnterPayload() => Interlocked.Increment(ref _payloadUsers); diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpProviderSession.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpProviderSession.cs index da355f43..4f183b78 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpProviderSession.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpProviderSession.cs @@ -7,6 +7,7 @@ using VectorNNTPD.Common.Articles; using System.Buffers; using System.Text; +using VectorNNTPD.BackFillerService.Diagnostics; using VectorNNTPD.Common.Core; namespace VectorNNTPD.BackFillerService.Nntp @@ -430,7 +431,9 @@ internal async Task DownloadArticleAsync( await beforeBusy().ConfigureAwait(false); } + var busyWaitStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; await _busy.WaitAsync(cancellationToken).ConfigureAwait(false); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.SessionBusyWait, busyWaitStarted); var previous = State; ExceptionDispatchInfo? consumerError = null; try @@ -447,6 +450,7 @@ internal async Task DownloadArticleAsync( AssignState(NntpSessionState.Busy); var commandLength = NntpProtocolIo.ArticlePrefix.Length + messageIdBytes.Length + NntpProtocolIo.Crlf.Length; var command = ArrayPool.Shared.Rent(commandLength); + var commandStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; try { NntpProtocolIo.ArticlePrefix.CopyTo(command.AsSpan()); @@ -466,6 +470,7 @@ await WriteCommandAsync( } var statusLine = await ReadStatusLineAsync(cancellationToken).ConfigureAwait(false); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.ArticleCommandToStatus, commandStarted); if (statusLine is null) { return MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, null, "NNTP ARTICLE status was empty."); @@ -485,29 +490,35 @@ await WriteCommandAsync( if (consumePayload is null) { byte[] payload; + var payloadStarted = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; try { payload = await reader .ReadArticlePayloadAsync(maxArticleBytes, _options.ReceiveTimeout, cancellationToken) .ConfigureAwait(false); NntpLogMessages.WireArticlePayloadComplete(_logger, _wireIdentity, payload.Length); + ArticleWorkStageProfiler.Installed?.Record(ArticleWorkStageId.ArticlePayloadReceive, payloadStarted); } catch (EndOfStreamException) { + ArticleWorkStageProfiler.Installed?.RecordFailure(ArticleWorkStageId.ArticlePayloadReceive); return MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, code, "NNTP article ended before terminator."); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + ArticleWorkStageProfiler.Installed?.RecordFailure(ArticleWorkStageId.ArticlePayloadReceive); return MarkUnhealthy(ArticleRetrievalKind.Cancelled, code, "ARTICLE receive was cancelled."); } catch (OperationCanceledException) { + ArticleWorkStageProfiler.Installed?.RecordTimeout(ArticleWorkStageId.ArticlePayloadReceive); return MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, code, "ARTICLE receive timed out."); } catch (InvalidOperationException ex) when (ex.Message.Contains("MaxArticleBytes", StringComparison.Ordinal)) { // Destuffed size exceeded the hard ceiling: permanently unusable for Article Work. // Retire the session because the multiline response is no longer synchronized. + ArticleWorkStageProfiler.Installed?.RecordFailure(ArticleWorkStageId.ArticlePayloadReceive); return MarkUnhealthy(ArticleRetrievalKind.InvalidArticle, code, "NNTP article exceeded MaxArticleBytes."); } @@ -526,6 +537,7 @@ await WriteCommandAsync( } var accepted = false; + var payloadStartedWithConsumer = ArticleWorkStageProfiler.IsEnabled ? Stopwatch.GetTimestamp() : 0; try { _ = await reader.ReadArticlePayloadAsync( @@ -542,7 +554,14 @@ await WriteCommandAsync( accepted = true; try { - return consumePayload(memory); + var parseStarted = ArticleWorkStageProfiler.IsEnabled + ? Stopwatch.GetTimestamp() + : 0; + var created = consumePayload(memory); + ArticleWorkStageProfiler.Installed?.Record( + ArticleWorkStageId.ParseCanonical, + parseStarted); + return created; } catch (Exception ex) when (ex is not PayloadConsumerException) { @@ -552,21 +571,28 @@ await WriteCommandAsync( }, cancellationToken) .ConfigureAwait(false); + ArticleWorkStageProfiler.Installed?.Record( + ArticleWorkStageId.ArticlePayloadReceive, + payloadStartedWithConsumer); } catch (EndOfStreamException) { + ArticleWorkStageProfiler.Installed?.RecordFailure(ArticleWorkStageId.ArticlePayloadReceive); return MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, code, "NNTP article ended before terminator."); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + ArticleWorkStageProfiler.Installed?.RecordFailure(ArticleWorkStageId.ArticlePayloadReceive); return MarkUnhealthy(ArticleRetrievalKind.Cancelled, code, "ARTICLE receive was cancelled."); } catch (OperationCanceledException) { + ArticleWorkStageProfiler.Installed?.RecordTimeout(ArticleWorkStageId.ArticlePayloadReceive); return MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, code, "ARTICLE receive timed out."); } catch (InvalidOperationException ex) when (ex.Message.Contains("MaxArticleBytes", StringComparison.Ordinal)) { + ArticleWorkStageProfiler.Installed?.RecordFailure(ArticleWorkStageId.ArticlePayloadReceive); return MarkUnhealthy(ArticleRetrievalKind.InvalidArticle, code, "NNTP article exceeded MaxArticleBytes."); } diff --git a/src/VectorNNTPD.BackFillerService/Retention/ArticleRetentionMetrics.cs b/src/VectorNNTPD.BackFillerService/Retention/ArticleRetentionMetrics.cs new file mode 100644 index 00000000..053357f3 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Retention/ArticleRetentionMetrics.cs @@ -0,0 +1,92 @@ +using System.Diagnostics.Metrics; + +namespace VectorNNTPD.BackFillerService.Retention +{ + /// + /// Publishes live occupancy as .NET runtime metrics. + /// Values are read from the authority on each observe callback; no duplicate counters are kept. + /// + /// + /// Meter name is intended for dotnet-counters / EventPipe sampling + /// (same tooling used for System.Runtime). Instruments carry no Message-ID or article labels. + /// Registration does not change retention policy, TTL, reclaim, or ArrayPool behaviour. + /// Debug log events 5500–5502 are unchanged. + /// + internal sealed class ArticleRetentionMetrics : IHostedService, IDisposable + { + /// Meter name passed to dotnet-counters --counters. + internal const string MeterName = "VectorNNTPD.BackFillerService.ArticleRetention"; + + /// + /// Observable gauge of authority-owned ArtData bytes + /// (). Unit By (bytes). + /// + internal const string RetainedPayloadBytesInstrumentName = "retained_payload_bytes"; + + /// + /// Observable gauge of entries not yet physically disposed + /// (). Unit {article}. + /// + internal const string RetainedCountInstrumentName = "retained_count"; + + /// Authority observed by the gauges. Not owned or disposed by this type. + private readonly IArticleRetentionAuthority _authority; + + /// Meter that owns the observable instruments. Disposed with this instance. + private readonly Meter _meter; + + /// 0 until ; 1 after. Prevents double meter disposal. + private int _disposed; + + /// + /// Creates observable gauges bound to . + /// + /// Live retention authority. Source of truth for both gauges. + /// is . + internal ArticleRetentionMetrics(IArticleRetentionAuthority authority) + { + ArgumentNullException.ThrowIfNull(authority); + _authority = authority; + _meter = new Meter(MeterName); + _ = _meter.CreateObservableGauge( + RetainedPayloadBytesInstrumentName, + ObserveRetainedPayloadBytes, + unit: "By", + description: "Authority-owned retained article ArtData bytes (IArticleRetentionAuthority.RetainedPayloadBytes)."); + _ = _meter.CreateObservableGauge( + RetainedCountInstrumentName, + ObserveRetainedCount, + unit: "{article}", + description: "Retained entries not yet physically disposed (IArticleRetentionAuthority.RetainedCount)."); + } + + /// No-op. Instruments are active from construction until . + /// Unused. + /// A completed task. + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// No-op. Meter disposal runs via when the DI container releases this singleton. + /// Unused. + /// A completed task. + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// Disposes the so observe callbacks stop. Idempotent. + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + _meter.Dispose(); + } + + /// Callback for . + /// Current . + private long ObserveRetainedPayloadBytes() => _authority.RetainedPayloadBytes; + + /// Callback for . + /// Current as . + private long ObserveRetainedCount() => _authority.RetainedCount; + } +} diff --git a/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json b/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json index a749f26b..934ea129 100644 --- a/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json +++ b/src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json @@ -7,12 +7,10 @@ "DegradedThreshold": 0.75, "EnableSsl": false, "Hosts": [ - "rabbitmq1a.usenet.ninja", - "rabbitmq1b.usenet.ninja", - "rabbitmq1c.usenet.ninja" + "198.18.0.3" ], "Management": { - "BaseUrl": "http://rabbitmq1a.usenet.ninja:15672", + "BaseUrl": "http://198.18.0.3:15672", "RequestTimeoutSeconds": 5 }, "MaxConnections": 16, diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs index ca464639..6454651c 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs @@ -146,9 +146,13 @@ public async Task CreatePublishChannelAsync( throw new InvalidOperationException("RabbitMQ connection is not open for publication."); } + // Explicit limiter: the two-arg CreateChannelOptions leaves OutstandingPublisherConfirmationsRateLimiter + // null (unlimited). Align with RabbitMqPublisherConfirmLimits / Article Work response publisher gate. var options = new CreateChannelOptions( publisherConfirmationsEnabled: true, - publisherConfirmationTrackingEnabled: true); + publisherConfirmationTrackingEnabled: true, + outstandingPublisherConfirmationsRateLimiter: new ThrottlingRateLimiter( + RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations)); var channel = await _connection .CreateChannelAsync(options: options, cancellationToken: cancellationToken) .ConfigureAwait(false); diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqPublisherConfirmLimits.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqPublisherConfirmLimits.cs new file mode 100644 index 00000000..ac3b5fcd --- /dev/null +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqPublisherConfirmLimits.cs @@ -0,0 +1,20 @@ +namespace VectorNNTPD.Common.Messaging.RabbitMq +{ + /// + /// Shared outstanding publisher-confirm bound for confirm-tracked publish channels. + /// + /// + /// RabbitMQ.Client 7.2.2 documents a default ThrottlingRateLimiter limit of 128 for + /// outstanding publisher confirmations, but the two-argument CreateChannelOptions + /// constructor leaves the limiter (disabling client-side outstanding + /// throttling). Production publish channels set an explicit limiter to this value so app-level + /// and client-level bounds stay aligned. + /// + internal static class RabbitMqPublisherConfirmLimits + { + /// + /// Maximum outstanding confirmation-tracked publications on one confirm-enabled channel. + /// + internal const int MaxOutstandingConfirmations = 128; + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Accounts/LabUpstreamDestinationRewriterTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Accounts/LabUpstreamDestinationRewriterTests.cs new file mode 100644 index 00000000..3dd42f2e --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Accounts/LabUpstreamDestinationRewriterTests.cs @@ -0,0 +1,137 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.BackFillerService.Accounts; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; + +namespace VectorNNTPD.BackFillerService.Tests.Accounts +{ + /// + /// Unit coverage for lab destination rewrite: loopback retarget, credential clear, no AUTHINFO. + /// + public sealed class LabUpstreamDestinationRewriterTests + { + [Fact] + public void Disabled_rewrite_returns_same_instance() + { + var mapped = ProviderAccountMapper.Map([ProviderAccountTestRows.Create()]); + var providers = mapped.Providers; + var result = LabUpstreamDestinationRewriter.Apply( + providers, + new LabUpstreamIsolationOptions { Enabled = false, RewritePublishedDestinations = false }); + Assert.Same(providers, result); + Assert.Equal("news.example.test", Assert.Single(result).Host); + Assert.Equal(ProviderAccountTestRows.SecretPassword, Assert.Single(result).Password); + } + + [Fact] + public void Enabled_rewrite_retargets_cleartext_and_clears_credentials() + { + var mapped = ProviderAccountMapper.Map([ProviderAccountTestRows.Create()]); + var rewritten = LabUpstreamDestinationRewriter.Apply( + mapped.Providers, + LabRewrite()); + var provider = Assert.Single(rewritten); + Assert.Equal("Giganews", provider.Backbone); + Assert.Equal("127.0.0.1", provider.Host); + Assert.Equal(11119, provider.Port); + Assert.False(provider.UseTls); + Assert.Null(provider.Username); + Assert.Null(provider.Password); + Assert.False(provider.RequiresAuthentication); + Assert.Equal(4, provider.MaxSessions); + } + + [Fact] + public void Rewrite_with_enabled_false_throws() + { + var mapped = ProviderAccountMapper.Map([ProviderAccountTestRows.Create()]); + var ex = Assert.Throws(() => + LabUpstreamDestinationRewriter.Apply( + mapped.Providers, + new LabUpstreamIsolationOptions + { + Enabled = false, + AllowedPort = 11119, + RewritePublishedDestinations = true, + })); + Assert.Contains("Enabled=true", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Rewritten_provider_session_sends_no_authinfo() + { + var mapped = ProviderAccountMapper.Map([ProviderAccountTestRows.Create()]); + var provider = Assert.Single(LabUpstreamDestinationRewriter.Apply(mapped.Providers, LabRewrite())); + + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + factory.Enqueue(server); + await using var session = new NntpProviderSession( + provider, + NntpSessionOptions.Default with + { + CommandTimeout = TimeSpan.FromSeconds(2), + ReceiveTimeout = TimeSpan.FromSeconds(2), + ConnectTimeout = TimeSpan.FromSeconds(2), + }, + NullLogger.Instance); + + Assert.Null(await session.ConnectAsync(factory, CancellationToken.None)); + Assert.Equal("CAPABILITIES", Assert.Single(server.Commands)); + Assert.DoesNotContain( + server.Commands, + static command => command.StartsWith("AUTHINFO", StringComparison.Ordinal)); + Assert.DoesNotContain( + ProviderAccountTestRows.SecretPassword, + string.Join('\n', server.Commands), + StringComparison.Ordinal); + Assert.DoesNotContain( + "nntp-user", + string.Join('\n', server.Commands), + StringComparison.Ordinal); + } + + [Fact] + public async Task Production_credentials_still_send_authinfo_when_rewrite_disabled() + { + var mapped = ProviderAccountMapper.Map([ProviderAccountTestRows.Create()]); + var provider = Assert.Single( + LabUpstreamDestinationRewriter.Apply( + mapped.Providers, + new LabUpstreamIsolationOptions { Enabled = false, RewritePublishedDestinations = false })); + Assert.True(provider.RequiresAuthentication); + + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + server.Respond(static command => command.StartsWith("AUTHINFO USER", StringComparison.Ordinal) + ? "381 password required\r\n" + : "281 ok\r\n"); + factory.Enqueue(server); + await using var session = new NntpProviderSession( + provider, + NntpSessionOptions.Default with + { + CommandTimeout = TimeSpan.FromSeconds(2), + ReceiveTimeout = TimeSpan.FromSeconds(2), + ConnectTimeout = TimeSpan.FromSeconds(2), + }, + NullLogger.Instance); + + Assert.Null(await session.ConnectAsync(factory, CancellationToken.None)); + Assert.Contains(server.Commands, static c => c.StartsWith("AUTHINFO USER", StringComparison.Ordinal)); + Assert.Contains(server.Commands, static c => c.StartsWith("AUTHINFO PASS", StringComparison.Ordinal)); + } + + private static LabUpstreamIsolationOptions LabRewrite() => + new() + { + Enabled = true, + AllowedHost = "127.0.0.1", + AllowedPort = 11119, + RejectTls = true, + RewritePublishedDestinations = true, + }; + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Accounts/ProviderAccountConfigurationServiceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Accounts/ProviderAccountConfigurationServiceTests.cs index 86c9e81b..1bb92336 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Accounts/ProviderAccountConfigurationServiceTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Accounts/ProviderAccountConfigurationServiceTests.cs @@ -596,7 +596,8 @@ public void Article_work_request_path_does_not_take_a_mysql_dependency() private static ServiceHarness CreateHarness( FakeProviderAccountSource source, - TimeSpan? refreshInterval = null) + TimeSpan? refreshInterval = null, + LabUpstreamIsolationOptions? labUpstreamIsolation = null) { var options = BackFillerTestOptions.CreateValid(); options.BackFillerAccountRefreshIntervalSeconds = 3600; @@ -643,10 +644,112 @@ NntpSessionOptions.Default with Options.Create(new BackFillerOptions()), acme, logger, - rabbit); + rabbit, + Options.Create(labUpstreamIsolation ?? new LabUpstreamIsolationOptions())); return new ServiceHarness(service, catalog, registry, transport, logger, shared, acme, rabbit); } + /// + /// Lab rewrite retargets published dial fields to the allowlist without mutating source rows. + /// + [Fact] + public async Task Lab_rewrite_publishes_loopback_cleartext_without_credentials() + { + var source = new FakeProviderAccountSource + { + Rows = [ProviderAccountTestRows.Create()], + }; + var lab = new LabUpstreamIsolationOptions + { + Enabled = true, + AllowedHost = "127.0.0.1", + AllowedPort = 11119, + RejectTls = true, + RewritePublishedDestinations = true, + }; + await using var harness = CreateHarness(source, labUpstreamIsolation: lab); + + await harness.Service.StartAsync(CancellationToken.None); + + var provider = Assert.Single(harness.Service.PublishedProviders); + Assert.Equal("Giganews", provider.Backbone); + Assert.Equal("127.0.0.1", provider.Host); + Assert.Equal(11119, provider.Port); + Assert.False(provider.UseTls); + Assert.Null(provider.Username); + Assert.Null(provider.Password); + Assert.False(provider.RequiresAuthentication); + Assert.Equal("news.example.test", source.Rows[0].Hostname); + Assert.Equal(ProviderAccountTestRows.SecretPassword, source.Rows[0].Password); + Assert.All( + harness.Logger.Messages, + message => Assert.DoesNotContain( + ProviderAccountTestRows.SecretPassword, + message, + StringComparison.Ordinal)); + } + + /// + /// Refresh with rewrite enabled cannot restore a WAN dial target from MySQL-shaped rows. + /// + [Fact] + public async Task Lab_rewrite_survives_refresh_when_source_still_lists_wan_host() + { + var source = new FakeProviderAccountSource + { + Rows = [ProviderAccountTestRows.Create()], + }; + var lab = new LabUpstreamIsolationOptions + { + Enabled = true, + AllowedHost = "127.0.0.1", + AllowedPort = 11119, + RejectTls = true, + RewritePublishedDestinations = true, + }; + await using var harness = CreateHarness(source, labUpstreamIsolation: lab); + await harness.Service.StartAsync(CancellationToken.None); + + source.Rows = + [ + ProviderAccountTestRows.Create(hostname: "news.giganews.example", port: 563, useSsl: "y"), + ]; + Assert.True(await harness.Service.RefreshOnceAsync(CancellationToken.None)); + + var provider = Assert.Single(harness.Service.PublishedProviders); + Assert.Equal("127.0.0.1", provider.Host); + Assert.Equal(11119, provider.Port); + Assert.False(provider.UseTls); + Assert.Null(provider.Username); + Assert.Null(provider.Password); + Assert.Contains( + harness.Logger.Messages, + static message => message.Contains("snapshot unchanged", StringComparison.OrdinalIgnoreCase)); + } + + /// + /// With rewrite disabled, production dial fields and credentials remain as mapped from source rows. + /// + [Fact] + public async Task Production_mode_retains_mapped_host_and_credentials() + { + var source = new FakeProviderAccountSource + { + Rows = [ProviderAccountTestRows.Create()], + }; + await using var harness = CreateHarness(source); + + await harness.Service.StartAsync(CancellationToken.None); + + var provider = Assert.Single(harness.Service.PublishedProviders); + Assert.Equal("news.example.test", provider.Host); + Assert.Equal(563, provider.Port); + Assert.True(provider.UseTls); + Assert.Equal("nntp-user", provider.Username); + Assert.Equal(ProviderAccountTestRows.SecretPassword, provider.Password); + Assert.True(provider.RequiresAuthentication); + } + /// /// Polls every 1 ms until the condition is true or five seconds elapse. /// Throws on timeout; does not invoke refresh or start services. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs index 19ac9b3e..bf89d532 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs @@ -556,6 +556,81 @@ public async Task Publisher_never_settles_the_original_delivery() Assert.Single(context.PublishChannel.Publications); } + /// + /// Multiple calls may await confirmation concurrently + /// (bounded by ). + /// + [Fact] + public async Task Concurrent_publishes_overlap_confirm_waits_and_preserve_correlation() + { + await using var context = await PublicationContext.StartAsync(FakePublishConfirmBehavior.Wait); + context.PublishChannel.ConfirmGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var reachedTwo = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.PublishChannel.AfterEnqueue = () => + { + if (context.PublishChannel.MaxConfirmInFlight >= 2) + { + reachedTwo.TrySetResult(); + } + }; + + var first = context.Publisher.PublishAsync( + CreateSuccessIntent("corr-a", "reply-a"), + CancellationToken.None); + var second = context.Publisher.PublishAsync( + CreateSuccessIntent("corr-b", "reply-b"), + CancellationToken.None); + + await reachedTwo.Task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.True(context.PublishChannel.MaxConfirmInFlight >= 2); + + context.PublishChannel.ConfirmGate.TrySetResult(); + await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(2, context.PublishChannel.Publications.Count); + Assert.Contains( + context.PublishChannel.Publications, + static p => p.CorrelationId == "corr-a" && p.RoutingKey == "reply-a"); + Assert.Contains( + context.PublishChannel.Publications, + static p => p.CorrelationId == "corr-b" && p.RoutingKey == "reply-b"); + Assert.All(context.PublishChannel.Publications, static p => Assert.False(p.Persistent)); + Assert.All(context.PublishChannel.Publications, static p => Assert.True(p.Mandatory)); + } + + /// + /// When overlapping confirms are released into a nack outcome, callers fail individually and none succeed. + /// + [Fact] + public async Task Concurrent_publish_nack_fails_each_caller_without_false_success() + { + await using var context = await PublicationContext.StartAsync(FakePublishConfirmBehavior.Wait); + var bothOverlapping = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + context.PublishChannel.AfterEnqueue = () => + { + if (context.PublishChannel.MaxConfirmInFlight >= 2) + { + bothOverlapping.TrySetResult(); + } + }; + context.PublishChannel.ConfirmGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var first = context.Publisher.PublishAsync( + CreateSuccessIntent("corr-1", "reply-1"), + CancellationToken.None); + var second = context.Publisher.PublishAsync( + CreateSuccessIntent("corr-2", "reply-2"), + CancellationToken.None); + + await bothOverlapping.Task.WaitAsync(TimeSpan.FromSeconds(2)); + context.PublishChannel.ConfirmBehavior = FakePublishConfirmBehavior.Nack; + context.PublishChannel.ConfirmGate.TrySetResult(); + + await Assert.ThrowsAsync(async () => await first); + await Assert.ThrowsAsync(async () => await second); + Assert.Equal(2, context.PublishChannel.Publications.Count); + } + /// /// Single pipeline pass after publish nack: one handler invocation, one publication attempt, one NACK. /// @@ -785,6 +860,20 @@ public void Production_response_code_does_not_block_synchronously() /// Handler returning canonical success endpoint fields for publication tests. /// /// Controllable handler configured for success JSON. + /// Builds a Success intent with distinct correlation/reply routing for concurrent publish tests. + private static ArticleWorkResponseIntent CreateSuccessIntent(string correlationId, string replyTo) => + new( + ArticleWorkOutcome.Success, + Guid.Parse(ArticleWorkTestDeliveries.CanonicalRequestId), + ArticleWorkTestDeliveries.CanonicalMessageId, + "Giganews", + correlationId, + replyTo, + Error: null, + ArticleWorkTestDeliveries.CanonicalFqdn, + ArticleWorkTestDeliveries.CanonicalListenerPort, + ArticleWorkTestDeliveries.CanonicalArticleIdHex); + private static ControllableArticleWorkHandler SuccessHandler() => new() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMultiInstanceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMultiInstanceTests.cs index 92fdf8ad..f68c5601 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMultiInstanceTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/SilentConsumerMonitor/SilentConsumerMultiInstanceTests.cs @@ -334,6 +334,49 @@ public void Session_removal_prunes_only_removed_session_identity() afterResolve.Single(r => r.Key.Condition == SilentConsumerCondition.ProcessingStall && r.Key.QueueOrBackbone == key2).Phase); + + // Resolved session-removed machines are dropped; a later cycle must not re-emit that identity. + var afterPrune = engine.ObserveInstance(remaining, 1); + Assert.DoesNotContain( + afterPrune, + r => r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key1); + Assert.DoesNotContain( + afterPrune, + r => r.Key.Condition == SilentConsumerCondition.DispatchStall + && r.Key.QueueOrBackbone == key1); + Assert.Contains( + afterPrune, + r => r.Key.Condition == SilentConsumerCondition.ProcessingStall + && r.Key.QueueOrBackbone == key2); + } + + /// + /// A never-opened session machine is dropped immediately on session removal (Clear + Inactive). + /// + [Fact] + public void Session_removal_drops_clear_machines_without_waiting_for_resolve_window() + { + var time = new FakeTimeProvider(DateTimeOffset.Parse("2026-10-25T21:00:00Z")); + var engine = new SilentConsumerMonitorEngine(() => "a.example", NullLogger.Instance, time); + var key1 = ArticleWorkConsumerSession.ComposeSessionKey("Eweka", 1); + + var both = new[] + { + Session("Eweka", 1, stalled: false, queued: 0, active: 0), + Session("Eweka", 2, stalled: false, queued: 0, active: 0), + }; + _ = engine.ObserveInstance(both, 2); + + var remaining = new[] { Session("Eweka", 2, stalled: false, queued: 0, active: 0) }; + var afterRemoval = engine.ObserveInstance(remaining, 1); + Assert.Contains( + afterRemoval, + r => r.Key.QueueOrBackbone == key1 + && (r.Detail?.Contains("session removed", StringComparison.OrdinalIgnoreCase) ?? false)); + + var afterPrune = engine.ObserveInstance(remaining, 1); + Assert.DoesNotContain(afterPrune, r => r.Key.QueueOrBackbone == key1); } /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs new file mode 100644 index 00000000..09651b49 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs @@ -0,0 +1,180 @@ +using VectorNNTPD.BackFillerService.Diagnostics; + +namespace VectorNNTPD.BackFillerService.Tests.Diagnostics +{ + public sealed class StageLatencyHistogramTests + { + [Fact] + public void Empty_histogram_reports_zeros() + { + var h = new StageLatencyHistogram(); + Assert.Equal(0, h.Count); + Assert.Equal(0, h.MeanMilliseconds); + Assert.Equal(0, h.ApproximatePercentileMilliseconds(0.50)); + Assert.Equal(0, h.ApproximatePercentileMilliseconds(0.99)); + Assert.Equal(0, h.Failures); + Assert.Equal(0, h.Timeouts); + } + + [Fact] + public void Records_count_sum_mean_and_max() + { + var h = new StageLatencyHistogram(); + h.RecordMicroseconds(1_000); + h.RecordMicroseconds(3_000); + h.RecordMicroseconds(2_000); + Assert.Equal(3, h.Count); + Assert.Equal(6_000, h.SumMicroseconds); + Assert.Equal(3_000, h.MaxMicroseconds); + Assert.Equal(2.0, h.MeanMilliseconds, precision: 3); + } + + [Fact] + public void Percentiles_use_bucket_upper_edges() + { + var h = new StageLatencyHistogram(); + for (var i = 0; i < 100; i++) + { + h.RecordMicroseconds(1_000); // 1 ms → bucket covering up to 1024 µs + } + + Assert.True(h.ApproximatePercentileMilliseconds(0.50) >= 1.0); + Assert.True(h.ApproximatePercentileMilliseconds(0.99) >= 1.0); + Assert.True(h.ApproximatePercentileMilliseconds(0.99) <= 2.0); + } + + [Fact] + public void Failures_and_timeouts_are_independent_of_duration_count() + { + var h = new StageLatencyHistogram(); + h.RecordFailure(); + h.RecordTimeout(); + h.RecordTimeout(); + Assert.Equal(0, h.Count); + Assert.Equal(1, h.Failures); + Assert.Equal(2, h.Timeouts); + } + + [Fact] + public void Concurrent_updates_aggregate() + { + var h = new StageLatencyHistogram(); + Parallel.For(0, 1000, _ => h.RecordMicroseconds(500)); + Assert.Equal(1000, h.Count); + Assert.Equal(500_000, h.SumMicroseconds); + } + + [Fact] + public void Bucket_index_is_bounded() + { + Assert.Equal(0, StageLatencyHistogram.BucketIndex(0)); + Assert.Equal(0, StageLatencyHistogram.BucketIndex(1)); + Assert.InRange(StageLatencyHistogram.BucketIndex(1L << 40), 0, StageLatencyHistogram.BucketCount - 1); + } + + [Fact] + public void Disabled_profiler_records_nothing() + { + var profiler = new ArticleWorkStageProfiler(enabled: false); + ArticleWorkStageProfiler.Install(profiler); + try + { + Assert.False(ArticleWorkStageProfiler.IsEnabled); + profiler.BeginActive(); + profiler.Record(ArticleWorkStageId.HandlerTotal, 1); + profiler.EndActive(); + Assert.Equal(0, profiler[ArticleWorkStageId.HandlerTotal].Count); + Assert.Equal(0, profiler.Completed); + } + finally + { + ArticleWorkStageProfiler.Install(null); + } + } + + [Fact] + public void Enabled_profiler_tracks_active_and_stages() + { + var profiler = new ArticleWorkStageProfiler(enabled: true); + ArticleWorkStageProfiler.Install(profiler); + try + { + Assert.True(ArticleWorkStageProfiler.IsEnabled); + profiler.BeginActive(); + Assert.Equal(1, profiler.Active); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + profiler.Record(ArticleWorkStageId.Retain, started); + profiler.EndActive(failed: true); + Assert.Equal(0, profiler.Active); + Assert.Equal(1, profiler.PeakActive); + Assert.Equal(1, profiler.Completed); + Assert.Equal(1, profiler.Failed); + Assert.Equal(1, profiler[ArticleWorkStageId.Retain].Count); + var snap = profiler.CaptureSnapshot(); + Assert.Contains(snap.Stages, static s => s.Stage == nameof(ArticleWorkStageId.Retain) && s.Count == 1); + } + finally + { + ArticleWorkStageProfiler.Install(null); + } + } + + [Fact] + public void Publisher_sub_stages_aggregate_and_appear_in_snapshot() + { + var profiler = new ArticleWorkStageProfiler(enabled: true); + ArticleWorkStageProfiler.Install(profiler); + try + { + profiler[ArticleWorkStageId.ReplySerialize].RecordMicroseconds(100); + profiler[ArticleWorkStageId.ReplyPublishGateWait].RecordMicroseconds(20_000); + profiler[ArticleWorkStageId.ReplyPublishConfirm].RecordMicroseconds(800); + profiler[ArticleWorkStageId.ReplyPublish].RecordMicroseconds(21_000); + + Assert.Equal(1, profiler[ArticleWorkStageId.ReplyPublishGateWait].Count); + Assert.Equal(20.0, profiler[ArticleWorkStageId.ReplyPublishGateWait].MeanMilliseconds, precision: 3); + Assert.Equal(0.8, profiler[ArticleWorkStageId.ReplyPublishConfirm].MeanMilliseconds, precision: 3); + + var snap = profiler.CaptureSnapshot(); + Assert.Contains( + snap.Stages, + static s => s.Stage == nameof(ArticleWorkStageId.ReplyPublishGateWait) + && s.Count == 1 + && s.Nesting.Contains("_publishGate", StringComparison.Ordinal) + && s.Nesting.Contains("MaxConcurrentConfirmedPublications", StringComparison.Ordinal)); + Assert.Contains( + snap.Stages, + static s => s.Stage == nameof(ArticleWorkStageId.ReplyPublishConfirm) + && s.Nesting.Contains("BasicPublishAsync", StringComparison.Ordinal)); + } + finally + { + ArticleWorkStageProfiler.Install(null); + } + } + + [Fact] + public void Concurrent_publisher_stage_records_do_not_lose_counts() + { + var profiler = new ArticleWorkStageProfiler(enabled: true); + ArticleWorkStageProfiler.Install(profiler); + try + { + Parallel.For( + 0, + 500, + _ => + { + profiler.Record(ArticleWorkStageId.ReplyPublishGateWait, System.Diagnostics.Stopwatch.GetTimestamp()); + profiler.Record(ArticleWorkStageId.ReplyPublishConfirm, System.Diagnostics.Stopwatch.GetTimestamp()); + }); + Assert.Equal(500, profiler[ArticleWorkStageId.ReplyPublishGateWait].Count); + Assert.Equal(500, profiler[ArticleWorkStageId.ReplyPublishConfirm].Count); + } + finally + { + ArticleWorkStageProfiler.Install(null); + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs index 2c994541..db6cf0ef 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs @@ -9,6 +9,7 @@ using VectorNNTPD.BackFillerService.ArticleWork; using VectorNNTPD.BackFillerService.ArticleWork.SilentConsumerMonitor; using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Diagnostics; using VectorNNTPD.Common.Core; using VectorNNTPD.BackFillerService.Hosting; using VectorNNTPD.BackFillerService.Hosting.Systemd; @@ -26,6 +27,7 @@ using VectorNNTPD.Common.Networking.Certificates; using VectorNNTPD.Common.Articles.Parsing; using VectorNNTPD.Common.Configuration; +using RabbitMQ.Client; using System.Net.Sockets; using System.Net; @@ -37,7 +39,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Hosting public sealed class BackFillerHostCompositionTests { /// - /// Hosted service order (eleven BackFiller-hosted services ending with + /// Hosted service order (thirteen BackFiller-hosted services ending with /// after ), /// application-service triple (DNS, ACME, listener), and key singleton aliases match the current composition contract. /// @@ -50,7 +52,7 @@ public void AddBackFillerHosting_registers_rabbitmq_then_article_work_consumers( var hosted = host.Services.GetServices() .Where(static service => service.GetType().Assembly == typeof(BackFillerServiceCollectionExtensions).Assembly) .ToArray(); - Assert.Equal(11, hosted.Length); + Assert.Equal(13, hosted.Length); Assert.IsType(hosted[0]); Assert.IsType(hosted[1]); Assert.IsType(hosted[2]); @@ -62,9 +64,11 @@ public void AddBackFillerHosting_registers_rabbitmq_then_article_work_consumers( Assert.IsType( host.Services.GetRequiredService()); Assert.Same(host.Services.GetRequiredService(), hosted[7]); - Assert.Same(host.Services.GetRequiredService(), hosted[8]); - Assert.Same(host.Services.GetRequiredService(), hosted[9]); - Assert.IsType(hosted[10]); + Assert.Same(host.Services.GetRequiredService(), hosted[8]); + Assert.Same(host.Services.GetRequiredService(), hosted[9]); + Assert.Same(host.Services.GetRequiredService(), hosted[10]); + Assert.Same(host.Services.GetRequiredService(), hosted[11]); + Assert.IsType(hosted[12]); Assert.Same( host.Services.GetRequiredService(), host.Services.GetRequiredService()); @@ -128,6 +132,49 @@ public void AddBackFillerHosting_registers_rabbitmq_then_article_work_consumers( or "NewsDateParser"); } + /// + /// Production DI defaults leave lab isolation and stage profiling off, keep the plain TCP transport, + /// and wire the pipelined response publisher with a confirm bound matching RabbitMQ.Client 7.2.2's + /// default outstanding-confirm rate limiter (128). + /// + [Fact] + public void AddBackFillerHosting_production_defaults_isolate_lab_features_and_pipeline_publisher() + { + using var host = CreateHost(); + + var lab = host.Services.GetRequiredService>().Value; + Assert.False(lab.Enabled); + Assert.False(lab.RewritePublishedDestinations); + + var profiling = host.Services.GetRequiredService>().Value; + Assert.False(profiling.Enabled); + + Assert.IsType(host.Services.GetRequiredService()); + Assert.Same( + host.Services.GetRequiredService(), + host.Services.GetRequiredService()); + Assert.Equal( + RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations, + ArticleWorkResponsePublisher.MaxConcurrentConfirmedPublications); + Assert.Equal(128, ArticleWorkResponsePublisher.MaxConcurrentConfirmedPublications); + + // Production CreatePublishChannelAsync must not use the two-arg CreateChannelOptions overload: + // that leaves OutstandingPublisherConfirmationsRateLimiter null (unlimited). Explicit limiter required. + var unboundedDefaults = new CreateChannelOptions( + publisherConfirmationsEnabled: true, + publisherConfirmationTrackingEnabled: true); + Assert.Null(unboundedDefaults.OutstandingPublisherConfirmationsRateLimiter); + + var productionOptions = new CreateChannelOptions( + publisherConfirmationsEnabled: true, + publisherConfirmationTrackingEnabled: true, + outstandingPublisherConfirmationsRateLimiter: new ThrottlingRateLimiter( + RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations)); + Assert.True(productionOptions.PublisherConfirmationsEnabled); + Assert.True(productionOptions.PublisherConfirmationTrackingEnabled); + Assert.NotNull(productionOptions.OutstandingPublisherConfirmationsRateLimiter); + } + /// /// The Cloudflare API client and ACME component factory resolve from Common. /// Listener A/AAAA publication stays withheld until bind, and BackFiller defines no DNS query client. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LabUpstreamIsolationCompositionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LabUpstreamIsolationCompositionTests.cs new file mode 100644 index 00000000..0c3f5c0f --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LabUpstreamIsolationCompositionTests.cs @@ -0,0 +1,241 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using VectorNNTPD.BackFillerService.Accounts; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Hosting; +using VectorNNTPD.BackFillerService.Logging; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Cloudflare; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Messaging.RabbitMq; +using System.Net; +using System.Net.Sockets; + +namespace VectorNNTPD.BackFillerService.Tests.Hosting +{ + /// + /// Verifies DI wires the lab upstream allowlist only when explicitly enabled. + /// + public sealed class LabUpstreamIsolationCompositionTests + { + [Fact] + public void Disabled_by_default_resolves_TcpNntpTransportFactory() + { + using var host = CreateHost([]); + var transport = host.Services.GetRequiredService(); + Assert.IsType(transport); + Assert.Same(transport, host.Services.GetRequiredService()); + } + + [Fact] + public void Enabled_with_valid_config_resolves_allowlist_decorator() + { + using var host = CreateHost( + [ + new KeyValuePair("BackFiller:LabUpstreamIsolation:Enabled", "true"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedHost", "127.0.0.1"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedPort", "11119"), + ]); + var transport = host.Services.GetRequiredService(); + Assert.IsType(transport); + Assert.IsType(host.Services.GetRequiredService()); + Assert.NotSame(transport, host.Services.GetRequiredService()); + } + + [Fact] + public void Enabled_with_invalid_config_fails_ValidateOnStart() + { + var builder = CreateBuilder( + [ + new KeyValuePair("BackFiller:LabUpstreamIsolation:Enabled", "true"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedHost", "localhost"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedPort", "11119"), + ]); + using var host = builder.Build(); + var ex = Assert.Throws(() => + _ = host.Services.GetRequiredService>().Value); + Assert.Contains("AllowedHost", ex.Message, StringComparison.Ordinal); + } + + /// + /// Proves the DI-selected allowlist activates on + /// (not merely that the type is registered). A WAN destination must fail closed before TCP. + /// + [Fact] + public async Task Enabled_DI_factory_ConnectAsync_rejects_wan_before_socket() + { + using var host = CreateHost( + [ + new KeyValuePair("BackFiller:LabUpstreamIsolation:Enabled", "true"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedHost", "127.0.0.1"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedPort", "11119"), + ]); + var transport = host.Services.GetRequiredService(); + Assert.IsType(transport); + + var wan = new BackFillerProviderDefinition( + "Giganews", + "news.example.com", + 563, + UseTls: true, + Username: null, + Password: null, + MinSessions: 0, + MaxSessions: 1); + + var ex = await Assert.ThrowsAsync(() => + transport.ConnectAsync(wan, NntpSessionOptions.Default, CancellationToken.None)); + Assert.Contains("Lab upstream isolation refused", ex.Message, StringComparison.Ordinal); + Assert.DoesNotContain("SocketException", ex.ToString(), StringComparison.Ordinal); + } + + /// + /// Proves RejectTls is enforced through the composed factory for an otherwise allowlisted host/port. + /// + [Fact] + public async Task Enabled_DI_factory_ConnectAsync_rejects_tls_without_rewriting() + { + using var host = CreateHost( + [ + new KeyValuePair("BackFiller:LabUpstreamIsolation:Enabled", "true"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedHost", "127.0.0.1"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedPort", "11119"), + ]); + var transport = host.Services.GetRequiredService(); + var tlsPeer = new BackFillerProviderDefinition( + "Giganews", + "127.0.0.1", + 11119, + UseTls: true, + Username: null, + Password: null, + MinSessions: 0, + MaxSessions: 1); + + var ex = await Assert.ThrowsAsync(() => + transport.ConnectAsync(tlsPeer, NntpSessionOptions.Default, CancellationToken.None)); + Assert.Contains("UseTls=true", ex.Message, StringComparison.Ordinal); + Assert.Contains("not rewritten", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Rewrite_without_Enabled_fails_ValidateOnStart() + { + var builder = CreateBuilder( + [ + new KeyValuePair("BackFiller:LabUpstreamIsolation:Enabled", "false"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:RewritePublishedDestinations", "true"), + ]); + using var host = builder.Build(); + var ex = Assert.Throws(() => + _ = host.Services.GetRequiredService>().Value); + Assert.Contains("RewritePublishedDestinations", ex.Message, StringComparison.Ordinal); + Assert.Contains("Enabled=true", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rewrite_with_RejectTls_false_fails_ValidateOnStart() + { + var builder = CreateBuilder( + [ + new KeyValuePair("BackFiller:LabUpstreamIsolation:Enabled", "true"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedHost", "127.0.0.1"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedPort", "11119"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:RejectTls", "false"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:RewritePublishedDestinations", "true"), + ]); + using var host = builder.Build(); + var ex = Assert.Throws(() => + _ = host.Services.GetRequiredService>().Value); + Assert.Contains("RejectTls", ex.Message, StringComparison.Ordinal); + } + + /// + /// Real DI composition: rewrite + allowlist retarget published providers to 127.0.0.1:11119 + /// without mutating FakeProviderAccountSource rows (stand-in for MySQL). + /// + [Fact] + public async Task DI_rewrite_publishes_loopback_and_allowlist_rejects_wan() + { + using var host = CreateHost( + [ + new KeyValuePair("BackFiller:LabUpstreamIsolation:Enabled", "true"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedHost", "127.0.0.1"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:AllowedPort", "11119"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:RejectTls", "true"), + new KeyValuePair("BackFiller:LabUpstreamIsolation:RewritePublishedDestinations", "true"), + ]); + + var source = (FakeProviderAccountSource)host.Services.GetRequiredService(); + source.Rows = [ProviderAccountTestRows.Create()]; + var accounts = host.Services.GetRequiredService(); + await accounts.StartAsync(CancellationToken.None); + + var published = Assert.Single(accounts.PublishedProviders); + Assert.Equal("127.0.0.1", published.Host); + Assert.Equal(11119, published.Port); + Assert.False(published.UseTls); + Assert.Null(published.Username); + Assert.Null(published.Password); + Assert.Equal("news.example.test", source.Rows[0].Hostname); + Assert.Equal(ProviderAccountTestRows.SecretPassword, source.Rows[0].Password); + + var transport = host.Services.GetRequiredService(); + Assert.IsType(transport); + var wan = new BackFillerProviderDefinition( + "Giganews", + "news.example.com", + 563, + UseTls: true, + Username: "nntp-user", + Password: ProviderAccountTestRows.SecretPassword, + MinSessions: 0, + MaxSessions: 1); + var refused = await Assert.ThrowsAsync(() => + transport.ConnectAsync(wan, NntpSessionOptions.Default, CancellationToken.None)); + Assert.Contains("Lab upstream isolation refused", refused.Message, StringComparison.Ordinal); + + await accounts.DisposeAsync(); + } + + private static IHost CreateHost(IEnumerable> extra) => + CreateBuilder(extra).Build(); + + private static HostApplicationBuilder CreateBuilder(IEnumerable> extra) + { + var builder = Host.CreateApplicationBuilder([]); + var pairs = BackFillerTestOptions.CreateValidConfigurationPairs(); + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + pairs["BackFiller:BindPortTls"] = ((IPEndPoint)listener.LocalEndpoint).Port.ToString(); + listener.Stop(); + pairs["BackFiller:BindAddress:0"] = "*"; + pairs["BackFiller:Logging:File:LogDir"] = Directory.CreateTempSubdirectory("bf-lab-iso-").FullName; + foreach (var pair in extra) + { + pairs[pair.Key] = pair.Value; + } + + builder.Configuration.AddInMemoryCollection(pairs); + builder.Services.AddSingleton(new FakeLocalIpAddressAssignee(assignAll: true)); + builder.Services.AddSingleton(new NoOpCloudflareDnsReconciler()); + builder.Services.AddSingleton( + new FakePhysicalMemoryProvider(64L * 1024 * 1024 * 1024)); + builder.Services.AddSingleton(new FakeBackFillerRabbitMqConnectionFactory()); + builder.Services.AddSingleton(new FakeProviderAccountSource()); + builder.ConfigureBackFillerLogging(); + builder.ConfigureBackFillerPlatformHosting(); + builder.AddBackFillerHosting(); + SuccessfulNntpDbConnectionFactory.Replace(builder.Services); + // Avoid live MySQL for shared-config reads when account StartAsync is exercised. + builder.Services.RemoveAll(); + builder.Services.AddSingleton(); + return builder; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/LabUpstreamAllowlistNntpTransportFactoryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/LabUpstreamAllowlistNntpTransportFactoryTests.cs new file mode 100644 index 00000000..aefadf17 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/LabUpstreamAllowlistNntpTransportFactoryTests.cs @@ -0,0 +1,266 @@ +using Microsoft.Extensions.Options; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Nntp; + +namespace VectorNNTPD.BackFillerService.Tests.Nntp +{ + public sealed class LabUpstreamAllowlistNntpTransportFactoryTests + { + [Fact] + public async Task Valid_loopback_destination_invokes_inner_after_validation() + { + var inner = new RecordingTransportFactory(); + var factory = CreateAllowlist(inner, port: 11119); + var provider = Provider("Giganews", "127.0.0.1", 11119, useTls: false); + await using var stream = await factory.ConnectAsync(provider, NntpSessionOptions.Default, CancellationToken.None); + Assert.Equal(1, inner.ConnectCount); + Assert.Same(provider, inner.LastProvider); + Assert.NotNull(stream); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("localhost")] + [InlineData("news.example.com")] + [InlineData("8.8.8.8")] + [InlineData("10.0.0.1")] + [InlineData("::1")] + [InlineData("2001:db8::1")] + public async Task Invalid_hosts_never_invoke_inner(string? host) + { + var inner = new RecordingTransportFactory(); + var factory = CreateAllowlist(inner, port: 11119); + var provider = Provider("Giganews", host ?? "", 11119, useTls: false); + var ex = await Assert.ThrowsAsync(() => + factory.ConnectAsync(provider, NntpSessionOptions.Default, CancellationToken.None)); + Assert.Contains("Lab upstream isolation refused", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, inner.ConnectCount); + } + + [Fact] + public async Task Mismatched_port_never_invokes_inner() + { + var inner = new RecordingTransportFactory(); + var factory = CreateAllowlist(inner, port: 11119); + var provider = Provider("Giganews", "127.0.0.1", 119, useTls: false); + await Assert.ThrowsAsync(() => + factory.ConnectAsync(provider, NntpSessionOptions.Default, CancellationToken.None)); + Assert.Equal(0, inner.ConnectCount); + } + + [Fact] + public async Task Tls_provider_rejected_when_RejectTls_true() + { + var inner = new RecordingTransportFactory(); + var factory = CreateAllowlist(inner, port: 11119, rejectTls: true); + var provider = Provider("Giganews", "127.0.0.1", 11119, useTls: true); + var ex = await Assert.ThrowsAsync(() => + factory.ConnectAsync(provider, NntpSessionOptions.Default, CancellationToken.None)); + Assert.Contains("UseTls=true", ex.Message, StringComparison.Ordinal); + Assert.Contains("not rewritten", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(0, inner.ConnectCount); + } + + [Fact] + public async Task Retry_path_reapplies_validation_each_connect() + { + var inner = new RecordingTransportFactory(); + var factory = CreateAllowlist(inner, port: 11119); + var bad = Provider("Eweka", "news.example.com", 563, useTls: true); + var good = Provider("Giganews", "127.0.0.1", 11119, useTls: false); + + await Assert.ThrowsAsync(() => + factory.ConnectAsync(bad, NntpSessionOptions.Default, CancellationToken.None)); + Assert.Equal(0, inner.ConnectCount); + + await using var stream = await factory.ConnectAsync(good, NntpSessionOptions.Default, CancellationToken.None); + Assert.Equal(1, inner.ConnectCount); + + await Assert.ThrowsAsync(() => + factory.ConnectAsync(bad, NntpSessionOptions.Default, CancellationToken.None)); + Assert.Equal(1, inner.ConnectCount); + } + + [Fact] + public async Task One_local_account_does_not_mask_another_disallowed_destination() + { + var inner = new RecordingTransportFactory(); + var factory = CreateAllowlist(inner, port: 11119); + await using var _ = await factory.ConnectAsync( + Provider("Giganews", "127.0.0.1", 11119, false), + NntpSessionOptions.Default, + CancellationToken.None); + await Assert.ThrowsAsync(() => + factory.ConnectAsync( + Provider("Eweka", "198.18.0.1", 563, true), + NntpSessionOptions.Default, + CancellationToken.None)); + Assert.Equal(1, inner.ConnectCount); + } + + [Fact] + public void Constructor_rejects_disabled_or_incomplete_options() + { + var inner = new RecordingTransportFactory(); + Assert.Throws(() => + new LabUpstreamAllowlistNntpTransportFactory( + inner, + new LabUpstreamIsolationOptions { Enabled = false, AllowedPort = 11119 })); + Assert.Throws(() => + new LabUpstreamAllowlistNntpTransportFactory( + inner, + new LabUpstreamIsolationOptions + { + Enabled = true, + AllowedHost = "localhost", + AllowedPort = 11119, + })); + Assert.Throws(() => + new LabUpstreamAllowlistNntpTransportFactory( + inner, + new LabUpstreamIsolationOptions { Enabled = true, AllowedPort = null })); + } + + [Fact] + public void Validator_succeeds_when_disabled_regardless_of_port() + { + var result = new LabUpstreamIsolationOptionsValidator().Validate( + null, + new LabUpstreamIsolationOptions { Enabled = false, AllowedPort = null }); + Assert.True(result.Succeeded); + } + + [Theory] + [InlineData(null, null)] + [InlineData("localhost", 11119)] + [InlineData("127.0.0.1", null)] + [InlineData("127.0.0.1", 0)] + [InlineData("127.0.0.1", 70000)] + [InlineData("::1", 11119)] + public void Validator_fails_closed_when_enabled_and_invalid(string? host, int? port) + { + var options = new LabUpstreamIsolationOptions + { + Enabled = true, + AllowedHost = host ?? "", + AllowedPort = port, + }; + var result = new LabUpstreamIsolationOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + } + + [Fact] + public void Validator_accepts_enabled_loopback_port() + { + var result = new LabUpstreamIsolationOptionsValidator().Validate( + null, + new LabUpstreamIsolationOptions + { + Enabled = true, + AllowedHost = "127.0.0.1", + AllowedPort = 11119, + }); + Assert.True(result.Succeeded); + } + + [Fact] + public void Validator_accepts_enabled_rewrite_with_reject_tls() + { + var result = new LabUpstreamIsolationOptionsValidator().Validate( + null, + new LabUpstreamIsolationOptions + { + Enabled = true, + AllowedHost = "127.0.0.1", + AllowedPort = 11119, + RejectTls = true, + RewritePublishedDestinations = true, + }); + Assert.True(result.Succeeded); + } + + [Fact] + public void Validator_fails_rewrite_when_disabled() + { + var result = new LabUpstreamIsolationOptionsValidator().Validate( + null, + new LabUpstreamIsolationOptions + { + Enabled = false, + RewritePublishedDestinations = true, + }); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + static f => f.Contains("RewritePublishedDestinations", StringComparison.Ordinal)); + } + + [Fact] + public void Validator_fails_rewrite_when_RejectTls_false() + { + var result = new LabUpstreamIsolationOptionsValidator().Validate( + null, + new LabUpstreamIsolationOptions + { + Enabled = true, + AllowedHost = "127.0.0.1", + AllowedPort = 11119, + RejectTls = false, + RewritePublishedDestinations = true, + }); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, static f => f.Contains("RejectTls", StringComparison.Ordinal)); + } + + [Fact] + public void Disabled_path_uses_inner_factory_directly_without_decorator_construction() + { + // Composition contract: when Enabled=false, DI returns TcpNntpTransportFactory (or recording inner) + // without wrapping. Unit-level: constructing the decorator with Enabled=false throws. + Assert.Throws(() => + new LabUpstreamAllowlistNntpTransportFactory( + new RecordingTransportFactory(), + new LabUpstreamIsolationOptions { Enabled = false, AllowedPort = 1 })); + } + + private static LabUpstreamAllowlistNntpTransportFactory CreateAllowlist( + RecordingTransportFactory inner, + int port, + bool rejectTls = true) => + new( + inner, + new LabUpstreamIsolationOptions + { + Enabled = true, + AllowedHost = "127.0.0.1", + AllowedPort = port, + RejectTls = rejectTls, + }); + + private static BackFillerProviderDefinition Provider( + string backbone, + string host, + int port, + bool useTls) => + new(backbone, host, port, useTls, null, null, 0, 1); + + private sealed class RecordingTransportFactory : INntpTransportFactory + { + public int ConnectCount { get; private set; } + + public BackFillerProviderDefinition? LastProvider { get; private set; } + + public Task ConnectAsync( + BackFillerProviderDefinition provider, + NntpSessionOptions options, + CancellationToken cancellationToken) + { + ConnectCount++; + LastProvider = provider; + return Task.FromResult(new MemoryStream()); + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpPayloadScratchPoolTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpPayloadScratchPoolTests.cs index 40bbc8f1..12ccb258 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpPayloadScratchPoolTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpPayloadScratchPoolTests.cs @@ -5,20 +5,21 @@ namespace VectorNNTPD.BackFillerService.Tests.Nntp { /// - /// Rent, growth, trim, and disposal of payload scratch. - /// The retained request after a payload read is 1 MiB. A larger rental is returned once that read finishes. + /// Rent, growth, release, and disposal of payload scratch. + /// After each ARTICLE payload finishes (including when the consumer throws), the scratch lease is returned. + /// The next ARTICLE rents again. Disposal must not double-return. /// public sealed class NntpPayloadScratchPoolTests { - /// Request size the reader keeps for later ordinary articles. + /// Initial rent request size for ordinary articles (matches production). private const int WorkingSetBytes = 1024 * 1024; /// Article ceiling used when a test must be allowed to pass 1 MiB. private const int MaximumArticleBytes = 5 * 1024 * 1024; - /// Two small payloads on one reader rent a single scratch. + /// Two small payloads on one reader each return their scratch; bytes stay correct. [Fact] - public async Task Ordinary_payloads_reuse_one_scratch_without_corruption() + public async Task Ordinary_payloads_return_scratch_between_articles_without_corruption() { var first = Pattern(48); var second = Pattern(80, seed: 9); @@ -26,14 +27,14 @@ public async Task Ordinary_payloads_reuse_one_scratch_without_corruption() using var reader = Reader(Frame(first).Concat(Frame(second)).ToArray(), pool); Assert.Equal(first, await ReadExactAsync(reader, WorkingSetBytes)); - var rentsAfterFirst = pool.RentCount; - Assert.Equal(second, await ReadExactAsync(reader, WorkingSetBytes)); + Assert.Equal(1, pool.RentCount); + Assert.Equal(1, pool.ReturnCount); + Assert.Equal(0, pool.OutstandingCount); - Assert.Equal(rentsAfterFirst, pool.RentCount); - Assert.Equal(0, pool.ReturnCount); - Assert.Equal(1, pool.OutstandingCount); - Assert.True(pool.OutstandingRequests().Single() <= WorkingSetBytes); - Assert.True(pool.OutstandingLengths().Single() >= pool.OutstandingRequests().Single()); + Assert.Equal(second, await ReadExactAsync(reader, WorkingSetBytes)); + Assert.Equal(2, pool.RentCount); + Assert.Equal(2, pool.ReturnCount); + Assert.Equal(0, pool.OutstandingCount); } /// An array longer than the rental request still carries the payload and is returned once. @@ -45,8 +46,9 @@ public async Task Rented_array_longer_than_the_request_is_returned_once() using var reader = Reader(Frame(payload), pool); Assert.Equal(payload, await ReadExactAsync(reader, maxBytes: 256)); Assert.Equal(1, pool.RentCount); - Assert.True(pool.OutstandingLengths().Single() >= pool.OutstandingRequests().Single() + 128); - Assert.Equal(0, pool.ReturnCount); + Assert.Equal(1, pool.ReturnCount); + Assert.Equal(0, pool.OutstandingCount); + Assert.True(pool.ReturnedArrays.Single().Length >= 256 + 128); reader.Dispose(); reader.Dispose(); @@ -54,26 +56,27 @@ public async Task Rented_array_longer_than_the_request_is_returned_once() Assert.Equal(0, pool.OutstandingCount); } - /// A payload that crosses 1 MiB returns the replaced scratch, and the destuffed bytes match. + /// Growth returns replaced buffers; the final lease is released after the article. [Fact] - public async Task Growth_returns_the_replaced_buffer_and_preserves_received_bytes() + public async Task Growth_returns_replaced_buffers_and_releases_after_article() { var payload = Pattern(WorkingSetBytes + 64); var pool = CreatePooled(); using (var reader = Reader(Frame(payload), pool)) { Assert.Equal(payload, await ReadExactAsync(reader, MaximumArticleBytes)); + Assert.True(pool.RentCount > 1); + Assert.Equal(0, pool.OutstandingCount); + Assert.Equal(pool.RentCount, pool.ReturnCount); Assert.Contains(pool.ReturnedRequests, static request => request > 0 && request <= WorkingSetBytes); - Assert.Equal(WorkingSetBytes, pool.OutstandingRequests().Single()); - Assert.True(pool.OutstandingLengths().Single() >= WorkingSetBytes); } Assert.Equal(pool.RentCount, pool.ReturnCount); } - /// Exactly 1 MiB does not grow. One byte past it is trimmed back to a 1 MiB request. + /// Exactly 1 MiB rents once and returns. One byte past it grows then releases all leases. [Fact] - public async Task Working_set_boundary_trims_only_after_the_request_grows() + public async Task Working_set_boundary_releases_after_exact_and_grown_payloads() { var exact = Pattern(WorkingSetBytes); var exactPool = CreatePooled(); @@ -81,8 +84,8 @@ public async Task Working_set_boundary_trims_only_after_the_request_grows() { Assert.Equal(exact, await ReadExactAsync(reader, MaximumArticleBytes)); Assert.Equal(1, exactPool.RentCount); - Assert.Equal(WorkingSetBytes, exactPool.OutstandingRequests().Single()); - Assert.Empty(exactPool.ReturnedRequests); + Assert.Equal(1, exactPool.ReturnCount); + Assert.Equal(0, exactPool.OutstandingCount); } var over = Pattern(WorkingSetBytes + 1); @@ -91,13 +94,14 @@ public async Task Working_set_boundary_trims_only_after_the_request_grows() { Assert.Equal(over, await ReadExactAsync(reader, MaximumArticleBytes)); Assert.True(overPool.RentCount > 1); - Assert.Equal(WorkingSetBytes, overPool.OutstandingRequests().Single()); + Assert.Equal(0, overPool.OutstandingCount); + Assert.Equal(overPool.RentCount, overPool.ReturnCount); } } - /// A 5 MiB article is received, then the next ordinary article reuses the trimmed 1 MiB request. + /// A 5 MiB article releases its scratch; the next ordinary article rents again and succeeds. [Fact] - public async Task Large_article_is_trimmed_before_the_next_ordinary_article() + public async Task Large_article_release_then_ordinary_article_rents_again() { var large = Pattern(MaximumArticleBytes); var ordinary = Pattern(120, seed: 3); @@ -105,18 +109,20 @@ public async Task Large_article_is_trimmed_before_the_next_ordinary_article() using var reader = Reader(Frame(large).Concat(Frame(ordinary)).ToArray(), pool); Assert.Equal(large, await ReadExactAsync(reader, MaximumArticleBytes)); - Assert.Equal(WorkingSetBytes, pool.OutstandingRequests().Single()); + Assert.Equal(0, pool.OutstandingCount); var rentsAfterLarge = pool.RentCount; + var returnsAfterLarge = pool.ReturnCount; + Assert.Equal(rentsAfterLarge, returnsAfterLarge); Assert.Equal(ordinary, await ReadExactAsync(reader, MaximumArticleBytes)); - Assert.Equal(rentsAfterLarge, pool.RentCount); - Assert.Equal(WorkingSetBytes, pool.OutstandingRequests().Single()); - Assert.True(pool.OutstandingLengths().Single() >= WorkingSetBytes); + Assert.True(pool.RentCount > rentsAfterLarge); + Assert.Equal(0, pool.OutstandingCount); + Assert.Equal(pool.RentCount, pool.ReturnCount); } /// Bytes still in the receive buffer after a large article are the following article. [Fact] - public async Task Trim_between_articles_keeps_the_unread_tail() + public async Task Release_between_articles_keeps_the_unread_tail() { var large = Pattern(WorkingSetBytes + 32); var ordinary = Pattern(24, seed: 5); @@ -124,10 +130,10 @@ public async Task Trim_between_articles_keeps_the_unread_tail() using var reader = Reader(Frame(large).Concat(Frame(ordinary)).ToArray(), pool); Assert.Equal(large, await ReadExactAsync(reader, MaximumArticleBytes)); - Assert.Equal(WorkingSetBytes, pool.OutstandingRequests().Single()); + Assert.Equal(0, pool.OutstandingCount); Assert.Equal(ordinary, await ReadExactAsync(reader, MaximumArticleBytes)); + Assert.Equal(0, pool.OutstandingCount); } - /// An immediate terminator rents nothing, and a second disposal does not return a buffer. [Fact] public async Task Empty_payload_and_repeated_disposal_return_nothing() @@ -142,9 +148,12 @@ public async Task Empty_payload_and_repeated_disposal_return_nothing() Assert.Equal(0, pool.ReturnCount); } - /// The callback observes the scratch before disposal, and disposal returns it once. + /// + /// The callback still owns the scratch while it runs; after it throws, finally returns the lease. + /// Dispose must not double-return. + /// [Fact] - public async Task Callback_failure_does_not_return_the_active_scratch() + public async Task Callback_failure_returns_scratch_only_after_consume_unwinds() { var payload = Pattern(48); var pool = new TrackingArrayPool(slackBytes: 32); @@ -156,14 +165,13 @@ public async Task Callback_failure_does_not_return_the_active_scratch() memory => FailWhileScratchIsLive(memory, payload, pool), CancellationToken.None)); Assert.Equal("consumer failed", exception.Message); - Assert.Equal(1, pool.OutstandingCount); - Assert.True(pool.ReturnCount < pool.RentCount); + Assert.Equal(0, pool.OutstandingCount); + Assert.Equal(pool.RentCount, pool.ReturnCount); reader.Dispose(); Assert.Equal(pool.RentCount, pool.ReturnCount); Assert.Equal(0, pool.OutstandingCount); } - /// A cancelled read returns any scratch it rented when the reader is disposed. [Fact] public async Task Cancellation_before_the_payload_stays_balanced() @@ -257,11 +265,11 @@ public async Task Maximum_size_violation_preserves_the_error_and_balances_the_po } /// - /// Prints scratch rent traffic for ordinary, large, and repeated-session reads. + /// Prints scratch rent traffic for ordinary, large, and repeated-session reads under Policy B. /// Counts are evidence. They are not a pass/fail budget. /// [Fact] - public async Task Measure_payload_scratch_rent_and_retained_request() + public async Task Measure_payload_scratch_rent_and_release_after_article() { var ordinary = Pattern(4096); var ordinaryWire = Frame(ordinary); @@ -271,11 +279,13 @@ public async Task Measure_payload_scratch_rent_and_retained_request() for (var i = 0; i < 16; i++) { Assert.Equal(ordinary.Length, await ReadLengthAsync(reader, MaximumArticleBytes)); + Assert.Equal(0, pool.OutstandingCount); } } Console.WriteLine( $"payload-scratch ordinary n=16 rents={pool.RentCount} returns={pool.ReturnCount} outstanding={pool.OutstandingCount}"); + Assert.Equal(16, pool.RentCount); Assert.Equal(pool.RentCount, pool.ReturnCount); pool.ResetCounts(); @@ -283,15 +293,17 @@ public async Task Measure_payload_scratch_rent_and_retained_request() using (var reader = Reader(Frame(large).Concat(ordinaryWire).ToArray(), pool)) { Assert.Equal(large.Length, await ReadLengthAsync(reader, MaximumArticleBytes)); - var afterLarge = pool.OutstandingRequests().Single(); + Assert.Equal(0, pool.OutstandingCount); var rentsForLarge = pool.RentCount; Assert.Equal(ordinary.Length, await ReadLengthAsync(reader, MaximumArticleBytes)); Console.WriteLine( - $"payload-scratch large-then-ordinary largeRents={rentsForLarge} retainedRequest={afterLarge} ordinaryExtraRents={pool.RentCount - rentsForLarge} outstandingRequest={pool.OutstandingRequests().Single()}"); - Assert.Equal(WorkingSetBytes, afterLarge); - Assert.Equal(0, pool.RentCount - rentsForLarge); + $"payload-scratch large-then-ordinary largeRents={rentsForLarge} ordinaryExtraRents={pool.RentCount - rentsForLarge} outstanding={pool.OutstandingCount}"); + Assert.True(pool.RentCount > rentsForLarge); + Assert.Equal(0, pool.OutstandingCount); } + Assert.Equal(pool.RentCount, pool.ReturnCount); + pool.ResetCounts(); long secondArticleAllocated; using (var reader = Reader(Frame(large).Concat(Frame(large)).ToArray(), pool)) @@ -321,7 +333,6 @@ public async Task Measure_payload_scratch_rent_and_retained_request() Assert.Equal(0, pool.OutstandingCount); Assert.Equal(pool.RentCount, pool.ReturnCount); } - /// Creates the same pool shape the reader uses in production. private static TrackingArrayPool CreatePooled() => new(ArrayPool.Create(maxArrayLength: 8 * 1024 * 1024, maxArraysPerBucket: 4)); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Retention/ArticleRetentionMetricsTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Retention/ArticleRetentionMetricsTests.cs new file mode 100644 index 00000000..7b6b63b6 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Retention/ArticleRetentionMetricsTests.cs @@ -0,0 +1,241 @@ +using System.Diagnostics.Metrics; +using VectorNNTPD.BackFillerService.Retention; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; + +namespace VectorNNTPD.BackFillerService.Tests.Retention +{ + /// + /// Serializes so MeterListener observes a single MeterName at a time. + /// + [CollectionDefinition(nameof(ArticleRetentionMetricsTests), DisableParallelization = true)] + public sealed class ArticleRetentionMetricsTestsCollection + { + } + + /// + /// Observable retention gauges mirror occupancy via MeterListener. + /// + [Collection(nameof(ArticleRetentionMetricsTests))] + public sealed class ArticleRetentionMetricsTests + { + /// Fixed clock for ManualTimeProvider. + private static readonly DateTimeOffset Start = new(2026, 9, 26, 12, 0, 0, TimeSpan.Zero); + + /// Empty authority reports zero bytes and zero entries on both gauges. + [Fact] + public async Task Empty_authority_observes_zero_bytes_and_zero_entries() + { + await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 1024 * 1024); + using var metrics = new ArticleRetentionMetrics(authority); + var sample = Observe(); + Assert.Equal(0L, sample.PayloadBytes); + Assert.Equal(0L, sample.Count); + } + + /// One retain updates both gauges to the article ArtSize and count 1. + [Fact] + public async Task Retain_updates_payload_bytes_and_count_gauges() + { + await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 1024 * 1024); + using var metrics = new ArticleRetentionMetrics(authority); + var prepared = RetentionTestArticles.RetainPrepared(authority, ""); + + var sample = Observe(); + Assert.Equal(prepared.Record.ArtSize, sample.PayloadBytes); + Assert.Equal(1L, sample.Count); + Assert.Equal(authority.RetainedPayloadBytes, sample.PayloadBytes); + Assert.Equal(authority.RetainedCount, sample.Count); + } + + /// Two distinct Message-IDs aggregate bytes and count on the gauges. + [Fact] + public async Task Multiple_retains_aggregate_on_gauges() + { + await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 1024 * 1024); + using var metrics = new ArticleRetentionMetrics(authority); + var first = RetentionTestArticles.RetainPrepared(authority, "", "one\r\n"); + var second = RetentionTestArticles.RetainPrepared(authority, "", "two-bytes\r\n"); + + var sample = Observe(); + Assert.Equal(first.Record.ArtSize + second.Record.ArtSize, sample.PayloadBytes); + Assert.Equal(2L, sample.Count); + } + + /// TTL expiry that clears the authority also clears both gauges. + [Fact] + public async Task Expiry_updates_gauges_to_zero() + { + var time = new ManualTimeProvider(Start); + await using var authority = ArticleRetentionAuthorityTests.Create(time, maxBytes: 1024 * 1024, ttl: TimeSpan.FromSeconds(10)); + using var metrics = new ArticleRetentionMetrics(authority); + _ = RetentionTestArticles.RetainPrepared(authority, ""); + Assert.Equal(1L, Observe().Count); + + time.Advance(TimeSpan.FromSeconds(10)); + ArticleRetentionAuthorityTests.AssertNotFound(authority, ""); + + var sample = Observe(); + Assert.Equal(0L, sample.PayloadBytes); + Assert.Equal(0L, sample.Count); + } + + /// Committed ARTICLE delivery unindexes and both gauges return to zero. + [Fact] + public async Task Commit_delivery_clears_gauges() + { + await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 1024 * 1024); + using var metrics = new ArticleRetentionMetrics(authority); + var prepared = RetentionTestArticles.RetainPrepared(authority, ""); + using (var claim = ArticleRetentionAuthorityTests.Claim(authority, "", prepared.ArtData)) + { + claim.Commit(); + } + + var sample = Observe(); + Assert.Equal(0L, sample.PayloadBytes); + Assert.Equal(0L, sample.Count); + Assert.Equal(0, authority.RetainedPayloadBytes); + Assert.Equal(0, authority.RetainedCount); + } + + /// Repeated SweepExpired after empty retention does not drive gauges negative. + [Fact] + public async Task Repeated_sweep_on_empty_authority_stays_at_zero() + { + await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 1024 * 1024); + using var metrics = new ArticleRetentionMetrics(authority); + Assert.Equal(0, authority.SweepExpired()); + Assert.Equal(0, authority.SweepExpired()); + + var sample = Observe(); + Assert.Equal(0L, sample.PayloadBytes); + Assert.Equal(0L, sample.Count); + } + + /// Concurrent retains and gauge observes stay non-negative and match authority totals. + [Fact] + public async Task Concurrent_retain_and_observe_stay_consistent_with_authority() + { + await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 16 * 1024 * 1024); + using var metrics = new ArticleRetentionMetrics(authority); + var barrier = new Barrier(4); + var tasks = new Task[3]; + for (var i = 0; i < tasks.Length; i++) + { + var index = i; + tasks[index] = Task.Run(() => + { + barrier.SignalAndWait(); + for (var n = 0; n < 20; n++) + { + _ = RetentionTestArticles.RetainPrepared( + authority, + $"<{index}-{n}@metrics.test>", + $"body-{index}-{n}\r\n"); + } + }); + } + + var observeTask = Task.Run(() => + { + barrier.SignalAndWait(); + for (var n = 0; n < 40; n++) + { + // During concurrent retains, gauges and a later property read are not one + // atomic snapshot; only require non-negative observations mid-flight. + var sample = Observe(); + Assert.True(sample.PayloadBytes >= 0); + Assert.True(sample.Count >= 0); + } + }); + + await Task.WhenAll(tasks.Append(observeTask)); + + // After writers finish, gauges must match the authority (source of truth). + var final = Observe(); + Assert.Equal(60L, final.Count); + Assert.Equal(authority.RetainedPayloadBytes, final.PayloadBytes); + Assert.Equal(authority.RetainedCount, final.Count); + } + + /// Dispose is idempotent; a replacement metrics instance observes the same authority again. + [Fact] + public async Task Dispose_is_idempotent_and_replacement_observes_again() + { + await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 1024 * 1024); + var metrics = new ArticleRetentionMetrics(authority); + var prepared = RetentionTestArticles.RetainPrepared(authority, ""); + metrics.Dispose(); + metrics.Dispose(); + + using var replacement = new ArticleRetentionMetrics(authority); + var sample = Observe(); + Assert.Equal(prepared.Record.ArtSize, sample.PayloadBytes); + Assert.Equal(1L, sample.Count); + } + + /// Null authority is rejected. + [Fact] + public void Constructor_rejects_null_authority() + { + Assert.Throws(static () => new ArticleRetentionMetrics(null!)); + } + + /// Records one observe pass for both retention instruments. + private static (long PayloadBytes, long Count) Observe() + { + var (listener, holder) = StartListening(); + using (listener) + { + listener.RecordObservableInstruments(); + Assert.True(holder.PayloadBytes.HasValue); + Assert.True(holder.Count.HasValue); + return (holder.PayloadBytes.GetValueOrDefault(), holder.Count.GetValueOrDefault()); + } + } + + /// Creates a started listener for the article-retention meter instruments. + private static (MeterListener Listener, MeasurementHolder Holder) StartListening() + { + var holder = new MeasurementHolder(); + var listener = new MeterListener(); + listener.InstrumentPublished = static (instrument, meterListener) => + { + if (instrument.Meter.Name != ArticleRetentionMetrics.MeterName) + { + return; + } + + if (instrument.Name is ArticleRetentionMetrics.RetainedPayloadBytesInstrumentName + or ArticleRetentionMetrics.RetainedCountInstrumentName) + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback((instrument, measurement, _, _) => + { + if (instrument.Name == ArticleRetentionMetrics.RetainedPayloadBytesInstrumentName) + { + holder.PayloadBytes = measurement; + } + else if (instrument.Name == ArticleRetentionMetrics.RetainedCountInstrumentName) + { + holder.Count = measurement; + } + }); + listener.Start(); + return (listener, holder); + } + + /// Holds the last observed gauge values for one MeterListener session. + private sealed class MeasurementHolder + { + /// Last sample. + public long? PayloadBytes; + + /// Last sample. + public long? Count; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs index 68f0cab1..bcbbe3c2 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs @@ -811,6 +811,15 @@ internal enum FakePublishConfirmBehavior internal sealed class FakeBackFillerRabbitMqPublishChannel(long generation) : IRabbitMqPublishChannel { + /// Guards for concurrent confirmed publishes. + private readonly Lock _publicationsGate = new(); + + /// Current confirm-wait depth for overlap observation. + private int _confirmInFlight; + + /// Peak confirm-wait depth observed on this channel. + private int _maxConfirmInFlight; + /// Broker publish-channel generation supplied at creation. public long Generation { get; } = generation; @@ -835,6 +844,9 @@ internal sealed class FakeBackFillerRabbitMqPublishChannel(long generation) : IR /// Optional hook invoked immediately after enqueue and before confirm simulation. public Action? AfterEnqueue { get; set; } + /// Peak number of overlapping confirm waits observed on this channel. + internal int MaxConfirmInFlight => Volatile.Read(ref _maxConfirmInFlight); + /// Publications accepted by this channel in invocation order. public List Publications { get; } = []; @@ -892,37 +904,63 @@ public async Task PublishConfirmedAsync(RabbitMqConfirmedPublication publication throw new InvalidOperationException("publish failed"); } - Publications.Add(publication); - Enqueued?.TrySetResult(); - AfterEnqueue?.Invoke(); - - switch (ConfirmBehavior) - { - case FakePublishConfirmBehavior.Confirm: - return; - case FakePublishConfirmBehavior.Nack: - throw new InvalidOperationException("publisher nack"); - case FakePublishConfirmBehavior.Unroutable: - throw new RabbitMqPublicationUnroutableException(); - case FakePublishConfirmBehavior.Timeout: - throw new OperationCanceledException("publisher confirm timeout", cancellationToken); - case FakePublishConfirmBehavior.CloseChannel: - IsOpen = false; - ChannelFaulted?.Invoke( - this, - new RabbitMqChannelFaultedEventArgs( - RabbitMqChannelFaultKind.ChannelShutdown, - "channel closed during confirmation")); - throw new InvalidOperationException("RabbitMQ publish channel closed during confirmation."); - default: + lock (_publicationsGate) + { + Publications.Add(publication); + } + + var inFlight = Interlocked.Increment(ref _confirmInFlight); + try + { + int observed; + while ((observed = Volatile.Read(ref _maxConfirmInFlight)) < inFlight + && Interlocked.CompareExchange(ref _maxConfirmInFlight, inFlight, observed) != observed) + { + } + + Enqueued?.TrySetResult(); + AfterEnqueue?.Invoke(); + + if (ConfirmBehavior == FakePublishConfirmBehavior.Wait) + { if (ConfirmGate is not null) { await ConfirmGate.Task.WaitAsync(cancellationToken).ConfigureAwait(false); - return; } + else + { + await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); + throw new OperationCanceledException(cancellationToken); + } + } - await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false); - throw new OperationCanceledException(cancellationToken); + // Re-read after an optional Wait gate so tests can switch outcome for overlapping callers. + switch (ConfirmBehavior) + { + case FakePublishConfirmBehavior.Confirm: + case FakePublishConfirmBehavior.Wait: + return; + case FakePublishConfirmBehavior.Nack: + throw new InvalidOperationException("publisher nack"); + case FakePublishConfirmBehavior.Unroutable: + throw new RabbitMqPublicationUnroutableException(); + case FakePublishConfirmBehavior.Timeout: + throw new OperationCanceledException("publisher confirm timeout", cancellationToken); + case FakePublishConfirmBehavior.CloseChannel: + IsOpen = false; + ChannelFaulted?.Invoke( + this, + new RabbitMqChannelFaultedEventArgs( + RabbitMqChannelFaultKind.ChannelShutdown, + "channel closed during confirmation")); + throw new InvalidOperationException("RabbitMQ publish channel closed during confirmation."); + default: + return; + } + } + finally + { + Interlocked.Decrement(ref _confirmInFlight); } } diff --git a/tests/VectorNNTPD.Common.Tests/Articles/CanonicalArticleFramingTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/CanonicalArticleFramingTests.cs index 6c26418a..19ad7714 100644 --- a/tests/VectorNNTPD.Common.Tests/Articles/CanonicalArticleFramingTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Articles/CanonicalArticleFramingTests.cs @@ -128,6 +128,55 @@ public void Malformed_spans_are_not_frameable() AssertRejected("Subject: s\r\n\0\r\n\r\nbody\r\n"u8.ToArray()); } + /// + /// A single bare LF between otherwise CRLF header lines is not frameable (RFC 3977 §3.1.1 exclusive CRLF). + /// Matches the observed .artifacts/Articles defect: one bare LF in the header region, exclusive CRLF body. + /// + [Fact] + public void Single_bare_lf_in_header_region_is_not_frameable() + { + var article = "Message-ID: \r\nPath: peer!for-mail\nLines: 1\r\n\r\nbody\r\n"u8.ToArray(); + var framing = CanonicalArticleFraming.Analyze(article); + Assert.False(framing.IsFrameable); + Assert.True(framing.BodyStart > 0); + } + + /// + /// Inserting CR before that bare LF yields exclusive CRLF and a frameable article; bodies stay unchanged. + /// + [Fact] + public void Normalizing_header_bare_lf_to_crlf_makes_article_frameable() + { + var bare = "Message-ID: \r\nPath: peer!for-mail\nLines: 1\r\n\r\nbody\r\n"u8.ToArray(); + Assert.False(CanonicalArticleFraming.Analyze(bare).IsFrameable); + + var normalized = NormalizeBareLfToCrlf(bare); + var framing = CanonicalArticleFraming.Analyze(normalized); + Assert.True(framing.IsFrameable); + Assert.Equal("body\r\n"u8, normalized.AsSpan(framing.BodyStart)); + Assert.Equal(bare.Length + 1, normalized.Length); + } + + /// Replaces each bare LF with CRLF; existing CRLF pairs are left intact. + private static byte[] NormalizeBareLfToCrlf(ReadOnlySpan source) + { + var buffer = new byte[source.Length + 16]; + var written = 0; + byte previous = 0; + foreach (var value in source) + { + if (value == (byte)'\n' && previous != (byte)'\r') + { + buffer[written++] = (byte)'\r'; + } + + buffer[written++] = value; + previous = value; + } + + return buffer.AsSpan(0, written).ToArray(); + } + /// Asserts is not a multiline block. private static void AssertRejected(byte[] article) { From 569ff182805055878042ecbe4217c934e2a7a002 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 28 Oct 2026 08:33:28 +0200 Subject: [PATCH 2/8] Add local ARTICLE peer harness and local-upstream stress support Introduce an env-gated loopback NNTP peer with fail-closed isolation checks and extend the E2E stress harness for LocalPeerSmoke workload discovery. --- .../LocalArticlePeer/CorpusArticleIndex.cs | 196 +++++++++ .../Lab/LocalArticlePeer/LabBinaryIdentity.cs | 71 ++++ .../LabRunningProcessIdentity.cs | 125 ++++++ .../LocalArticlePeer/LocalArticleNntpPeer.cs | 395 ++++++++++++++++++ .../LocalArticleNntpPeerMetrics.cs | 90 ++++ .../LocalArticleNntpPeerOptions.cs | 69 +++ .../LocalArticleNntpPeerTests.cs | 290 +++++++++++++ .../LocalArticlePeerCorpusTests.cs | 69 +++ .../LocalArticlePeerHostBench.cs | 249 +++++++++++ .../LocalArticlePeerWorkload.cs | 82 ++++ .../LocalPeerSmokeAccounting.cs | 120 ++++++ .../LocalPeerSmokeAccountingTests.cs | 101 +++++ .../LocalUpstreamIsolation.cs | 97 +++++ .../LocalUpstreamIsolationGate.cs | 267 ++++++++++++ .../LocalUpstreamIsolationGateTests.cs | 220 ++++++++++ .../LocalUpstreamIsolationTests.cs | 131 ++++++ .../LoopbackAllowlistNntpTransportFactory.cs | 61 +++ .../Lab/LocalArticlePeer/NntpWireFramer.cs | 67 +++ .../LocalArticlePeer/NntpWireFramerTests.cs | 50 +++ .../LocalArticlePeer/PreloadedArticleStore.cs | 131 ++++++ .../PreloadedArticleStoreTests.cs | 266 ++++++++++++ .../UpstreamProviderSnapshot.cs | 79 ++++ .../Stress/BackFillerE2EStressBench.cs | 193 +++++++++ .../Stress/BackFillerE2EStressHarness.cs | 26 +- .../Stress/StressHarnessOptions.cs | 17 + 25 files changed, 3461 insertions(+), 1 deletion(-) create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndex.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LabBinaryIdentity.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LabRunningProcessIdentity.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerMetrics.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerOptions.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerCorpusTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerHostBench.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerWorkload.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalPeerSmokeAccounting.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalPeerSmokeAccountingTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolation.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGate.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGateTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LoopbackAllowlistNntpTransportFactory.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/NntpWireFramer.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/NntpWireFramerTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/UpstreamProviderSnapshot.cs diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndex.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndex.cs new file mode 100644 index 00000000..7f2d2b53 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndex.cs @@ -0,0 +1,196 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Maps Message-ID → relative corpus path using header-only scans. Does not retain article bodies. + /// + internal sealed class CorpusArticleIndex + { + private readonly ConcurrentDictionary _messageIdToRelativePath = + new(StringComparer.Ordinal); + + private CorpusArticleIndex(string corpusRoot, IReadOnlyDictionary map, string fingerprint) + { + CorpusRoot = Path.GetFullPath(corpusRoot); + FingerprintSha256 = fingerprint; + foreach (var pair in map) + { + _messageIdToRelativePath[pair.Key] = pair.Value; + } + } + + internal string CorpusRoot { get; } + + internal int Count => _messageIdToRelativePath.Count; + + internal string FingerprintSha256 { get; } + + internal IEnumerable MessageIds => _messageIdToRelativePath.Keys; + + internal bool TryResolve(string messageId, out string absolutePath) + { + absolutePath = ""; + if (!_messageIdToRelativePath.TryGetValue(messageId, out var relative)) + { + return false; + } + + absolutePath = Path.Combine(CorpusRoot, relative.Replace('/', Path.DirectorySeparatorChar)); + return File.Exists(absolutePath); + } + + /// + /// Builds an index by reading at most from each file under + /// . Skips files without a Message-ID or header/body separator in the probe. + /// + internal static async Task BuildAsync( + string corpusRoot, + int headerProbeBytes = 16 * 1024, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(corpusRoot); + var root = Path.GetFullPath(corpusRoot); + if (!Directory.Exists(root)) + { + throw new DirectoryNotFoundException(root); + } + + var map = new Dictionary(StringComparer.Ordinal); + var fingerprintMaterial = new List(); + var buffer = new byte[headerProbeBytes]; + foreach (var file in Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories) + .OrderBy(static p => p, StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + int read; + await using (var stream = new FileStream( + file, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 4096, + FileOptions.Asynchronous | FileOptions.SequentialScan)) + { + read = await stream.ReadAsync(buffer.AsMemory(0, headerProbeBytes), cancellationToken) + .ConfigureAwait(false); + } + + if (read <= 0) + { + continue; + } + + var span = buffer.AsSpan(0, read); + if (span.IndexOf("\r\n\r\n"u8) < 0 && span.IndexOf("\n\n"u8) < 0) + { + continue; + } + + if (!TryReadMessageId(span, out var messageId)) + { + continue; + } + + var relative = Path.GetRelativePath(root, file).Replace('\\', '/'); + if (map.TryAdd(messageId, relative)) + { + fingerprintMaterial.Add($"{messageId}\t{relative}\t{new FileInfo(file).Length}"); + } + } + + fingerprintMaterial.Sort(StringComparer.Ordinal); + var fpBytes = Encoding.UTF8.GetBytes(string.Join('\n', fingerprintMaterial)); + var fingerprint = Convert.ToHexString(SHA256.HashData(fpBytes)); + return new CorpusArticleIndex(root, map, fingerprint); + } + + internal async Task WriteManifestAsync(string path, CancellationToken cancellationToken) + { + var payload = new + { + CorpusRoot, + Count, + FingerprintSha256, + Entries = _messageIdToRelativePath + .OrderBy(static p => p.Key, StringComparer.Ordinal) + .Select(static p => new { MessageId = p.Key, RelativePath = p.Value }) + .Take(32) + .ToArray(), + EntriesNote = "First 32 Message-IDs only; full count and fingerprint identify the corpus.", + }; + await File.WriteAllTextAsync( + path, + JsonSerializer.Serialize(payload, new JsonSerializerOptions { WriteIndented = true }), + cancellationToken) + .ConfigureAwait(false); + } + + private static bool TryReadMessageId(ReadOnlySpan article, out string messageId) + { + messageId = ""; + var lineStart = 0; + while (lineStart < article.Length) + { + var rest = article[lineStart..]; + var newline = rest.IndexOf((byte)'\n'); + var line = newline < 0 ? rest : rest[..newline]; + if (line.Length > 0 && line[^1] == (byte)'\r') + { + line = line[..^1]; + } + + if (line.Length == 0) + { + return false; + } + + if (line.Length >= 11 + && (line.StartsWith("Message-ID:"u8) + || line.StartsWith("Message-Id:"u8) + || StartsWithIgnoreCase(line, "message-id:"u8))) + { + var value = line[11..].Trim((byte)' '); + messageId = Encoding.ASCII.GetString(value); + return messageId.Length > 0; + } + + if (newline < 0) + { + return false; + } + + lineStart += newline + 1; + } + + return false; + } + + private static bool StartsWithIgnoreCase(ReadOnlySpan line, ReadOnlySpan asciiLowerPrefix) + { + if (line.Length < asciiLowerPrefix.Length) + { + return false; + } + + for (var i = 0; i < asciiLowerPrefix.Length; i++) + { + var c = line[i]; + if (c >= (byte)'A' && c <= (byte)'Z') + { + c = (byte)(c + 32); + } + + if (c != asciiLowerPrefix[i]) + { + return false; + } + } + + return true; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LabBinaryIdentity.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LabBinaryIdentity.cs new file mode 100644 index 00000000..e03df346 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LabBinaryIdentity.cs @@ -0,0 +1,71 @@ +using System.Security.Cryptography; +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Captures SHA-256 identities for lab reports without recording secrets. + /// + internal static class LabBinaryIdentity + { + /// Snapshot of loaded assemblies and runtime suitable for JSON reports. + internal sealed class Snapshot + { + public required string CapturedUtc { get; init; } + + public required string RuntimeVersion { get; init; } + + public required string FrameworkDescription { get; init; } + + public required string? BackFillerPath { get; init; } + + public required string? BackFillerSha256 { get; init; } + + public required string? CommonPath { get; init; } + + public required string? CommonSha256 { get; init; } + + public required string PeerAssemblyPath { get; init; } + + public required string PeerAssemblySha256 { get; init; } + } + + /// Hashes the file at as uppercase hex SHA-256. + internal static string Sha256File(string path) + { + using var stream = File.OpenRead(path); + return Convert.ToHexString(SHA256.HashData(stream)); + } + + /// Captures identities of BackFiller, Common, and the test assembly hosting the peer. + internal static Snapshot Capture() + { + var backFiller = typeof(VectorNNTPD.BackFillerService.Nntp.NntpProviderSession).Assembly.Location; + var common = typeof(VectorNNTPD.Common.Core.ExceptionHelpers).Assembly.Location; + var peer = typeof(LocalArticleNntpPeer).Assembly.Location; + return new Snapshot + { + CapturedUtc = DateTimeOffset.UtcNow.ToString("o"), + RuntimeVersion = Environment.Version.ToString(), + FrameworkDescription = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription, + BackFillerPath = string.IsNullOrEmpty(backFiller) ? null : backFiller, + BackFillerSha256 = string.IsNullOrEmpty(backFiller) ? null : Sha256File(backFiller), + CommonPath = string.IsNullOrEmpty(common) ? null : common, + CommonSha256 = string.IsNullOrEmpty(common) ? null : Sha256File(common), + PeerAssemblyPath = peer, + PeerAssemblySha256 = Sha256File(peer), + }; + } + + /// Writes JSON to . + internal static async Task WriteAsync(string path, CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(Capture(), new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(path, json, cancellationToken).ConfigureAwait(false); + } + + /// SHA-256 of sanitized config text (no credentials expected in input). + internal static string Sha256Utf8Text(string text) => + Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(text))); + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LabRunningProcessIdentity.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LabRunningProcessIdentity.cs new file mode 100644 index 00000000..6f19c235 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LabRunningProcessIdentity.cs @@ -0,0 +1,125 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Captures identity of an already-running process without starting services. + /// Distinguishes test assemblies from published binaries from live process modules. + /// + internal static class LabRunningProcessIdentity + { + internal sealed class ProcessSnapshot + { + public required int ProcessId { get; init; } + + public required string ProcessName { get; init; } + + public required string? MainModulePath { get; init; } + + public required string? MainModuleSha256 { get; init; } + + public required DateTime StartTimeUtc { get; init; } + + public required string RuntimeDescription { get; init; } + + public required IReadOnlyList Modules { get; init; } + + public required string Note { get; init; } + } + + internal sealed class ModuleHash + { + public required string Path { get; init; } + + public required string Sha256 { get; init; } + } + + /// + /// Attempts to capture identity for . Returns null when the + /// process is gone or inaccessible. Does not start processes. + /// + internal static ProcessSnapshot? TryCapture(int processId, params string[] moduleNameContains) + { + try + { + using var process = Process.GetProcessById(processId); + string? mainPath = null; + string? mainSha = null; + try + { + mainPath = process.MainModule?.FileName; + if (!string.IsNullOrWhiteSpace(mainPath) && File.Exists(mainPath)) + { + mainSha = Sha256File(mainPath); + } + } + catch (Exception ex) when (ex is InvalidOperationException or NotSupportedException or System.ComponentModel.Win32Exception) + { + // Main module may be inaccessible for some hosts; continue with partial evidence. + } + + var modules = new List(); + try + { + foreach (ProcessModule module in process.Modules) + { + var path = module.FileName; + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + continue; + } + + if (moduleNameContains.Length > 0 + && !moduleNameContains.Any(fragment => + path.Contains(fragment, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + modules.Add(new ModuleHash { Path = path, Sha256 = Sha256File(path) }); + } + } + catch (Exception ex) when (ex is InvalidOperationException or NotSupportedException or System.ComponentModel.Win32Exception) + { + // Module enumeration can fail; partial snapshot still useful. + } + + return new ProcessSnapshot + { + ProcessId = process.Id, + ProcessName = process.ProcessName, + MainModulePath = mainPath, + MainModuleSha256 = mainSha, + StartTimeUtc = process.StartTime.ToUniversalTime(), + RuntimeDescription = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription, + Modules = modules, + Note = + "Captured from a live process. This is not a test-assembly hash and is not proof " + + "of effective MySQL account-pool state (record UpstreamProviderSnapshot separately).", + }; + } + catch (ArgumentException) + { + return null; + } + catch (InvalidOperationException) + { + return null; + } + } + + internal static async Task WriteAsync(string path, ProcessSnapshot snapshot, CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(snapshot, new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(path, json, cancellationToken).ConfigureAwait(false); + } + + private static string Sha256File(string path) + { + using var stream = File.OpenRead(path); + return Convert.ToHexString(SHA256.HashData(stream)); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs new file mode 100644 index 00000000..8be58a0b --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs @@ -0,0 +1,395 @@ +using System.Buffers; +using System.Net; +using System.Net.Sockets; +using System.Text; +using VectorNNTPD.Common.Core; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Lab-only loopback NNTP peer that serves corpus ARTICLEs for BackFiller client connect/AUTH/ARTICLE. + /// Cleartext only (no STARTTLS advertisement). Not a production NNTP server. + /// + internal sealed class LocalArticleNntpPeer : IAsyncDisposable + { + private readonly LocalArticleNntpPeerOptions _options; + private readonly CorpusArticleIndex _index; + private readonly PreloadedArticleStore? _preloaded; + private readonly TcpListener _listener; + private readonly CancellationTokenSource _cts = new(); + private readonly SemaphoreSlim _sessionGate; + private readonly Task _acceptLoop; + private int _disposed; + + /// Starts listening after validating bind address and indexing the corpus. + internal LocalArticleNntpPeer( + LocalArticleNntpPeerOptions options, + CorpusArticleIndex index, + PreloadedArticleStore? preloaded = null) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _index = index ?? throw new ArgumentNullException(nameof(index)); + _preloaded = preloaded; + if (_options.PreloadArticlesIntoMemory && _preloaded is null) + { + throw new InvalidOperationException( + "PreloadArticlesIntoMemory requires a non-null PreloadedArticleStore."); + } + + _options.ValidateBindOrThrow(); + Metrics = new LocalArticleNntpPeerMetrics(); + _sessionGate = new SemaphoreSlim(_options.MaxConcurrentSessions, _options.MaxConcurrentSessions); + _listener = new TcpListener(_options.BindAddress, _options.Port); + _listener.Start(); + Port = ((IPEndPoint)_listener.LocalEndpoint).Port; + _acceptLoop = AcceptLoopAsync(_cts.Token); + } + + /// Bound TCP port (ephemeral when options requested port 0). + internal int Port { get; } + + /// Aggregate peer counters. + internal LocalArticleNntpPeerMetrics Metrics { get; } + + /// Corpus index used for Message-ID resolution. + internal CorpusArticleIndex Index => _index; + + /// Preloaded ARTICLE store when memory-resident mode is enabled; otherwise null. + internal PreloadedArticleStore? PreloadedStore => _preloaded; + + /// memory when serving precomputed ARTICLE bytes; otherwise file. + internal string PeerMode => _preloaded is not null ? "memory" : "file"; + + /// + /// Builds a corpus index, optionally preloads framed ARTICLE responses, then starts the peer. + /// Prefer this over constructing with a raw empty index. + /// + internal static async Task StartAsync( + LocalArticleNntpPeerOptions options, + CancellationToken cancellationToken = default) + { + var index = await CorpusArticleIndex.BuildAsync(options.CorpusRoot, cancellationToken: cancellationToken) + .ConfigureAwait(false); + PreloadedArticleStore? preloaded = null; + if (options.PreloadArticlesIntoMemory) + { + preloaded = await PreloadedArticleStore + .LoadAsync(index, options.MaxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + + return new LocalArticleNntpPeer(options, index, preloaded); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + await _cts.CancelAsync().ConfigureAwait(false); + try + { + _listener.Stop(); + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + } + + try + { + await _acceptLoop.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + } + + _cts.Dispose(); + _sessionGate.Dispose(); + } + + private async Task AcceptLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (ObjectDisposedException) + { + return; + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + Metrics.RecordProtocolError(); + continue; + } + + if (!await _sessionGate.WaitAsync(0, cancellationToken).ConfigureAwait(false)) + { + Metrics.ConnectionRejected(); + client.Dispose(); + continue; + } + + Metrics.ConnectionAccepted(); + _ = Task.Run( + async () => + { + try + { + await HandleClientAsync(client, cancellationToken).ConfigureAwait(false); + } + finally + { + client.Dispose(); + _sessionGate.Release(); + } + }, + CancellationToken.None); + } + } + + private async Task HandleClientAsync(TcpClient client, CancellationToken cancellationToken) + { + Metrics.SessionStarted(); + try + { + client.NoDelay = true; + await using var stream = client.GetStream(); + await WriteAsciiAsync(stream, "200 LocalArticleNntpPeer ready\r\n", cancellationToken) + .ConfigureAwait(false); + + var authenticated = !_options.RequiresAuthentication; + while (!cancellationToken.IsCancellationRequested) + { + var line = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false); + if (line is null) + { + return; + } + + if (line.Equals("CAPABILITIES", StringComparison.OrdinalIgnoreCase)) + { + // Do not advertise STARTTLS — cleartext lab peer; client UseTls must be false. + await WriteAsciiAsync( + stream, + "101 Capability list follows\r\nVERSION 2\r\nREADER\r\nAUTHINFO USER\r\n.\r\n", + cancellationToken) + .ConfigureAwait(false); + continue; + } + + if (line.StartsWith("AUTHINFO USER ", StringComparison.OrdinalIgnoreCase)) + { + if (!_options.RequiresAuthentication) + { + await WriteAsciiAsync(stream, "281 Authentication accepted\r\n", cancellationToken) + .ConfigureAwait(false); + authenticated = true; + continue; + } + + var user = line["AUTHINFO USER ".Length..]; + if (!string.Equals(user, _options.Username, StringComparison.Ordinal)) + { + await WriteAsciiAsync(stream, "481 Authentication failed\r\n", cancellationToken) + .ConfigureAwait(false); + Metrics.RecordProtocolError(); + return; + } + + await WriteAsciiAsync(stream, "381 Password required\r\n", cancellationToken) + .ConfigureAwait(false); + var passLine = await ReadLineAsync(stream, cancellationToken).ConfigureAwait(false); + if (passLine is null + || !passLine.StartsWith("AUTHINFO PASS ", StringComparison.OrdinalIgnoreCase) + || !string.Equals( + passLine["AUTHINFO PASS ".Length..], + _options.Password, + StringComparison.Ordinal)) + { + await WriteAsciiAsync(stream, "481 Authentication failed\r\n", cancellationToken) + .ConfigureAwait(false); + Metrics.RecordProtocolError(); + return; + } + + await WriteAsciiAsync(stream, "281 Authentication accepted\r\n", cancellationToken) + .ConfigureAwait(false); + authenticated = true; + continue; + } + + if (line.Equals("DATE", StringComparison.OrdinalIgnoreCase)) + { + var stamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss"); + await WriteAsciiAsync(stream, $"111 {stamp}\r\n", cancellationToken).ConfigureAwait(false); + continue; + } + + if (line.Equals("QUIT", StringComparison.OrdinalIgnoreCase)) + { + await WriteAsciiAsync(stream, "205 Connection closing\r\n", cancellationToken) + .ConfigureAwait(false); + return; + } + + if (line.StartsWith("ARTICLE ", StringComparison.OrdinalIgnoreCase)) + { + if (_options.RequiresAuthentication && !authenticated) + { + await WriteAsciiAsync(stream, "480 Authentication required\r\n", cancellationToken) + .ConfigureAwait(false); + Metrics.RecordProtocolError(); + continue; + } + + await ServeArticleAsync(stream, line["ARTICLE ".Length..].Trim(), cancellationToken) + .ConfigureAwait(false); + continue; + } + + await WriteAsciiAsync(stream, "500 Unknown command\r\n", cancellationToken).ConfigureAwait(false); + Metrics.RecordProtocolError(); + } + } + finally + { + Metrics.SessionEnded(); + } + } + + private async Task ServeArticleAsync(NetworkStream stream, string messageId, CancellationToken cancellationToken) + { + Metrics.ArticleBegin(); + var settled = false; + try + { + if (_options.ArticleDelay > TimeSpan.Zero) + { + await Task.Delay(_options.ArticleDelay, cancellationToken).ConfigureAwait(false); + } + + if (_preloaded is not null) + { + if (!_preloaded.TryGet(messageId, out var preloaded)) + { + // Record before write so clients observing the status line cannot race the counter. + Metrics.RecordArticleMissing(); + settled = true; + await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken) + .ConfigureAwait(false); + return; + } + + // Shared immutable buffer: concurrent sessions may write the same array; none mutate it. + await stream.WriteAsync(preloaded.WireResponse, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + Metrics.RecordArticleSuccess(preloaded.LogicalByteCount, preloaded.WireResponse.Length); + settled = true; + return; + } + + if (!_index.TryResolve(messageId, out var path)) + { + // Record before write so clients observing the status line cannot race the counter. + Metrics.RecordArticleMissing(); + settled = true; + await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken) + .ConfigureAwait(false); + return; + } + + byte[] logical; + byte[] framed; + try + { + (logical, framed) = await NntpWireFramer + .ReadAndFrameAsync(path, _options.MaxArticleBytes, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + Metrics.RecordArticleError(); + settled = true; + await WriteAsciiAsync(stream, "403 Article unavailable\r\n", cancellationToken) + .ConfigureAwait(false); + return; + } + + // Status text is optional for the client on 220; keep Message-ID for debuggability. + var status = Encoding.ASCII.GetBytes($"220 0 {messageId} article follows\r\n"); + await stream.WriteAsync(status, cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(framed, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + Metrics.RecordArticleSuccess(logical.Length, status.Length + framed.Length); + settled = true; + } + catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) + { + if (!settled) + { + Metrics.RecordArticleError(); + } + + throw; + } + } + + private static async Task WriteAsciiAsync(Stream stream, string text, CancellationToken cancellationToken) + { + var bytes = Encoding.ASCII.GetBytes(text); + await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ReadLineAsync(Stream stream, CancellationToken cancellationToken) + { + var buffer = ArrayPool.Shared.Rent(1024); + try + { + var length = 0; + while (true) + { + var rented = ArrayPool.Shared.Rent(1); + try + { + var n = await stream.ReadAsync(rented.AsMemory(0, 1), cancellationToken).ConfigureAwait(false); + if (n == 0) + { + return length == 0 ? null : Encoding.ASCII.GetString(buffer, 0, length); + } + + if (length >= buffer.Length) + { + return null; + } + + buffer[length++] = rented[0]; + if (length >= 2 && buffer[length - 2] == (byte)'\r' && buffer[length - 1] == (byte)'\n') + { + return Encoding.ASCII.GetString(buffer, 0, length - 2); + } + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerMetrics.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerMetrics.cs new file mode 100644 index 00000000..a0a11ac0 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerMetrics.cs @@ -0,0 +1,90 @@ +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// Aggregate counters for one lifetime. + internal sealed class LocalArticleNntpPeerMetrics + { + private long _acceptedConnections; + private long _rejectedConnections; + private long _articleRequests; + private long _articleSuccess; + private long _articleMissing; + private long _articleErrors; + private long _logicalBytes; + private long _wireBytes; + private long _protocolErrors; + private int _outstandingArticles; + private int _maxOutstandingArticles; + private int _activeSessions; + private int _maxActiveSessions; + + internal long AcceptedConnections => Volatile.Read(ref _acceptedConnections); + internal long RejectedConnections => Volatile.Read(ref _rejectedConnections); + internal long ArticleRequests => Volatile.Read(ref _articleRequests); + internal long ArticleSuccess => Volatile.Read(ref _articleSuccess); + internal long ArticleMissing => Volatile.Read(ref _articleMissing); + internal long ArticleErrors => Volatile.Read(ref _articleErrors); + internal long LogicalBytesTransferred => Volatile.Read(ref _logicalBytes); + internal long WireBytesTransferred => Volatile.Read(ref _wireBytes); + internal long ProtocolErrors => Volatile.Read(ref _protocolErrors); + internal int OutstandingArticles => Volatile.Read(ref _outstandingArticles); + internal int MaxOutstandingArticles => Volatile.Read(ref _maxOutstandingArticles); + internal int ActiveSessions => Volatile.Read(ref _activeSessions); + internal int MaxActiveSessions => Volatile.Read(ref _maxActiveSessions); + + internal void ConnectionAccepted() => Interlocked.Increment(ref _acceptedConnections); + + internal void ConnectionRejected() => Interlocked.Increment(ref _rejectedConnections); + + internal void SessionStarted() + { + var n = Interlocked.Increment(ref _activeSessions); + UpdateMax(ref _maxActiveSessions, n); + } + + internal void SessionEnded() => Interlocked.Decrement(ref _activeSessions); + + internal void ArticleBegin() + { + Interlocked.Increment(ref _articleRequests); + var n = Interlocked.Increment(ref _outstandingArticles); + UpdateMax(ref _maxOutstandingArticles, n); + } + + internal void RecordArticleSuccess(int logicalBytes, int wireBytes) + { + Interlocked.Increment(ref _articleSuccess); + Interlocked.Add(ref _logicalBytes, logicalBytes); + Interlocked.Add(ref _wireBytes, wireBytes); + Interlocked.Decrement(ref _outstandingArticles); + } + + internal void RecordArticleMissing() + { + Interlocked.Increment(ref _articleMissing); + Interlocked.Decrement(ref _outstandingArticles); + } + + internal void RecordArticleError() + { + Interlocked.Increment(ref _articleErrors); + Interlocked.Decrement(ref _outstandingArticles); + } + + internal void RecordProtocolError() => Interlocked.Increment(ref _protocolErrors); + + private static void UpdateMax(ref int location, int candidate) + { + var observed = Volatile.Read(ref location); + while (candidate > observed) + { + var prev = Interlocked.CompareExchange(ref location, candidate, observed); + if (prev == observed) + { + return; + } + + observed = prev; + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerOptions.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerOptions.cs new file mode 100644 index 00000000..33a6255a --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerOptions.cs @@ -0,0 +1,69 @@ +using System.Net; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// Lab-only bind and corpus options for . + internal sealed class LocalArticleNntpPeerOptions + { + /// Environment flag required to bind outside loopback. + internal const string AllowNonLoopbackEnvironmentVariable = + "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_ALLOW_NON_LOOPBACK"; + + /// Absolute corpus root (logical article files). + public required string CorpusRoot { get; init; } + + /// Bind address. Defaults to loopback. + public IPAddress BindAddress { get; init; } = IPAddress.Loopback; + + /// TCP port. Zero selects an ephemeral port. + public int Port { get; init; } + + /// Maximum concurrent accepted sessions. + public int MaxConcurrentSessions { get; init; } = 64; + + /// Maximum logical article bytes accepted from disk. + public int MaxArticleBytes { get; init; } = 5 * 1024 * 1024; + + /// + /// When set, AUTHINFO USER/PASS must match. When null, authentication is not required + /// (client must also omit credentials). + /// + public string? Username { get; init; } + + /// Password paired with . + public string? Password { get; init; } + + /// Optional artificial delay before ARTICLE body (separate latency experiments). + public TimeSpan ArticleDelay { get; init; } = TimeSpan.Zero; + + /// + /// When true, frame every indexed article once at startup and serve immutable precomputed + /// ARTICLE wire bytes. Lab diagnostic mode only; default remains file-backed. + /// + public bool PreloadArticlesIntoMemory { get; init; } + + /// True when either credential is configured (client must authenticate). + internal bool RequiresAuthentication => + !string.IsNullOrWhiteSpace(Username) || !string.IsNullOrWhiteSpace(Password); + + /// + /// Validates bind safety: non-loopback requires + /// =1. + /// + internal void ValidateBindOrThrow() + { + if (IPAddress.IsLoopback(BindAddress)) + { + return; + } + + var allow = Environment.GetEnvironmentVariable(AllowNonLoopbackEnvironmentVariable); + if (!string.Equals(allow, "1", StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Refusing to bind LocalArticleNntpPeer to non-loopback {BindAddress}. " + + $"Set {AllowNonLoopbackEnvironmentVariable}=1 for explicit lab override only."); + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs new file mode 100644 index 00000000..f1fefa2f --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs @@ -0,0 +1,290 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.BackFillerService.Nntp; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + public sealed class LocalArticleNntpPeerTests + { + [Fact] + public async Task Peer_serves_greeting_capabilities_article_and_quit() + { + var dir = CreateFixtureCorpus(out var messageId); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = 0, MaxConcurrentSessions = 4 }); + + Assert.True(peer.Port > 0); + Assert.Equal(1, peer.Index.Count); + + using var client = new System.Net.Sockets.TcpClient(); + await client.ConnectAsync(System.Net.IPAddress.Loopback, peer.Port); + await using var stream = client.GetStream(); + Assert.StartsWith("200 ", await ReadLineAsync(stream)); + + await WriteAsync(stream, "CAPABILITIES\r\n"u8.ToArray()); + Assert.StartsWith("101 ", await ReadLineAsync(stream)); + while (true) + { + var cap = await ReadLineAsync(stream); + if (cap == ".") + { + break; + } + + Assert.False(cap.Contains("STARTTLS", StringComparison.OrdinalIgnoreCase)); + } + + await WriteAsync(stream, EncodingAscii($"ARTICLE {messageId}\r\n")); + Assert.StartsWith("220 ", await ReadLineAsync(stream)); + var bodyLines = new List(); + while (true) + { + var line = await ReadLineAsync(stream); + if (line == ".") + { + break; + } + + bodyLines.Add(line); + } + + Assert.Contains(bodyLines, static l => l.StartsWith("Message-ID:", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(bodyLines, static l => l == "hello"); + + await WriteAsync(stream, "QUIT\r\n"u8.ToArray()); + Assert.StartsWith("205 ", await ReadLineAsync(stream)); + + Assert.Equal(1, peer.Metrics.ArticleSuccess); + Assert.Equal(0, peer.Metrics.ArticleMissing); + Assert.Equal(0, peer.Metrics.OutstandingArticles); + } + + [Fact] + public async Task Peer_returns_430_for_unknown_message_id() + { + var dir = CreateFixtureCorpus(out _); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = 0 }); + + using var client = new System.Net.Sockets.TcpClient(); + await client.ConnectAsync(System.Net.IPAddress.Loopback, peer.Port); + await using var stream = client.GetStream(); + _ = await ReadLineAsync(stream); + await WriteAsync(stream, "ARTICLE \r\n"u8.ToArray()); + Assert.StartsWith("430 ", await ReadLineAsync(stream)); + Assert.Equal(1, peer.Metrics.ArticleMissing); + } + + [Fact] + public async Task Peer_auth_flow_accepts_configured_credentials() + { + var dir = CreateFixtureCorpus(out var messageId); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = dir, + Port = 0, + Username = "labuser", + Password = "labpass", + }); + + using var client = new System.Net.Sockets.TcpClient(); + await client.ConnectAsync(System.Net.IPAddress.Loopback, peer.Port); + await using var stream = client.GetStream(); + _ = await ReadLineAsync(stream); + await WriteAsync(stream, "AUTHINFO USER labuser\r\n"u8.ToArray()); + Assert.StartsWith("381 ", await ReadLineAsync(stream)); + await WriteAsync(stream, "AUTHINFO PASS labpass\r\n"u8.ToArray()); + Assert.StartsWith("281 ", await ReadLineAsync(stream)); + await WriteAsync(stream, EncodingAscii($"ARTICLE {messageId}\r\n")); + Assert.StartsWith("220 ", await ReadLineAsync(stream)); + while (await ReadLineAsync(stream) != ".") + { + } + } + + [Fact] + public async Task BackFiller_client_downloads_article_from_peer_without_tls_or_auth() + { + var dir = CreateFixtureCorpus(out var messageId); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = 0 }); + + LocalUpstreamIsolation.EnsureLocalUpstreamOrThrow("127.0.0.1"); + + var provider = new BackFillerProviderDefinition( + "LabLocal", + "127.0.0.1", + peer.Port, + UseTls: false, + Username: null, + Password: null, + MinSessions: 0, + MaxSessions: 1); + await using var session = new NntpProviderSession( + provider, + NntpSessionOptions.Default, + NullLogger.Instance); + var transport = new TcpNntpTransportFactory(); + Assert.Null(await session.ConnectAsync(transport, CancellationToken.None)); + + using var result = await session.DownloadArticleAsync(messageId, CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, result.Kind); + Assert.NotNull(result.Article); + Assert.Contains("hello"u8, result.Article!.Memory.Span); + Assert.Equal(1, peer.Metrics.ArticleSuccess); + } + + /// + /// Lab rewrite shape (null credentials, cleartext loopback) is accepted by the peer without AUTHINFO. + /// + [Fact] + public async Task Peer_accepts_unauthenticated_flow_matching_lab_rewrite() + { + var dir = CreateFixtureCorpus(out var messageId); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = dir, + Port = 0, + Username = null, + Password = null, + }); + Assert.False( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Username = null, Password = null } + .RequiresAuthentication); + + using var client = new System.Net.Sockets.TcpClient(); + await client.ConnectAsync(System.Net.IPAddress.Loopback, peer.Port); + await using var stream = client.GetStream(); + Assert.StartsWith("200 ", await ReadLineAsync(stream)); + await WriteAsync(stream, "CAPABILITIES\r\n"u8.ToArray()); + Assert.StartsWith("101 ", await ReadLineAsync(stream)); + while (await ReadLineAsync(stream) != ".") + { + } + + await WriteAsync(stream, EncodingAscii($"ARTICLE {messageId}\r\n")); + Assert.StartsWith("220 ", await ReadLineAsync(stream)); + while (await ReadLineAsync(stream) != ".") + { + } + + Assert.Equal(1, peer.Metrics.ArticleSuccess); + Assert.Equal(0, peer.Metrics.ProtocolErrors); + } + + [Fact] + public async Task Concurrent_article_requests_complete() + { + var dir = CreateFixtureCorpus(out var messageId); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = 0, MaxConcurrentSessions = 16 }); + + var tasks = Enumerable.Range(0, 8).Select(async _ => + { + var provider = new BackFillerProviderDefinition( + "LabLocal", "127.0.0.1", peer.Port, false, null, null, 0, 1); + await using var session = new NntpProviderSession( + provider, NntpSessionOptions.Default, NullLogger.Instance); + Assert.Null(await session.ConnectAsync(new TcpNntpTransportFactory(), CancellationToken.None)); + using var result = await session.DownloadArticleAsync(messageId, CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, result.Kind); + }); + await Task.WhenAll(tasks); + Assert.Equal(8, peer.Metrics.ArticleSuccess); + Assert.Equal(0, peer.Metrics.OutstandingArticles); + } + + [Fact] + public async Task Dispose_releases_listener_and_allows_rebind() + { + var dir = CreateFixtureCorpus(out _); + int port; + await using (var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = 0 })) + { + port = peer.Port; + Assert.True(port > 0); + } + + await using var peer2 = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = port }); + Assert.Equal(port, peer2.Port); + } + + [Fact] + public async Task Rejects_sessions_beyond_concurrency_limit() + { + var dir = CreateFixtureCorpus(out var messageId); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = 0, MaxConcurrentSessions = 1 }); + + var provider = new BackFillerProviderDefinition( + "LabLocal", "127.0.0.1", peer.Port, false, null, null, 0, 1); + await using var session = new NntpProviderSession( + provider, NntpSessionOptions.Default, NullLogger.Instance); + Assert.Null(await session.ConnectAsync(new TcpNntpTransportFactory(), CancellationToken.None)); + + // Hold the single session open while a raw TCP connect should be rejected. + using var extra = new System.Net.Sockets.TcpClient(); + await extra.ConnectAsync(System.Net.IPAddress.Loopback, peer.Port); + var deadline = DateTime.UtcNow.AddSeconds(2); + while (peer.Metrics.RejectedConnections < 1 && DateTime.UtcNow < deadline) + { + await Task.Yield(); + } + + Assert.True(peer.Metrics.RejectedConnections >= 1); + + using var result = await session.DownloadArticleAsync(messageId, CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, result.Kind); + } + + [Fact] + public async Task LabBinaryIdentity_capture_hashes_loaded_assemblies() + { + var snap = LabBinaryIdentity.Capture(); + Assert.False(string.IsNullOrWhiteSpace(snap.BackFillerSha256)); + Assert.False(string.IsNullOrWhiteSpace(snap.CommonSha256)); + Assert.False(string.IsNullOrWhiteSpace(snap.PeerAssemblySha256)); + Assert.False(string.IsNullOrWhiteSpace(snap.RuntimeVersion)); + } + + private static string CreateFixtureCorpus(out string messageId) + { + messageId = $""; + var dir = Path.Combine(Path.GetTempPath(), "vnntp-local-peer-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + var body = $"Message-ID: {messageId}\r\nFrom: lab@test\r\n\r\nhello\r\n"; + File.WriteAllText(Path.Combine(dir, "article1"), body); + return dir; + } + + private static byte[] EncodingAscii(string s) => System.Text.Encoding.ASCII.GetBytes(s); + + private static async Task WriteAsync(Stream stream, byte[] bytes) => + await stream.WriteAsync(bytes); + + private static async Task ReadLineAsync(Stream stream) + { + var buffer = new List(128); + var one = new byte[1]; + while (true) + { + var n = await stream.ReadAsync(one); + if (n == 0) + { + break; + } + + buffer.Add(one[0]); + if (buffer.Count >= 2 && buffer[^2] == (byte)'\r' && buffer[^1] == (byte)'\n') + { + return System.Text.Encoding.ASCII.GetString(buffer.GetRange(0, buffer.Count - 2).ToArray()); + } + } + + return System.Text.Encoding.ASCII.GetString(buffer.ToArray()); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerCorpusTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerCorpusTests.cs new file mode 100644 index 00000000..0855d9aa --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerCorpusTests.cs @@ -0,0 +1,69 @@ +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Optional compatibility checks against .artifacts/Articles when present. + /// Skips when the corpus directory is missing (CI without local artifacts). + /// + public sealed class LocalArticlePeerCorpusTests + { + private static readonly string CorpusRoot = Path.GetFullPath( + Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "..", ".artifacts", "Articles")); + + [Fact] + public async Task Real_corpus_indexes_and_serves_one_article_when_present() + { + if (!Directory.Exists(CorpusRoot)) + { + return; + } + + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = CorpusRoot, + Port = 0, + MaxConcurrentSessions = 4, + }); + + Assert.True(peer.Index.Count > 0, "Expected corpus Message-IDs"); + var messageId = peer.Index.MessageIds.First(); + Assert.True(peer.Index.TryResolve(messageId, out var path)); + Assert.True(File.Exists(path)); + + var provider = new VectorNNTPD.BackFillerService.Nntp.BackFillerProviderDefinition( + "LabLocal", "127.0.0.1", peer.Port, false, null, null, 0, 1); + await using var session = new VectorNNTPD.BackFillerService.Nntp.NntpProviderSession( + provider, + VectorNNTPD.BackFillerService.Nntp.NntpSessionOptions.Default, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + Assert.Null( + await session.ConnectAsync( + new VectorNNTPD.BackFillerService.Nntp.TcpNntpTransportFactory(), + CancellationToken.None)); + using var result = await session.DownloadArticleAsync(messageId, CancellationToken.None); + Assert.Equal(VectorNNTPD.BackFillerService.Nntp.ArticleRetrievalKind.ArticleRetrieved, result.Kind); + Assert.True(result.Article!.Memory.Length > 0); + + var outDir = Path.GetFullPath( + Path.Combine( + AppContext.BaseDirectory, + "..", + "..", + "..", + "..", + "..", + "..", + ".artifacts", + "incident", + "smoke-soak", + "local-article-peer-benchmark-20261027")); + Directory.CreateDirectory(outDir); + await peer.Index.WriteManifestAsync( + Path.Combine(outDir, "corpus-fingerprint.json"), + CancellationToken.None); + await LabBinaryIdentity.WriteAsync( + Path.Combine(outDir, "assembly-identity-correctness.json"), + CancellationToken.None); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerHostBench.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerHostBench.cs new file mode 100644 index 00000000..1a362d99 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerHostBench.cs @@ -0,0 +1,249 @@ +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Opt-in lab host that keeps alive for a later authorized benchmark. + /// Disabled unless VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER=1. Not a production service. + /// + /// + /// Correctness unit tests start ephemeral peers and dispose them. This entry point is only for + /// operator-authorized local-peer stress and must not be enabled during implementation validation. + /// + public sealed class LocalArticlePeerHostBench + { + /// Master enable for the long-lived peer host process. + public const string EnableEnvironmentVariable = "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER"; + + /// Corpus root override; defaults to repo .artifacts/Articles. + public const string CorpusEnvironmentVariable = "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_CORPUS"; + + /// Bind port (default 11119). Use a fixed port so lab MySQL can target the peer. + public const string PortEnvironmentVariable = "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_PORT"; + + /// Max concurrent sessions (default 64). + public const string MaxSessionsEnvironmentVariable = "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_MAX_SESSIONS"; + + /// Optional AUTHINFO USER for lab auth exercise. + public const string UsernameEnvironmentVariable = "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_USERNAME"; + + /// Optional AUTHINFO PASS for lab auth exercise. + public const string PasswordEnvironmentVariable = "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_PASSWORD"; + + /// Directory for identity/manifest JSON (default under local-article-peer-benchmark artifacts). + public const string OutDirectoryEnvironmentVariable = "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_OUT"; + + /// + /// When 1, preload and frame the corpus once at startup (memory-resident ARTICLE responses). + /// Default remains file-backed per-request load/frame. + /// + public const string PreloadEnvironmentVariable = "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_PRELOAD"; + + /// + /// When enabled, starts the peer on loopback, writes identity + corpus fingerprint, then blocks + /// until cancelled. Does not contact real upstreams. + /// + [Fact] + public async Task Host_local_article_peer_when_enabled() + { + if (!string.Equals( + Environment.GetEnvironmentVariable(EnableEnvironmentVariable), + "1", + StringComparison.Ordinal)) + { + return; + } + + var repoRoot = FindRepositoryRoot(); + var corpus = Environment.GetEnvironmentVariable(CorpusEnvironmentVariable) + ?? Path.Join(repoRoot, ".artifacts", "Articles"); + if (!Directory.Exists(corpus)) + { + Assert.Fail($"Corpus missing: {corpus}"); + } + + var port = 11119; + if (int.TryParse(Environment.GetEnvironmentVariable(PortEnvironmentVariable), out var configuredPort) + && configuredPort is >= 0 and <= 65535) + { + port = configuredPort; + } + + var maxSessions = 64; + if (int.TryParse(Environment.GetEnvironmentVariable(MaxSessionsEnvironmentVariable), out var ms) + && ms > 0) + { + maxSessions = ms; + } + + var preload = string.Equals( + Environment.GetEnvironmentVariable(PreloadEnvironmentVariable), + "1", + StringComparison.Ordinal); + + var outDir = Environment.GetEnvironmentVariable(OutDirectoryEnvironmentVariable) + ?? Path.Join( + repoRoot, + ".artifacts", + "incident", + "smoke-soak", + "local-article-peer-benchmark-20261027", + "runtime"); + Directory.CreateDirectory(outDir); + + var options = new LocalArticleNntpPeerOptions + { + CorpusRoot = corpus, + Port = port, + MaxConcurrentSessions = maxSessions, + Username = Environment.GetEnvironmentVariable(UsernameEnvironmentVariable), + Password = Environment.GetEnvironmentVariable(PasswordEnvironmentVariable), + PreloadArticlesIntoMemory = preload, + }; + + Console.WriteLine( + $"LocalArticleNntpPeer starting mode={(preload ? "memory" : "file")} corpus={corpus} " + + $"preload={preload} (init is outside the measured interval)"); + await using var peer = await LocalArticleNntpPeer.StartAsync(options); + await LabBinaryIdentity.WriteAsync(Path.Join(outDir, "assembly-identity.json"), CancellationToken.None); + await peer.Index.WriteManifestAsync(Path.Join(outDir, "corpus-fingerprint.json"), CancellationToken.None); + var preloadMetrics = peer.PreloadedStore?.Metrics; + await File.WriteAllTextAsync( + Path.Join(outDir, "peer-endpoint.json"), + JsonSerializer.Serialize( + new + { + Host = "127.0.0.1", + peer.Port, + peer.PeerMode, + PreloadArticlesIntoMemory = preload, + UseTls = false, + StartTlsAdvertised = false, + RequiresAuthentication = options.RequiresAuthentication, + peer.Index.Count, + peer.Index.FingerprintSha256, + MaxConcurrentSessions = maxSessions, + Preload = preloadMetrics is null + ? null + : new + { + preloadMetrics.ArticlesLoaded, + preloadMetrics.LogicalBytesLoaded, + preloadMetrics.WireBytesLoaded, + LoadDurationMs = preloadMetrics.LoadDuration.TotalMilliseconds, + preloadMetrics.LoadFailures, + }, + SecurityNote = + "Cleartext lab peer. BackFiller provider row must use usessl=n and hostname=127.0.0.1.", + }, + new JsonSerializerOptions { WriteIndented = true })); + + var metricsPath = Path.Join(outDir, "peer-metrics.json"); + Environment.SetEnvironmentVariable("VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_METRICS_JSON", metricsPath); + + Console.WriteLine( + $"LocalArticleNntpPeer listening 127.0.0.1:{peer.Port} mode={peer.PeerMode} corpus={corpus} " + + $"count={peer.Index.Count} fp={peer.Index.FingerprintSha256} auth={options.RequiresAuthentication}"); + if (preloadMetrics is not null) + { + Console.WriteLine( + $"Preload articles={preloadMetrics.ArticlesLoaded} logicalBytes={preloadMetrics.LogicalBytesLoaded} " + + $"wireBytes={preloadMetrics.WireBytesLoaded} durationMs={preloadMetrics.LoadDuration.TotalMilliseconds:F0} " + + $"failures={preloadMetrics.LoadFailures}"); + } + + Console.WriteLine("Press Ctrl+C / cancel test to stop. Do not point this peer at production."); + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Console.CancelKeyPress += (_, e) => + { + e.Cancel = true; + tcs.TrySetResult(); + }; + + // Periodic snapshots so ramp/smoke can reconcile without relying on graceful Ctrl+C under redirected IO. + using var metricsTimer = new PeriodicTimer(TimeSpan.FromSeconds(2)); + var metricsPump = Task.Run(async () => + { + try + { + while (await metricsTimer.WaitForNextTickAsync()) + { + await WritePeerMetricsAsync(metricsPath, peer); + } + } + catch (OperationCanceledException) + { + // Host shutdown. + } + }); + + await tcs.Task; + metricsTimer.Dispose(); + try + { + await metricsPump; + } + catch (ObjectDisposedException) + { + // Timer disposed concurrently with WaitForNextTickAsync. + } + + await WritePeerMetricsAsync(metricsPath, peer); + Console.WriteLine($"Wrote peer metrics to {metricsPath}"); + } + + private static Task WritePeerMetricsAsync(string metricsPath, LocalArticleNntpPeer peer) + { + var preload = peer.PreloadedStore?.Metrics; + return File.WriteAllTextAsync( + metricsPath, + JsonSerializer.Serialize( + new + { + CapturedUtc = DateTimeOffset.UtcNow, + peer.PeerMode, + peer.Metrics.AcceptedConnections, + peer.Metrics.RejectedConnections, + peer.Metrics.ArticleRequests, + peer.Metrics.ArticleSuccess, + peer.Metrics.ArticleMissing, + peer.Metrics.ArticleErrors, + peer.Metrics.ProtocolErrors, + peer.Metrics.LogicalBytesTransferred, + peer.Metrics.WireBytesTransferred, + peer.Metrics.OutstandingArticles, + peer.Metrics.MaxOutstandingArticles, + peer.Metrics.ActiveSessions, + peer.Metrics.MaxActiveSessions, + Preload = preload is null + ? null + : new + { + preload.ArticlesLoaded, + preload.LogicalBytesLoaded, + preload.WireBytesLoaded, + LoadDurationMs = preload.LoadDuration.TotalMilliseconds, + preload.LoadFailures, + }, + }, + new JsonSerializerOptions { WriteIndented = true })); + } + + private static string FindRepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.slnx"))) + { + return dir.FullName; + } + + dir = dir.Parent; + } + + return Directory.GetCurrentDirectory(); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerWorkload.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerWorkload.cs new file mode 100644 index 00000000..4f6e38d2 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerWorkload.cs @@ -0,0 +1,82 @@ +using VectorNNTPD.BackFillerService.Tests.Stress; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Builds stress-compatible Message-ID workload files from the local Articles corpus index. + /// Required because news-log discovery IDs may not exist on the local peer disk. + /// + internal static class LocalArticlePeerWorkload + { + /// + /// Writes warmup/main/repeatable ID lists and a whose + /// is the Articles root so + /// accepts the workload when news-log env points at Articles. + /// + internal static async Task<(int Count, string Fingerprint)> WriteAsync( + CorpusArticleIndex index, + string workloadDirectory, + int seed, + int warmupCount = 50, + int mainCount = 2000, + int repeatableCount = 200, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(index); + ArgumentException.ThrowIfNullOrWhiteSpace(workloadDirectory); + Directory.CreateDirectory(workloadDirectory); + + var ids = index.MessageIds.OrderBy(static s => s, StringComparer.Ordinal).ToList(); + if (ids.Count == 0) + { + throw new InvalidOperationException("Corpus index contains no Message-IDs."); + } + + var rng = new Random(seed); + var shuffled = ids.ToList(); + for (var i = shuffled.Count - 1; i > 0; i--) + { + var j = rng.Next(i + 1); + (shuffled[i], shuffled[j]) = (shuffled[j], shuffled[i]); + } + + warmupCount = Math.Min(warmupCount, shuffled.Count); + mainCount = Math.Min(mainCount, shuffled.Count); + repeatableCount = Math.Min(repeatableCount, shuffled.Count); + + var warmup = shuffled.Take(warmupCount).ToArray(); + var main = shuffled.Take(mainCount).ToArray(); + var repeatable = shuffled.Take(repeatableCount).ToArray(); + + var warmupPath = Path.Join(workloadDirectory, "warmup-message-ids.txt"); + var mainPath = Path.Join(workloadDirectory, StressCorpusPaths.MainIdsAuditFileName); + var repeatablePath = Path.Join(workloadDirectory, "repeatable-message-ids.txt"); + await File.WriteAllLinesAsync(warmupPath, warmup, cancellationToken).ConfigureAwait(false); + await File.WriteAllLinesAsync(mainPath, main, cancellationToken).ConfigureAwait(false); + await File.WriteAllLinesAsync(repeatablePath, repeatable, cancellationToken).ConfigureAwait(false); + + var manifest = new WorkloadCorpusManifest + { + CorpusDirectory = index.CorpusRoot, + FileCount = index.Count, + LogRecordsExamined = index.Count, + ValidParsedIds = index.Count, + RejectedInvalidMessageIds = 0, + UniqueValidIds = index.Count, + WarmupSelected = warmup.Length, + MainSelected = main.Length, + RepeatableSelected = repeatable.Length, + Seed = seed, + MainIdsSha256 = WorkloadCorpusManifest.ComputeFileSha256(mainPath), + CreatedUtc = DateTimeOffset.UtcNow, + Notes = + "Lab local ARTICLE peer workload derived from .artifacts/Articles Message-ID index. " + + "Not news-log discovery. Set VECTORNNTPD_BACKFILLER_E2E_STRESS_NEWSLOGS to the Articles root " + + "so StressWorkloadGuard provenance matches.", + }; + await WorkloadCorpusManifest.WriteAsync(workloadDirectory, manifest, cancellationToken) + .ConfigureAwait(false); + return (index.Count, index.FingerprintSha256); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalPeerSmokeAccounting.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalPeerSmokeAccounting.cs new file mode 100644 index 00000000..bf4348c0 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalPeerSmokeAccounting.cs @@ -0,0 +1,120 @@ +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Deterministic completion gates for LocalPeerSmoke. Peer ARTICLE success alone is never enough. + /// + internal static class LocalPeerSmokeAccounting + { + /// Harness-visible counters used by smoke success criteria. + internal readonly record struct Evidence( + long RequestsStarted, + long WorkSuccess, + long WorkNotFound, + long WorkOtherFailure, + long ArticlesDelivered, + long OutstandingAtEnd, + long Timeouts, + long UnresolvedAtDispose, + long LateRepliesAfterTimeout, + long? PeerArticleRequests, + long? PeerArticleSuccess, + long? PeerArticleMissing, + long? PeerProtocolErrors); + + /// + /// Throws when smoke completion cannot be established. Does not invent counters for + /// stages the stress harness does not expose (e.g. per-message RMQ consume ACKs). + /// + internal static void AssertSuccessfulSmokeOrThrow(Evidence evidence) + { + if (evidence.RequestsStarted <= 0) + { + throw new InvalidOperationException( + "Smoke failed: no work items were submitted (RequestsStarted=0)."); + } + + if (evidence.ArticlesDelivered <= 0) + { + throw new InvalidOperationException( + "Smoke failed: ArticlesDelivered=0. Peer ARTICLE responses alone do not satisfy " + + "end-to-end delivery. Zero downstream TLS listener deliveries is a hard failure."); + } + + if (evidence.WorkSuccess <= 0) + { + throw new InvalidOperationException( + "Smoke failed: WorkSuccess=0 (missing RPC success completion) while deliveries were claimed."); + } + + if (evidence.ArticlesDelivered > evidence.WorkSuccess) + { + throw new InvalidOperationException( + $"Smoke failed: accounting mismatch ArticlesDelivered={evidence.ArticlesDelivered} " + + $"> WorkSuccess={evidence.WorkSuccess}."); + } + + if (evidence.OutstandingAtEnd != 0) + { + throw new InvalidOperationException( + $"Smoke failed: OutstandingAtEnd={evidence.OutstandingAtEnd} at completion deadline."); + } + + if (evidence.UnresolvedAtDispose != 0) + { + throw new InvalidOperationException( + $"Smoke failed: UnresolvedAtDispose={evidence.UnresolvedAtDispose}."); + } + + // Timeouts with zero deliveries already failed above; timeouts with some deliveries + // still fail smoke — a bounded smoke must be clean. + if (evidence.Timeouts > 0) + { + throw new InvalidOperationException( + $"Smoke failed: Timeouts={evidence.Timeouts}. Bounded smoke requires a clean run."); + } + + if (evidence.LateRepliesAfterTimeout > 0) + { + throw new InvalidOperationException( + $"Smoke failed: LateRepliesAfterTimeout={evidence.LateRepliesAfterTimeout}."); + } + + if (evidence.PeerArticleSuccess is { } peerOk) + { + if (peerOk < evidence.ArticlesDelivered) + { + throw new InvalidOperationException( + $"Smoke failed: peer ArticleSuccess={peerOk} < ArticlesDelivered={evidence.ArticlesDelivered}."); + } + + // Peer responses without downstream delivery: already covered by ArticlesDelivered<=0, + // but explicit when peerOk>0 and deliveries somehow zero is handled above. + if (peerOk > 0 && evidence.ArticlesDelivered == 0) + { + throw new InvalidOperationException( + "Smoke failed: peer returned ARTICLE success but ArticlesDelivered=0."); + } + } + + if (evidence.PeerArticleRequests is { } peerReq + && evidence.PeerArticleSuccess is { } peerSuccess + && peerReq < peerSuccess) + { + throw new InvalidOperationException( + $"Smoke failed: peer ArticleRequests={peerReq} < ArticleSuccess={peerSuccess}."); + } + + if (evidence.PeerArticleMissing is > 0) + { + throw new InvalidOperationException( + $"Smoke failed: peer ArticleMissing={evidence.PeerArticleMissing} for a corpus-backed smoke."); + } + + if (evidence.PeerProtocolErrors is > 0) + { + throw new InvalidOperationException( + $"Smoke failed: peer ProtocolErrors={evidence.PeerProtocolErrors}."); + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalPeerSmokeAccountingTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalPeerSmokeAccountingTests.cs new file mode 100644 index 00000000..fa9e09df --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalPeerSmokeAccountingTests.cs @@ -0,0 +1,101 @@ +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + public sealed class LocalPeerSmokeAccountingTests + { + [Fact] + public void Accepts_successful_end_to_end_completion() + { + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow(Clean(delivered: 5, workSuccess: 5, peerOk: 5)); + } + + [Fact] + public void Rejects_zero_downstream_deliveries_despite_peer_success() + { + var ex = Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 0, workSuccess: 5, peerOk: 5))); + Assert.Contains("ArticlesDelivered=0", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_missing_rpc_success() + { + var ex = Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 3, workSuccess: 0, peerOk: 3))); + Assert.Contains("WorkSuccess=0", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_failed_or_timeout_work() + { + var ex = Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 2, workSuccess: 2, peerOk: 2) with { Timeouts = 1 })); + Assert.Contains("Timeouts=", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_outstanding_work_at_deadline() + { + var ex = Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 2, workSuccess: 2, peerOk: 2) with { OutstandingAtEnd = 1 })); + Assert.Contains("OutstandingAtEnd=", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_mismatched_delivery_vs_work_success() + { + var ex = Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 5, workSuccess: 2, peerOk: 5))); + Assert.Contains("accounting mismatch", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_unresolved_cleanup() + { + var ex = Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 1, workSuccess: 1, peerOk: 1) with { UnresolvedAtDispose = 2 })); + Assert.Contains("UnresolvedAtDispose=", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_no_work_started() + { + var ex = Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 0, workSuccess: 0, peerOk: 0) with { RequestsStarted = 0 })); + Assert.Contains("RequestsStarted=0", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_peer_missing_or_protocol_errors() + { + Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 1, workSuccess: 1, peerOk: 1) with { PeerArticleMissing = 1 })); + Assert.Throws(() => + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + Clean(delivered: 1, workSuccess: 1, peerOk: 1) with { PeerProtocolErrors = 1 })); + } + + private static LocalPeerSmokeAccounting.Evidence Clean(long delivered, long workSuccess, long peerOk) => + new( + RequestsStarted: Math.Max(delivered, 1), + WorkSuccess: workSuccess, + WorkNotFound: 0, + WorkOtherFailure: 0, + ArticlesDelivered: delivered, + OutstandingAtEnd: 0, + Timeouts: 0, + UnresolvedAtDispose: 0, + LateRepliesAfterTimeout: 0, + PeerArticleRequests: peerOk, + PeerArticleSuccess: peerOk, + PeerArticleMissing: 0, + PeerProtocolErrors: 0); + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolation.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolation.cs new file mode 100644 index 00000000..a820aa08 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolation.cs @@ -0,0 +1,97 @@ +using System.Net; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Fail-closed helpers so lab benchmarks cannot silently target a real upstream. + /// + internal static class LocalUpstreamIsolation + { + /// Opt-in flag required alongside a loopback provider host for local-peer stress plans. + internal const string RequireLocalUpstreamEnvironmentVariable = + "VECTORNNTPD_BACKFILLER_E2E_STRESS_REQUIRE_LOCAL_UPSTREAM"; + + /// + /// Declared local peer hostname that BackFiller must use (lab MySQL hostname). + /// Required when is 1. + /// + internal const string LocalPeerHostEnvironmentVariable = + "VECTORNNTPD_BACKFILLER_E2E_STRESS_LOCAL_PEER_HOST"; + + /// + /// Declared local peer TCP port matching the running . + /// Required when is 1. + /// + internal const string LocalPeerPortEnvironmentVariable = + "VECTORNNTPD_BACKFILLER_E2E_STRESS_LOCAL_PEER_PORT"; + + /// + /// Returns whether is loopback (127.0.0.1, ::1, localhost). + /// + internal static bool IsLoopbackHost(string host) + { + if (string.IsNullOrWhiteSpace(host)) + { + return false; + } + + if (string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return IPAddress.TryParse(host, out var address) && IPAddress.IsLoopback(address); + } + + /// + /// Throws when is 1 and + /// is not loopback. + /// + internal static void EnsureLocalUpstreamOrThrow(string providerHost) + { + var require = Environment.GetEnvironmentVariable(RequireLocalUpstreamEnvironmentVariable); + if (!string.Equals(require, "1", StringComparison.Ordinal)) + { + return; + } + + if (!IsLoopbackHost(providerHost)) + { + throw new InvalidOperationException( + $"Local-upstream stress mode is enabled ({RequireLocalUpstreamEnvironmentVariable}=1) " + + $"but provider host '{providerHost}' is not loopback. Refusing to proceed."); + } + } + + /// + /// Fail-closed gate for local-peer stress: requires the flag, a loopback declared host, + /// and a valid peer port. Does not contact the network. + /// + /// Declared loopback host and port. + internal static (string Host, int Port) RequireLocalPeerEndpointOrThrow() + { + var require = Environment.GetEnvironmentVariable(RequireLocalUpstreamEnvironmentVariable); + if (!string.Equals(require, "1", StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Set {RequireLocalUpstreamEnvironmentVariable}=1 to run local-peer stress phases."); + } + + var host = Environment.GetEnvironmentVariable(LocalPeerHostEnvironmentVariable); + if (string.IsNullOrWhiteSpace(host) || !IsLoopbackHost(host)) + { + throw new InvalidOperationException( + $"{LocalPeerHostEnvironmentVariable} must be a loopback host when local-upstream mode is enabled."); + } + + if (!int.TryParse(Environment.GetEnvironmentVariable(LocalPeerPortEnvironmentVariable), out var port) + || port is < 1 or > 65535) + { + throw new InvalidOperationException( + $"{LocalPeerPortEnvironmentVariable} must be an integer TCP port in 1..65535."); + } + + return (host, port); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGate.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGate.cs new file mode 100644 index 00000000..51c19220 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGate.cs @@ -0,0 +1,267 @@ +using System.Net; +using System.Net.Sockets; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Fail-closed preflight for local ARTICLE peer stress. Validates declared env, every published + /// upstream in an operator snapshot, peer reachability, and port match against 127.0.0.1. + /// + /// + /// This gate does not wrap the running BackFiller . + /// Dial-time enforcement inside a separately running service requires a production-path allowlist + /// (documented in ISOLATION_AND_ACCOUNTING_FIXES.md) and is out of scope until approved. + /// + internal static class LocalUpstreamIsolationGate + { + /// Only accepted host for the planned local peer (literal IPv4; no DNS). + internal const string RequiredLiteralHost = "127.0.0.1"; + + /// Path to sanitized upstream snapshot JSON. + internal const string UpstreamSnapshotPathEnvironmentVariable = + "VECTORNNTPD_BACKFILLER_E2E_STRESS_UPSTREAM_SNAPSHOT"; + + /// Max age of the snapshot before it is considered stale (default 15 minutes). + internal const string SnapshotMaxAgeSecondsEnvironmentVariable = + "VECTORNNTPD_BACKFILLER_E2E_STRESS_UPSTREAM_SNAPSHOT_MAX_AGE_SECONDS"; + + /// Inputs for one isolation evaluation (fully injectable for tests). + internal sealed record Requirements + { + public required bool RequireLocalUpstreamFlag { get; init; } + + public string? DeclaredHost { get; init; } + + public int? DeclaredPort { get; init; } + + public required UpstreamProviderSnapshot? Snapshot { get; init; } + + public required TimeSpan MaxSnapshotAge { get; init; } + + public required DateTimeOffset NowUtc { get; init; } + + /// + /// Returns whether TCP connect to host:port succeeds. Injected so unit tests never dial WAN. + /// + public required Func PeerReachable { get; init; } + } + + /// + /// Builds from environment and optional snapshot path. + /// Does not open MySQL. Peer probe defaults to a loopback TCP connect attempt. + /// + internal static Requirements FromEnvironment(Func? peerReachable = null) + { + var require = string.Equals( + Environment.GetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable), + "1", + StringComparison.Ordinal); + + int? port = null; + if (int.TryParse( + Environment.GetEnvironmentVariable(LocalUpstreamIsolation.LocalPeerPortEnvironmentVariable), + out var parsed) + && parsed is >= 1 and <= 65535) + { + port = parsed; + } + + UpstreamProviderSnapshot? snapshot = null; + var snapshotPath = Environment.GetEnvironmentVariable(UpstreamSnapshotPathEnvironmentVariable); + if (!string.IsNullOrWhiteSpace(snapshotPath) && File.Exists(snapshotPath)) + { + snapshot = UpstreamProviderSnapshot.ReadAsync(snapshotPath, CancellationToken.None) + .GetAwaiter() + .GetResult(); + } + + var maxAgeSeconds = 15 * 60; + if (int.TryParse( + Environment.GetEnvironmentVariable(SnapshotMaxAgeSecondsEnvironmentVariable), + out var configuredAge) + && configuredAge > 0) + { + maxAgeSeconds = configuredAge; + } + + return new Requirements + { + RequireLocalUpstreamFlag = require, + DeclaredHost = Environment.GetEnvironmentVariable( + LocalUpstreamIsolation.LocalPeerHostEnvironmentVariable), + DeclaredPort = port, + Snapshot = snapshot, + MaxSnapshotAge = TimeSpan.FromSeconds(maxAgeSeconds), + NowUtc = DateTimeOffset.UtcNow, + PeerReachable = peerReachable ?? ProbeLoopbackTcp, + }; + } + + /// + /// Validates only the require-flag and declared 127.0.0.1:port (workload discovery). + /// Does not inspect published providers or peer reachability. + /// + internal static (string Host, int Port) EnsureDeclaredEndpointOrThrow(Requirements requirements) + { + ArgumentNullException.ThrowIfNull(requirements); + + if (!requirements.RequireLocalUpstreamFlag) + { + throw new InvalidOperationException( + $"Missing local-upstream requirement: set " + + $"{LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable}=1."); + } + + if (string.IsNullOrWhiteSpace(requirements.DeclaredHost)) + { + throw new InvalidOperationException( + $"Missing host: set {LocalUpstreamIsolation.LocalPeerHostEnvironmentVariable}={RequiredLiteralHost}."); + } + + if (!string.Equals(requirements.DeclaredHost, RequiredLiteralHost, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Declared peer host must be literal {RequiredLiteralHost} (no DNS, no localhost, no IPv6). " + + $"Got '{requirements.DeclaredHost}'."); + } + + if (requirements.DeclaredPort is null) + { + throw new InvalidOperationException( + $"Missing or invalid port: set {LocalUpstreamIsolation.LocalPeerPortEnvironmentVariable} " + + "to the peer TCP port (1..65535)."); + } + + return (RequiredLiteralHost, requirements.DeclaredPort.Value); + } + + /// + /// Fail-closed evaluation. Throws with actionable, + /// non-secret messages when isolation cannot be established. + /// + internal static (string Host, int Port) EnsureReadyOrThrow(Requirements requirements) + { + ArgumentNullException.ThrowIfNull(requirements); + ArgumentNullException.ThrowIfNull(requirements.PeerReachable); + + var (_, port) = EnsureDeclaredEndpointOrThrow(requirements); + if (requirements.Snapshot is null) + { + throw new InvalidOperationException( + "Unknown or uninspectable effective upstream destination: provide a sanitized " + + $"provider snapshot via {UpstreamSnapshotPathEnvironmentVariable}. " + + "Without it the harness cannot verify every published account is local-only."); + } + + ValidateSnapshot(requirements.Snapshot, port, requirements.NowUtc, requirements.MaxSnapshotAge); + + if (!requirements.PeerReachable(RequiredLiteralHost, port)) + { + throw new InvalidOperationException( + $"Local peer is unavailable at {RequiredLiteralHost}:{port}. " + + "Refusing to submit work (no external upstream fallback)."); + } + + return (RequiredLiteralHost, port); + } + + /// Validates every published provider against the intended peer endpoint. + internal static void ValidateSnapshot( + UpstreamProviderSnapshot snapshot, + int expectedPort, + DateTimeOffset nowUtc, + TimeSpan maxAge) + { + ArgumentNullException.ThrowIfNull(snapshot); + + if (string.IsNullOrWhiteSpace(snapshot.Source) + || string.Equals(snapshot.Source, "unknown", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + "Upstream snapshot Source is missing or unknown; refuse uninspectable effective destination."); + } + + if (string.IsNullOrWhiteSpace(snapshot.PoolRefreshEvidence)) + { + throw new InvalidOperationException( + "Upstream snapshot lacks PoolRefreshEvidence (stale or unverified session-pool configuration). " + + "Record log events 5601/5603 or equivalent after account refresh."); + } + + var age = nowUtc - snapshot.CapturedUtc; + if (age < TimeSpan.Zero || age > maxAge) + { + throw new InvalidOperationException( + $"Upstream snapshot is stale or from the future (age={age}, max={maxAge}). Recapture after pool refresh."); + } + + if (snapshot.Providers is null || snapshot.Providers.Count == 0) + { + throw new InvalidOperationException( + "Upstream snapshot lists zero providers; cannot prove local-only dialing."); + } + + foreach (var provider in snapshot.Providers) + { + if (string.IsNullOrWhiteSpace(provider.Backbone)) + { + throw new InvalidOperationException("Upstream snapshot contains a provider with empty Backbone."); + } + + if (string.IsNullOrWhiteSpace(provider.Host)) + { + throw new InvalidOperationException( + $"Provider '{provider.Backbone}' has a missing host."); + } + + if (!string.Equals(provider.Host, RequiredLiteralHost, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Provider '{provider.Backbone}' host '{provider.Host}' is not {RequiredLiteralHost}. " + + "Every published account for the lab ServerId must target the local peer."); + } + + if (provider.Port != expectedPort) + { + throw new InvalidOperationException( + $"Provider '{provider.Backbone}' port {provider.Port} does not match peer port {expectedPort}."); + } + + if (provider.UseTls) + { + throw new InvalidOperationException( + $"Provider '{provider.Backbone}' has UseTls=true; local peer is cleartext (usessl=n required)."); + } + + // Reject IPv6 literals if somehow present as Host string. + if (IPAddress.TryParse(provider.Host, out var address) + && address.AddressFamily == AddressFamily.InterNetworkV6) + { + throw new InvalidOperationException( + $"Provider '{provider.Backbone}' uses IPv6; planned peer requires {RequiredLiteralHost} only."); + } + } + } + + /// Default peer probe: TCP connect to loopback only (never used for non-loopback hosts). + internal static bool ProbeLoopbackTcp(string host, int port) + { + if (!string.Equals(host, RequiredLiteralHost, StringComparison.Ordinal)) + { + return false; + } + + try + { + using var client = new TcpClient(); + var connect = client.ConnectAsync(IPAddress.Loopback, port); + return connect.Wait(TimeSpan.FromSeconds(2)) && client.Connected; + } + catch (Exception ex) when (ex is SocketException or AggregateException or ObjectDisposedException) + { + return false; + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGateTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGateTests.cs new file mode 100644 index 00000000..417f5e42 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGateTests.cs @@ -0,0 +1,220 @@ +using VectorNNTPD.BackFillerService.Nntp; +using Microsoft.Extensions.Logging.Abstractions; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + public sealed class LocalUpstreamIsolationGateTests + { + [Fact] + public void Rejects_missing_require_flag() + { + var ex = Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirements() with + { + RequireLocalUpstreamFlag = false, + })); + Assert.Contains("Missing local-upstream requirement", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_missing_host() + { + var ex = Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirements() with { DeclaredHost = null })); + Assert.Contains("Missing host", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_missing_port() + { + var ex = Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirements() with { DeclaredPort = null })); + Assert.Contains("Missing or invalid port", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_localhost_and_ipv6_declared_host() + { + Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow( + ValidRequirements() with { DeclaredHost = "localhost" })); + Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow( + ValidRequirements() with { DeclaredHost = "::1" })); + Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow( + ValidRequirements() with { DeclaredHost = "8.8.8.8" })); + } + + [Fact] + public void Rejects_mismatched_peer_port_in_snapshot() + { + var snapshot = ValidSnapshot(port: 11119); + var ex = Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirementsWithSnapshot(snapshot) with + { + DeclaredPort = 22222, + })); + Assert.Contains("does not match peer port", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_external_provider_among_multiple_accounts() + { + var snapshot = new UpstreamProviderSnapshot + { + CapturedUtc = DateTimeOffset.UtcNow, + ServerId = 1, + Source = "unit-test", + PoolRefreshEvidence = "synthetic", + Providers = + [ + new UpstreamProviderEndpoint + { + Backbone = "Giganews", + Host = "127.0.0.1", + Port = 11119, + UseTls = false, + }, + new UpstreamProviderEndpoint + { + Backbone = "Eweka", + Host = "news.example.com", + Port = 563, + UseTls = true, + }, + ], + }; + var ex = Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirementsWithSnapshot(snapshot))); + Assert.Contains("Eweka", ex.Message, StringComparison.Ordinal); + Assert.Contains("not 127.0.0.1", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Rejects_missing_snapshot_as_uninspectable() + { + var ex = Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirementsWithSnapshot(null))); + Assert.Contains("uninspectable", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Rejects_stale_or_unverified_pool_snapshot() + { + var stale = ValidSnapshot() with + { + CapturedUtc = DateTimeOffset.UtcNow - TimeSpan.FromHours(2), + }; + Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirementsWithSnapshot(stale))); + + var unverified = ValidSnapshot() with { PoolRefreshEvidence = " " }; + Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirementsWithSnapshot(unverified))); + + var unknownSource = ValidSnapshot() with { Source = "unknown" }; + Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirementsWithSnapshot(unknownSource))); + } + + [Fact] + public void Rejects_unavailable_local_peer_without_external_fallback() + { + var ex = Assert.Throws(() => + LocalUpstreamIsolationGate.EnsureReadyOrThrow( + ValidRequirements() with { PeerReachable = static (_, _) => false })); + Assert.Contains("unavailable", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("no external upstream fallback", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Accepts_fully_verified_local_only_configuration() + { + var endpoint = LocalUpstreamIsolationGate.EnsureReadyOrThrow(ValidRequirements()); + Assert.Equal("127.0.0.1", endpoint.Host); + Assert.Equal(11119, endpoint.Port); + } + + [Fact] + public async Task Allowlist_transport_blocks_external_and_allows_local_peer() + { + var dir = Path.Combine(Path.GetTempPath(), "vnntp-allowlist-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + var messageId = $""; + await File.WriteAllTextAsync( + Path.Combine(dir, "a1"), + $"Message-ID: {messageId}\r\nFrom: a@b\r\n\r\nbody\r\n"); + + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = 0 }); + + var allowlist = new LoopbackAllowlistNntpTransportFactory( + new TcpNntpTransportFactory(), + "127.0.0.1", + peer.Port); + + // NntpProviderSession classifies transport exceptions as non-reusable provider failure (no throw). + var bad = new BackFillerProviderDefinition( + "Giganews", "news.example.com", 563, true, null, null, 0, 1); + await using (var session = new NntpProviderSession( + bad, NntpSessionOptions.Default, NullLogger.Instance)) + { + var refused = await session.ConnectAsync(allowlist, CancellationToken.None); + Assert.NotNull(refused); + Assert.Equal(ArticleRetrievalKind.ProviderFailure, refused!.Kind); + Assert.False(refused.SessionReusable); + } + + var good = new BackFillerProviderDefinition( + "Giganews", "127.0.0.1", peer.Port, false, null, null, 0, 1); + await using var okSession = new NntpProviderSession( + good, NntpSessionOptions.Default, NullLogger.Instance); + Assert.Null(await okSession.ConnectAsync(allowlist, CancellationToken.None)); + using var result = await okSession.DownloadArticleAsync(messageId, CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, result.Kind); + } + + private static LocalUpstreamIsolationGate.Requirements ValidRequirements() => + ValidRequirementsWithSnapshot(ValidSnapshot()); + + private static LocalUpstreamIsolationGate.Requirements ValidRequirementsWithSnapshot( + UpstreamProviderSnapshot? snapshot) => + new() + { + RequireLocalUpstreamFlag = true, + DeclaredHost = "127.0.0.1", + DeclaredPort = 11119, + Snapshot = snapshot, + MaxSnapshotAge = TimeSpan.FromMinutes(15), + NowUtc = DateTimeOffset.UtcNow, + PeerReachable = static (_, _) => true, + }; + + private static UpstreamProviderSnapshot ValidSnapshot(int port = 11119) => + new() + { + CapturedUtc = DateTimeOffset.UtcNow, + ServerId = 1, + Source = "unit-test", + PoolRefreshEvidence = "synthetic-refresh-ok", + Providers = + [ + new UpstreamProviderEndpoint + { + Backbone = "Giganews", + Host = "127.0.0.1", + Port = port, + UseTls = false, + }, + new UpstreamProviderEndpoint + { + Backbone = "Eweka", + Host = "127.0.0.1", + Port = port, + UseTls = false, + }, + ], + }; + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationTests.cs new file mode 100644 index 00000000..55d7984e --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationTests.cs @@ -0,0 +1,131 @@ +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + public sealed class LocalUpstreamIsolationTests + { + [Theory] + [InlineData("127.0.0.1", true)] + [InlineData("::1", true)] + [InlineData("localhost", true)] + [InlineData("news.example.com", false)] + [InlineData("8.8.8.8", false)] + public void IsLoopbackHost_classifies_hosts(string host, bool expected) => + Assert.Equal(expected, LocalUpstreamIsolation.IsLoopbackHost(host)); + + [Fact] + public void EnsureLocalUpstreamOrThrow_is_noop_when_flag_unset() + { + var previous = Environment.GetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable); + try + { + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable, + null); + LocalUpstreamIsolation.EnsureLocalUpstreamOrThrow("news.example.com"); + } + finally + { + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable, + previous); + } + } + + [Fact] + public void EnsureLocalUpstreamOrThrow_rejects_non_loopback_when_flag_set() + { + var previous = Environment.GetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable); + try + { + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable, + "1"); + Assert.Throws(() => + LocalUpstreamIsolation.EnsureLocalUpstreamOrThrow("news.example.com")); + LocalUpstreamIsolation.EnsureLocalUpstreamOrThrow("127.0.0.1"); + } + finally + { + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable, + previous); + } + } + + [Fact] + public void ValidateBindOrThrow_rejects_non_loopback_without_override() + { + var previous = Environment.GetEnvironmentVariable( + LocalArticleNntpPeerOptions.AllowNonLoopbackEnvironmentVariable); + try + { + Environment.SetEnvironmentVariable( + LocalArticleNntpPeerOptions.AllowNonLoopbackEnvironmentVariable, + null); + var options = new LocalArticleNntpPeerOptions + { + CorpusRoot = Path.GetTempPath(), + BindAddress = System.Net.IPAddress.Parse("8.8.8.8"), + }; + Assert.Throws(options.ValidateBindOrThrow); + } + finally + { + Environment.SetEnvironmentVariable( + LocalArticleNntpPeerOptions.AllowNonLoopbackEnvironmentVariable, + previous); + } + } + + [Fact] + public void RequireLocalPeerEndpointOrThrow_requires_flag_host_and_port() + { + var prevRequire = Environment.GetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable); + var prevHost = Environment.GetEnvironmentVariable( + LocalUpstreamIsolation.LocalPeerHostEnvironmentVariable); + var prevPort = Environment.GetEnvironmentVariable( + LocalUpstreamIsolation.LocalPeerPortEnvironmentVariable); + try + { + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable, + null); + Assert.Throws( + static () => LocalUpstreamIsolation.RequireLocalPeerEndpointOrThrow()); + + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable, + "1"); + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.LocalPeerHostEnvironmentVariable, + "news.example.com"); + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.LocalPeerPortEnvironmentVariable, + "11119"); + Assert.Throws( + static () => LocalUpstreamIsolation.RequireLocalPeerEndpointOrThrow()); + + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.LocalPeerHostEnvironmentVariable, + "127.0.0.1"); + var endpoint = LocalUpstreamIsolation.RequireLocalPeerEndpointOrThrow(); + Assert.Equal("127.0.0.1", endpoint.Host); + Assert.Equal(11119, endpoint.Port); + } + finally + { + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.RequireLocalUpstreamEnvironmentVariable, + prevRequire); + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.LocalPeerHostEnvironmentVariable, + prevHost); + Environment.SetEnvironmentVariable( + LocalUpstreamIsolation.LocalPeerPortEnvironmentVariable, + prevPort); + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LoopbackAllowlistNntpTransportFactory.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LoopbackAllowlistNntpTransportFactory.cs new file mode 100644 index 00000000..be0adda9 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LoopbackAllowlistNntpTransportFactory.cs @@ -0,0 +1,61 @@ +using VectorNNTPD.BackFillerService.Nntp; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Lab/test transport decorator that refuses any connect whose host/port is not the allowlisted + /// local peer. Demonstrates dial-time fail-closed behaviour for automated tests. + /// + /// + /// Not registered in production DI. Wiring an equivalent decorator into the running BackFiller + /// service is the minimum production-path change required for process-level dial enforcement + /// and requires separate approval (see ISOLATION_AND_ACCOUNTING_FIXES.md). + /// + internal sealed class LoopbackAllowlistNntpTransportFactory : INntpTransportFactory + { + private readonly INntpTransportFactory _inner; + private readonly string _allowedHost; + private readonly int _allowedPort; + + internal LoopbackAllowlistNntpTransportFactory( + INntpTransportFactory inner, + string allowedHost, + int allowedPort) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _allowedHost = allowedHost ?? throw new ArgumentNullException(nameof(allowedHost)); + if (allowedPort is < 1 or > 65535) + { + throw new ArgumentOutOfRangeException(nameof(allowedPort)); + } + + _allowedPort = allowedPort; + } + + /// + public Task ConnectAsync( + BackFillerProviderDefinition provider, + NntpSessionOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(provider); + if (!string.Equals(provider.Host, _allowedHost, StringComparison.Ordinal) + || provider.Port != _allowedPort) + { + throw new InvalidOperationException( + $"Lab allowlist refused upstream connect backbone={provider.Backbone} " + + $"host={provider.Host} port={provider.Port}; " + + $"allowed={_allowedHost}:{_allowedPort}. No external fallback."); + } + + if (provider.UseTls) + { + throw new InvalidOperationException( + $"Lab allowlist refused TLS upstream connect for backbone={provider.Backbone} " + + "(local peer is cleartext)."); + } + + return _inner.ConnectAsync(provider, options, cancellationToken); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/NntpWireFramer.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/NntpWireFramer.cs new file mode 100644 index 00000000..703525c6 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/NntpWireFramer.cs @@ -0,0 +1,67 @@ +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Converts logical on-disk article bytes into an NNTP multiline ARTICLE body (dot-stuffed + terminator). + /// Corpus files are not assumed to be wire-framed. + /// + internal static class NntpWireFramer + { + /// + /// Adds one stuffing dot to lines that begin with . and appends the terminating .CRLF. + /// Does not alter logical article content beyond transport stuffing. + /// + internal static byte[] FrameLogicalArticle(ReadOnlySpan article) + { + using var buffer = new MemoryStream(article.Length + 64); + var offset = 0; + while (offset < article.Length) + { + var lineEnd = article[offset..].IndexOf((byte)'\n'); + var next = lineEnd < 0 ? article.Length : offset + lineEnd + 1; + var contentEnd = lineEnd < 0 + ? article.Length + : (lineEnd > 0 && article[offset + lineEnd - 1] == (byte)'\r' + ? offset + lineEnd - 1 + : offset + lineEnd); + if (contentEnd > offset && article[offset] == (byte)'.') + { + buffer.WriteByte((byte)'.'); + } + + buffer.Write(article[offset..next]); + if (lineEnd < 0) + { + buffer.WriteByte((byte)'\r'); + buffer.WriteByte((byte)'\n'); + } + + offset = next; + } + + buffer.WriteByte((byte)'.'); + buffer.WriteByte((byte)'\r'); + buffer.WriteByte((byte)'\n'); + return buffer.ToArray(); + } + + /// + /// Streams a framed ARTICLE body from a logical article file without loading the whole file when possible. + /// For simplicity and correctness with mixed EOL, this implementation reads the file then frames + /// (bounded by ). + /// + internal static async Task<(byte[] Logical, byte[] Framed)> ReadAndFrameAsync( + string path, + int maxArticleBytes, + CancellationToken cancellationToken) + { + var logical = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); + if (logical.Length > maxArticleBytes) + { + throw new InvalidOperationException( + $"Article file exceeds maxArticleBytes ({logical.Length} > {maxArticleBytes}): {path}"); + } + + return (logical, FrameLogicalArticle(logical)); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/NntpWireFramerTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/NntpWireFramerTests.cs new file mode 100644 index 00000000..54847004 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/NntpWireFramerTests.cs @@ -0,0 +1,50 @@ +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + public sealed class NntpWireFramerTests + { + [Fact] + public void Frame_appends_terminator_and_stuffs_leading_dot_lines() + { + var logical = "Message-ID: \r\n\r\n.leading\r\nbody\r\n"u8.ToArray(); + var framed = NntpWireFramer.FrameLogicalArticle(logical); + Assert.True(framed.AsSpan().EndsWith(".\r\n"u8)); + var framedText = System.Text.Encoding.ASCII.GetString(framed); + // Leading '.' on a content line is stuffed as '..' on the same line (RFC 3977). + Assert.Contains("..leading\r\n", framedText, StringComparison.Ordinal); + Assert.False(logical.AsSpan().EndsWith(".\r\n"u8)); + } + + [Fact] + public void Frame_does_not_treat_already_framed_input_as_logical() + { + var logical = "Message-ID: \r\n\r\nx\r\n"u8.ToArray(); + var framed = NntpWireFramer.FrameLogicalArticle(logical); + var again = NntpWireFramer.FrameLogicalArticle(framed); + // Framing a wire body again stuffs the terminator line — peers must frame logical files once. + Assert.NotEqual(framed, again); + Assert.Contains("..\r\n", System.Text.Encoding.ASCII.GetString(again), StringComparison.Ordinal); + } + + [Fact] + public void Empty_logical_article_still_emits_terminator() + { + var framed = NntpWireFramer.FrameLogicalArticle([]); + Assert.Equal(".\r\n"u8.ToArray(), framed); + } + + /// + /// The lab peer preserves logical EOLs (stuffing only). A bare LF in the corpus remains on the wire, + /// so callers must supply exclusive-CRLF logical articles rather than relying on framing to normalize. + /// + [Fact] + public void Frame_preserves_bare_lf_in_logical_article() + { + var logical = "Message-ID: \r\nPath: x\n\r\nbody\r\n"u8.ToArray(); + var framed = NntpWireFramer.FrameLogicalArticle(logical); + Assert.Contains((byte)'\n', logical); + Assert.True(framed.AsSpan().IndexOf("Path: x\n\r\n"u8) >= 0); + Assert.True(framed.AsSpan().EndsWith(".\r\n"u8)); + } + + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs new file mode 100644 index 00000000..185f09cf --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs @@ -0,0 +1,131 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Lab-only immutable store of pre-framed ARTICLE responses. Built once at peer startup; + /// shared buffers are never mutated after construction. + /// + internal sealed class PreloadedArticleStore + { + private readonly ConcurrentDictionary _articles = + new(StringComparer.Ordinal); + + private PreloadedArticleStore( + IReadOnlyDictionary articles, + PreloadInitializationMetrics metrics) + { + foreach (var pair in articles) + { + _articles[pair.Key] = pair.Value; + } + + Metrics = metrics; + } + + /// Startup load counters (not reset during the measured interval). + internal PreloadInitializationMetrics Metrics { get; } + + /// Number of Message-IDs with a precomputed response. + internal int Count => _articles.Count; + + /// Resolves a precomputed ARTICLE response for . + internal bool TryGet(string messageId, out PreloadedArticle article) => + _articles.TryGetValue(messageId, out article!); + + /// + /// Loads every indexed article, frames once, and stores status+body wire bytes. + /// Throws when any article cannot be read or framed (fail-closed startup). + /// + internal static async Task LoadAsync( + CorpusArticleIndex index, + int maxArticleBytes, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(index); + ArgumentOutOfRangeException.ThrowIfLessThan(maxArticleBytes, 1); + + var sw = Stopwatch.StartNew(); + var map = new Dictionary(index.Count, StringComparer.Ordinal); + long logicalBytes = 0; + long wireBytes = 0; + var failures = 0; + Exception? firstFailure = null; + + foreach (var messageId in index.MessageIds.OrderBy(static id => id, StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!index.TryResolve(messageId, out var path)) + { + failures++; + firstFailure ??= new FileNotFoundException( + $"Indexed Message-ID could not be resolved to an existing file: {messageId}"); + continue; + } + + try + { + var (logical, framed) = await NntpWireFramer + .ReadAndFrameAsync(path, maxArticleBytes, cancellationToken) + .ConfigureAwait(false); + var status = Encoding.ASCII.GetBytes($"220 0 {messageId} article follows\r\n"); + var wire = new byte[status.Length + framed.Length]; + Buffer.BlockCopy(status, 0, wire, 0, status.Length); + Buffer.BlockCopy(framed, 0, wire, status.Length, framed.Length); + map[messageId] = new PreloadedArticle(wire, logical.Length); + logicalBytes += logical.Length; + wireBytes += wire.Length; + } + catch (Exception ex) + { + failures++; + firstFailure ??= ex; + } + } + + sw.Stop(); + var metrics = new PreloadInitializationMetrics( + ArticlesLoaded: map.Count, + LogicalBytesLoaded: logicalBytes, + WireBytesLoaded: wireBytes, + LoadDuration: sw.Elapsed, + LoadFailures: failures); + + if (failures > 0) + { + throw new InvalidOperationException( + $"PreloadedArticleStore failed to load {failures} article(s) " + + $"(loaded={map.Count}, duration={sw.Elapsed}).", + firstFailure); + } + + if (map.Count != index.Count) + { + throw new InvalidOperationException( + $"PreloadedArticleStore count mismatch: loaded={map.Count} index={index.Count}."); + } + + return new PreloadedArticleStore(map, metrics); + } + } + + /// Immutable precomputed ARTICLE wire response for one Message-ID. + /// Status line + framed body including terminator; never mutated. + /// Logical on-disk article size (destuffed payload length). + internal sealed record PreloadedArticle(byte[] WireResponse, int LogicalByteCount); + + /// One-shot preload initialization counters exposed for lab evidence. + /// Articles successfully framed into the store. + /// Sum of logical article sizes. + /// Sum of status+framed response sizes retained in memory. + /// Wall time for the preload pass. + /// Articles that failed to read or frame (startup aborts when > 0). + internal sealed record PreloadInitializationMetrics( + int ArticlesLoaded, + long LogicalBytesLoaded, + long WireBytesLoaded, + TimeSpan LoadDuration, + int LoadFailures); +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs new file mode 100644 index 00000000..faaf6d82 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs @@ -0,0 +1,266 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.BackFillerService.Nntp; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + public sealed class PreloadedArticleStoreTests + { + [Fact] + public async Task Preload_matches_file_path_framing_byte_for_byte_including_terminator() + { + var dir = CreateFixtureCorpus(out var messageId, includeDotLine: true); + var index = await CorpusArticleIndex.BuildAsync(dir); + Assert.True(index.TryResolve(messageId, out var path)); + + var (logical, framed) = await NntpWireFramer.ReadAndFrameAsync(path, 1024 * 1024, CancellationToken.None); + var status = Encoding.ASCII.GetBytes($"220 0 {messageId} article follows\r\n"); + var expectedWire = new byte[status.Length + framed.Length]; + Buffer.BlockCopy(status, 0, expectedWire, 0, status.Length); + Buffer.BlockCopy(framed, 0, expectedWire, status.Length, framed.Length); + + var store = await PreloadedArticleStore.LoadAsync(index, 1024 * 1024); + Assert.Equal(1, store.Count); + Assert.Equal(0, store.Metrics.LoadFailures); + Assert.True(store.TryGet(messageId, out var preloaded)); + Assert.Equal(logical.Length, preloaded.LogicalByteCount); + Assert.True(expectedWire.AsSpan().SequenceEqual(preloaded.WireResponse)); + Assert.True(EndsWithArticleTerminator(preloaded.WireResponse)); + Assert.Contains("..dotline"u8, preloaded.WireResponse.AsSpan()); + + await using var filePeer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions { CorpusRoot = dir, Port = 0 }); + await using var memPeer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = dir, + Port = 0, + PreloadArticlesIntoMemory = true, + }); + + var fileWire = await CaptureArticleResponseAsync(filePeer.Port, messageId); + var memWire = await CaptureArticleResponseAsync(memPeer.Port, messageId); + Assert.True(expectedWire.AsSpan().SequenceEqual(fileWire)); + Assert.True(expectedWire.AsSpan().SequenceEqual(memWire)); + Assert.Equal(1, filePeer.Metrics.ArticleSuccess); + Assert.Equal(1, memPeer.Metrics.ArticleSuccess); + Assert.Equal(filePeer.Metrics.LogicalBytesTransferred, memPeer.Metrics.LogicalBytesTransferred); + Assert.Equal(filePeer.Metrics.WireBytesTransferred, memPeer.Metrics.WireBytesTransferred); + } + + [Fact] + public async Task Concurrent_sessions_share_precomputed_buffers_without_corruption() + { + var dir = CreateFixtureCorpus(out var messageId, includeDotLine: true); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = dir, + Port = 0, + MaxConcurrentSessions = 32, + PreloadArticlesIntoMemory = true, + }); + + Assert.True(peer.PreloadedStore!.TryGet(messageId, out var article)); + var expected = article.WireResponse; + + var tasks = Enumerable.Range(0, 16).Select(async _ => + { + var wire = await CaptureArticleResponseAsync(peer.Port, messageId); + Assert.True(expected.AsSpan().SequenceEqual(wire)); + }); + await Task.WhenAll(tasks); + Assert.Equal(16, peer.Metrics.ArticleSuccess); + Assert.Equal(0, peer.Metrics.OutstandingArticles); + Assert.True(peer.PreloadedStore.TryGet(messageId, out var again)); + Assert.Same(expected, again.WireResponse); + } + + [Fact] + public async Task Missing_article_returns_430_in_memory_mode() + { + var dir = CreateFixtureCorpus(out _, includeDotLine: false); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = dir, + Port = 0, + PreloadArticlesIntoMemory = true, + }); + + using var client = new System.Net.Sockets.TcpClient(); + await client.ConnectAsync(System.Net.IPAddress.Loopback, peer.Port); + await using var stream = client.GetStream(); + _ = await ReadLineAsync(stream); + await stream.WriteAsync("ARTICLE \r\n"u8.ToArray()); + Assert.StartsWith("430 ", await ReadLineAsync(stream)); + Assert.Equal(1, peer.Metrics.ArticleMissing); + Assert.Equal(0, peer.Metrics.ArticleSuccess); + } + + [Fact] + public async Task Startup_fails_when_indexed_article_cannot_be_loaded() + { + var dir = CreateFixtureCorpus(out var messageId, includeDotLine: false); + var index = await CorpusArticleIndex.BuildAsync(dir); + Assert.Equal(1, index.Count); + Assert.True(index.TryResolve(messageId, out var path)); + File.Delete(path); + + var ex = await Assert.ThrowsAsync(async () => + await PreloadedArticleStore.LoadAsync(index, maxArticleBytes: 1024 * 1024)); + Assert.Contains("failed to load", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Memory_mode_counters_and_dispose_rebind() + { + var dir = CreateFixtureCorpus(out var messageId, includeDotLine: false); + int port; + await using (var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = dir, + Port = 0, + PreloadArticlesIntoMemory = true, + })) + { + port = peer.Port; + var wire = await CaptureArticleResponseAsync(peer.Port, messageId); + Assert.True(EndsWithArticleTerminator(wire)); + Assert.Equal(1, peer.Metrics.ArticleRequests); + Assert.Equal(1, peer.Metrics.ArticleSuccess); + Assert.True(peer.Metrics.LogicalBytesTransferred > 0); + Assert.True(peer.Metrics.WireBytesTransferred > peer.Metrics.LogicalBytesTransferred); + Assert.Equal(0, peer.Metrics.OutstandingArticles); + } + + await using var peer2 = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = dir, + Port = port, + PreloadArticlesIntoMemory = true, + }); + Assert.Equal(port, peer2.Port); + Assert.Equal("memory", peer2.PeerMode); + } + + [Fact] + public async Task BackFiller_client_downloads_from_memory_peer() + { + var dir = CreateFixtureCorpus(out var messageId, includeDotLine: false); + await using var peer = await LocalArticleNntpPeer.StartAsync( + new LocalArticleNntpPeerOptions + { + CorpusRoot = dir, + Port = 0, + PreloadArticlesIntoMemory = true, + }); + + LocalUpstreamIsolation.EnsureLocalUpstreamOrThrow("127.0.0.1"); + var provider = new BackFillerProviderDefinition( + "LabLocal", + "127.0.0.1", + peer.Port, + UseTls: false, + Username: null, + Password: null, + MinSessions: 0, + MaxSessions: 1); + await using var session = new NntpProviderSession( + provider, + NntpSessionOptions.Default, + NullLogger.Instance); + var transport = new TcpNntpTransportFactory(); + Assert.Null(await session.ConnectAsync(transport, CancellationToken.None)); + using var result = await session.DownloadArticleAsync(messageId, CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, result.Kind); + Assert.Contains("hello"u8, result.Article!.Memory.Span); + Assert.Equal(1, peer.Metrics.ArticleSuccess); + } + + private static string CreateFixtureCorpus(out string messageId, bool includeDotLine) + { + messageId = $""; + var dir = Path.Combine(Path.GetTempPath(), "vnntp-preload-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + var body = includeDotLine + ? $"Message-ID: {messageId}\r\nFrom: lab@test\r\n\r\nhello\r\n.dotline\r\n" + : $"Message-ID: {messageId}\r\nFrom: lab@test\r\n\r\nhello\r\n"; + File.WriteAllText(Path.Combine(dir, "article1"), body); + return dir; + } + + /// + /// Captures the raw ARTICLE response (status + framed body) by reading until the multiline terminator. + /// + private static async Task CaptureArticleResponseAsync(int port, string messageId) + { + using var client = new System.Net.Sockets.TcpClient(); + await client.ConnectAsync(System.Net.IPAddress.Loopback, port); + await using var stream = client.GetStream(); + _ = await ReadLineAsync(stream); + await stream.WriteAsync(Encoding.ASCII.GetBytes($"ARTICLE {messageId}\r\n")); + + using var buffer = new MemoryStream(); + var one = new byte[1]; + while (true) + { + var n = await stream.ReadAsync(one); + if (n == 0) + { + break; + } + + buffer.WriteByte(one[0]); + if (buffer.Length >= 5) + { + var arr = buffer.GetBuffer(); + var len = (int)buffer.Length; + if (arr[len - 5] == (byte)'\r' + && arr[len - 4] == (byte)'\n' + && arr[len - 3] == (byte)'.' + && arr[len - 2] == (byte)'\r' + && arr[len - 1] == (byte)'\n') + { + // Ensure this is a solitary terminator line (preceded by CRLF or start-of-body after status). + break; + } + } + } + + return buffer.ToArray(); + } + + private static bool EndsWithArticleTerminator(byte[] wire) => + wire.Length >= 5 + && wire[^5] == (byte)'\r' + && wire[^4] == (byte)'\n' + && wire[^3] == (byte)'.' + && wire[^2] == (byte)'\r' + && wire[^1] == (byte)'\n'; + + private static async Task ReadLineAsync(Stream stream) + { + var buffer = new List(128); + var one = new byte[1]; + while (true) + { + var n = await stream.ReadAsync(one); + if (n == 0) + { + break; + } + + buffer.Add(one[0]); + if (buffer.Count >= 2 && buffer[^2] == (byte)'\r' && buffer[^1] == (byte)'\n') + { + return Encoding.ASCII.GetString(buffer.GetRange(0, buffer.Count - 2).ToArray()); + } + } + + return Encoding.ASCII.GetString(buffer.ToArray()); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/UpstreamProviderSnapshot.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/UpstreamProviderSnapshot.cs new file mode 100644 index 00000000..fa3189a8 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/UpstreamProviderSnapshot.cs @@ -0,0 +1,79 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// + /// Operator-captured sanitized view of published BackFiller upstream endpoints for one lab ServerId. + /// Passwords must never be included. Used by before smoke. + /// + internal sealed record UpstreamProviderSnapshot + { + /// UTC capture time of the account-pool observation. + public required DateTimeOffset CapturedUtc { get; init; } + + /// Lab BackFiller ServerId the rows belong to. + public required int ServerId { get; init; } + + /// + /// How the snapshot was obtained (e.g. mysql-select, log-parse). + /// Empty or unknown values fail the gate. + /// + public required string Source { get; init; } + + /// + /// Optional note that session pools were observed refreshed after MySQL change + /// (e.g. log events 5601/5603). Required non-empty for smoke readiness. + /// + public required string PoolRefreshEvidence { get; init; } + + /// Published provider endpoints (no credentials). + public required IReadOnlyList Providers { get; init; } + + /// Deserializes a JSON snapshot file. + internal static async Task ReadAsync(string path, CancellationToken cancellationToken) + { + await using var stream = File.OpenRead(path); + var snapshot = await JsonSerializer + .DeserializeAsync(stream, UpstreamProviderSnapshotJsonContext.Default.UpstreamProviderSnapshot, cancellationToken) + .ConfigureAwait(false); + return snapshot + ?? throw new InvalidOperationException("Upstream provider snapshot JSON deserialized to null: " + path); + } + + /// Writes indented JSON (sanitized; caller must not include secrets). + internal static async Task WriteAsync( + string path, + UpstreamProviderSnapshot snapshot, + CancellationToken cancellationToken) + { + await using var stream = File.Create(path); + await JsonSerializer + .SerializeAsync( + stream, + snapshot, + UpstreamProviderSnapshotJsonContext.Default.UpstreamProviderSnapshot, + cancellationToken) + .ConfigureAwait(false); + } + } + + /// One published backbone endpoint without credentials. + internal sealed record UpstreamProviderEndpoint + { + public required string Backbone { get; init; } + + public required string Host { get; init; } + + public required int Port { get; init; } + + public required bool UseTls { get; init; } + } + + [JsonSerializable(typeof(UpstreamProviderSnapshot))] + [JsonSerializable(typeof(UpstreamProviderEndpoint))] + [JsonSourceGenerationOptions(WriteIndented = true)] + internal partial class UpstreamProviderSnapshotJsonContext : JsonSerializerContext + { + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs index 28812f2c..01f55786 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer; namespace VectorNNTPD.BackFillerService.Tests.Stress { @@ -590,5 +591,197 @@ private static int[] ParseSweepConcurrencyLevels() return [25, 50, 75, 100, 150, 200]; } + + /// + /// Lab-only: builds provenanced workload Message-IDs from .artifacts/Articles for the local peer. + /// Requires VECTORNNTPD_BACKFILLER_E2E_STRESS=1 and + /// VECTORNNTPD_BACKFILLER_E2E_STRESS_REQUIRE_LOCAL_UPSTREAM=1. + /// Does not start BackFiller, the peer, or contact real upstreams. + /// + [Fact] + public async Task Discover_workload_from_articles_for_local_peer_when_enabled() + { + if (!StressHarnessOptions.IsEnabled() || !StressHarnessOptions.RequiresLocalUpstream()) + { + return; + } + + _ = LocalUpstreamIsolationGate.EnsureDeclaredEndpointOrThrow( + LocalUpstreamIsolationGate.FromEnvironment()); + var options = StressHarnessOptions.FromEnvironment(); + var articlesRoot = options.NewsLogDirectory; + if (!Directory.Exists(articlesRoot)) + { + Assert.Fail( + "Local-peer discovery requires VECTORNNTPD_BACKFILLER_E2E_STRESS_NEWSLOGS to point at " + + $".artifacts/Articles (got missing path: {articlesRoot})."); + } + + var index = await CorpusArticleIndex.BuildAsync(articlesRoot); + Assert.True(index.Count > 0); + var (count, fingerprint) = await LocalArticlePeerWorkload.WriteAsync( + index, + options.WorkloadDirectory, + options.Seed); + await StressWorkloadGuard.RequireProvenancedWorkloadAsync(options, CancellationToken.None); + Console.WriteLine( + $"local-peer workload articles={count} fingerprint={fingerprint} dir={options.WorkloadDirectory}"); + } + + /// + /// Lab-only short smoke against a pre-started local ARTICLE peer and lab-pointed BackFiller. + /// Phase LocalPeerSmoke. Fail-closed unless local-upstream env is set to loopback. + /// Does not start/stop BackFiller, RabbitMQ, or the peer. + /// + [Fact] + public async Task LocalPeerSmoke_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var phase = StressHarnessOptions.GetPhaseOrDefault("LocalPeerSmoke"); + if (!string.Equals(phase, "LocalPeerSmoke", StringComparison.OrdinalIgnoreCase) + && !string.Equals(phase, "ALL", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + // Fail closed on declared env + sanitized published-provider snapshot + peer reachability. + // Does not inject into a separately running BackFiller transport; see ISOLATION_AND_ACCOUNTING_FIXES.md. + var (host, port) = LocalUpstreamIsolationGate.EnsureReadyOrThrow( + LocalUpstreamIsolationGate.FromEnvironment()); + Console.WriteLine( + $"LocalPeerSmoke isolation gate passed peer={host}:{port}; snapshot providers verified local-only."); + + var options = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(options); + var harness = new BackFillerE2EStressHarness(options); + var backbone = StressHarnessOptions.GetBackboneOrDefault("Giganews"); + var file = Path.Join(options.WorkloadDirectory, "warmup-message-ids.txt"); + var concurrency = StressHarnessOptions.GetConcurrencyOrDefault(1); + var durationSeconds = StressHarnessOptions.GetDurationSecondsOrDefault(20); + var summary = await harness.RunAsync( + "local-peer-smoke-c" + concurrency, + backbone, + file, + concurrency, + maxInFlight: Math.Max(2, concurrency * 2), + duration: TimeSpan.FromSeconds(durationSeconds), + maxRequests: 20, + CancellationToken.None); + Console.WriteLine(summary.FormatTableRow()); + + long? peerRequests = null; + long? peerSuccess = null; + long? peerMissing = null; + long? peerProtocol = null; + var peerMetricsPath = Environment.GetEnvironmentVariable( + "VECTORNNTPD_LAB_LOCAL_ARTICLE_PEER_METRICS_JSON"); + if (!string.IsNullOrWhiteSpace(peerMetricsPath) && File.Exists(peerMetricsPath)) + { + using var doc = JsonDocument.Parse(await File.ReadAllTextAsync(peerMetricsPath)); + var root = doc.RootElement; + if (root.TryGetProperty("ArticleRequests", out var ar)) + { + peerRequests = ar.GetInt64(); + } + + if (root.TryGetProperty("ArticleSuccess", out var asuc)) + { + peerSuccess = asuc.GetInt64(); + } + + if (root.TryGetProperty("ArticleMissing", out var am)) + { + peerMissing = am.GetInt64(); + } + + if (root.TryGetProperty("ProtocolErrors", out var pe)) + { + peerProtocol = pe.GetInt64(); + } + } + + LocalPeerSmokeAccounting.AssertSuccessfulSmokeOrThrow( + new LocalPeerSmokeAccounting.Evidence( + summary.RequestsStarted, + summary.WorkSuccess, + summary.WorkNotFound, + summary.WorkOtherFailure, + summary.ArticlesDelivered, + summary.OutstandingAtEnd, + summary.Timeouts, + summary.UnresolvedAtDispose, + summary.LateRepliesAfterTimeout, + peerRequests, + peerSuccess, + peerMissing, + peerProtocol)); + } + + /// + /// Lab-only controlled concurrency ramp against the local ARTICLE peer. + /// Phase LocalPeerRamp. Same fail-closed isolation as . + /// + [Fact] + public async Task LocalPeerRamp_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var phase = StressHarnessOptions.GetPhaseOrDefault("LocalPeerRamp"); + if (!string.Equals(phase, "LocalPeerRamp", StringComparison.OrdinalIgnoreCase) + && !string.Equals(phase, "ALL", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var (host, port) = LocalUpstreamIsolationGate.EnsureReadyOrThrow( + LocalUpstreamIsolationGate.FromEnvironment()); + Console.WriteLine($"LocalPeerRamp isolation gate passed peer={host}:{port}"); + + var options = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(options); + var harness = new BackFillerE2EStressHarness(options); + var backbone = StressHarnessOptions.GetBackboneOrDefault("Giganews"); + var file = Path.Join(options.WorkloadDirectory, StressCorpusPaths.MainIdsAuditFileName); + var durationSeconds = StressHarnessOptions.GetDurationSecondsOrDefault(30); + var levels = ParseConcurrencyLevels(); + // Lab peer MaxSessions is typically 128; do not exceed that without an explicit peer raise. + // Stages above production provider MaxSessions still measure queueing; stop on timeouts/outstanding. + const int maxLabPeerConcurrency = 128; + foreach (var concurrency in levels.Where(static c => c <= maxLabPeerConcurrency).DefaultIfEmpty(1)) + { + // Scale in-flight with the stage. Do not honor VECTORNNTPD_BACKFILLER_E2E_STRESS_MAX_INFLIGHT: + // a leftover value of 2 from LocalPeerSmoke silently caps PeakOutstanding and invents a + // false throughput plateau while ConfiguredConcurrency appears to ramp. + var maxInFlight = concurrency * 2; + Console.WriteLine( + $"LocalPeerRamp stage c={concurrency} maxInFlight={maxInFlight} durationSec={durationSeconds}"); + var summary = await harness.RunAsync( + $"local-peer-ramp-c{concurrency}", + backbone, + file, + concurrency, + maxInFlight, + TimeSpan.FromSeconds(durationSeconds), + maxRequests: null, + CancellationToken.None); + Console.WriteLine(summary.FormatTableRow()); + Console.WriteLine( + $"LocalPeerRamp stage done c={concurrency} peakOut={summary.PeakOutstanding} " + + $"art/s={summary.ArticlesPerSecond:F1} Gbps={summary.PayloadGbps:F3}"); + if (summary.Timeouts > summary.RequestsStarted * 0.25 + || summary.OutstandingAtEnd > 0) + { + Console.WriteLine($"LocalPeerRamp stopping after c={concurrency}"); + break; + } + } + } } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs index 82f62a96..fca35075 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs @@ -28,6 +28,29 @@ public BackFillerE2EStressHarness(StressHarnessOptions options) _options = options ?? throw new ArgumentNullException(nameof(options)); } + /// + /// Ensures the configured corpus directory is usable for the current mode. + /// Local-upstream lab mode accepts an Articles tree (no *.log required); + /// ordinary stress still requires news-log *.log files. + /// + private void RequireCorpusReadyForConfiguredMode() + { + if (StressHarnessOptions.RequiresLocalUpstream()) + { + if (!Directory.Exists(_options.NewsLogDirectory)) + { + throw new InvalidOperationException( + "Local-peer corpus directory is missing or unreadable: " + + _options.NewsLogDirectory + + ". Set VECTORNNTPD_BACKFILLER_E2E_STRESS_NEWSLOGS to .artifacts/Articles."); + } + + return; + } + + NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(_options.NewsLogDirectory); + } + /// Discovers Message-IDs from the configured corpus, writes workload files and manifest. public async Task DiscoverAndWriteWorkloadAsync( CancellationToken cancellationToken) @@ -121,7 +144,8 @@ public async Task RunAsync( ArgumentOutOfRangeException.ThrowIfLessThan(maxInFlight, 1); // Static publish gate: corpus + provenance + SHA before any AMQP connect/publish. - NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(_options.NewsLogDirectory); + // Local-peer smoke uses .artifacts/Articles (no *.log); news-log stress still requires *.log. + RequireCorpusReadyForConfiguredMode(); await StressWorkloadGuard.RequireProvenancedWorkloadAsync(_options, cancellationToken) .ConfigureAwait(false); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs index bbd559a4..4590d3c6 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs @@ -102,6 +102,13 @@ public sealed class StressHarnessOptions /// Minimum completed replies before abort thresholds are evaluated. public int AbortAfterRequests { get; init; } = 200; + /// + /// When 1, live stress must target a loopback local ARTICLE peer + /// (VECTORNNTPD_BACKFILLER_E2E_STRESS_LOCAL_PEER_HOST/_PORT). + /// + public const string RequireLocalUpstreamEnvironmentVariable = + "VECTORNNTPD_BACKFILLER_E2E_STRESS_REQUIRE_LOCAL_UPSTREAM"; + /// Returns when the opt-in environment flag is set to 1. public static bool IsEnabled() => string.Equals( @@ -109,6 +116,16 @@ public static bool IsEnabled() => "1", StringComparison.Ordinal); + /// + /// Returns when local-upstream isolation is requested + /// (=1). + /// + public static bool RequiresLocalUpstream() => + string.Equals( + Environment.GetEnvironmentVariable(RequireLocalUpstreamEnvironmentVariable), + "1", + StringComparison.Ordinal); + /// /// Builds options from environment variables. Throws when RabbitMQ credentials are missing. /// Does not log secret values. From b97fe10e5db9dbacb7ed391b63f25f70eec64b87 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 28 Oct 2026 13:28:57 +0200 Subject: [PATCH 3/8] Fix RabbitMQ config test host and remediate PR 255 CodeQL findings --- .../RabbitMq/RabbitMqClientConnection.cs | 28 ++-- ...ApplicationJsonConfigurationSourceTests.cs | 8 +- .../Diagnostics/StageLatencyHistogramTests.cs | 132 ++++++++---------- .../Hosting/BackFillerHostCompositionTests.cs | 6 +- .../LocalArticlePeer/CorpusArticleIndex.cs | 58 +++++++- .../CorpusArticleIndexTests.cs | 49 +++++++ .../LocalArticlePeer/LocalArticleNntpPeer.cs | 26 +++- .../LocalArticleNntpPeerTests.cs | 25 ++-- .../LocalArticlePeerCorpusTests.cs | 8 +- .../LocalUpstreamIsolationGateTests.cs | 4 +- .../LocalArticlePeer/PreloadedArticleStore.cs | 1 + .../PreloadedArticleStoreTests.cs | 4 +- .../Retention/ArticleRetentionMetricsTests.cs | 9 +- .../TestDoubles/FakeBackFillerRabbitMq.cs | 1 + 14 files changed, 241 insertions(+), 118 deletions(-) create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndexTests.cs diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs index 6454651c..32703871 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs @@ -148,15 +148,25 @@ public async Task CreatePublishChannelAsync( // Explicit limiter: the two-arg CreateChannelOptions leaves OutstandingPublisherConfirmationsRateLimiter // null (unlimited). Align with RabbitMqPublisherConfirmLimits / Article Work response publisher gate. - var options = new CreateChannelOptions( - publisherConfirmationsEnabled: true, - publisherConfirmationTrackingEnabled: true, - outstandingPublisherConfirmationsRateLimiter: new ThrottlingRateLimiter( - RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations)); - var channel = await _connection - .CreateChannelAsync(options: options, cancellationToken: cancellationToken) - .ConfigureAwait(false); - return new RabbitMqClientPublishChannel(channel, generation); + // On success the channel owns and disposes the limiter (RabbitMQ.Client 7.2.2 Channel.Dispose/DisposeAsync). + var limiter = new ThrottlingRateLimiter( + RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations); + try + { + var options = new CreateChannelOptions( + publisherConfirmationsEnabled: true, + publisherConfirmationTrackingEnabled: true, + outstandingPublisherConfirmationsRateLimiter: limiter); + var channel = await _connection + .CreateChannelAsync(options: options, cancellationToken: cancellationToken) + .ConfigureAwait(false); + return new RabbitMqClientPublishChannel(channel, generation); + } + catch + { + await limiter.DisposeAsync().ConfigureAwait(false); + throw; + } } /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/ApplicationJsonConfigurationSourceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/ApplicationJsonConfigurationSourceTests.cs index 6703af7e..3459a2b5 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/ApplicationJsonConfigurationSourceTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/ApplicationJsonConfigurationSourceTests.cs @@ -240,7 +240,7 @@ public void Shipped_rabbitmq_file_is_copied_and_loads() var configuration = builder.Build(); Assert.Equal("5672", configuration["RabbitMQ:Port"]); - Assert.Equal("rabbitmq1a.usenet.ninja", configuration["RabbitMQ:Hosts:0"]); + Assert.Equal("198.18.0.3", configuration["RabbitMQ:Hosts:0"]); Assert.Equal(bool.FalseString, configuration["RabbitMQ:EnableSsl"]); Assert.Equal("/", configuration["RabbitMQ:VirtualHost"]); @@ -249,9 +249,7 @@ public void Shipped_rabbitmq_file_is_copied_and_loads() var validation = new RabbitMqOptionsValidator().Validate(null, rabbit); Assert.True(validation.Succeeded, string.Join("; ", validation.Failures ?? [])); Assert.NotNull(rabbit.Hosts); - Assert.Equal( - ["rabbitmq1a.usenet.ninja", "rabbitmq1b.usenet.ninja", "rabbitmq1c.usenet.ninja"], - rabbit.Hosts); + Assert.Equal(["198.18.0.3"], rabbit.Hosts); Assert.Equal(5672, rabbit.Port); Assert.Equal(false, rabbit.EnableSsl); Assert.Equal("/", rabbit.VirtualHost); @@ -260,7 +258,7 @@ public void Shipped_rabbitmq_file_is_copied_and_loads() Assert.Equal(5, rabbit.ArticleWorkDeliveryLimit); Assert.Equal(10000, rabbit.ArticleWorkQueueMaxLength); Assert.Equal(60000, rabbit.ArticleWorkQueueMessageTtlMilliseconds); - Assert.Equal("http://rabbitmq1a.usenet.ninja:15672", rabbit.Management.BaseUrl); + Assert.Equal("http://198.18.0.3:15672", rabbit.Management.BaseUrl); Assert.Equal(5, rabbit.Management.RequestTimeoutSeconds); } finally diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs index 09651b49..4f44bd5a 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Diagnostics/StageLatencyHistogramTests.cs @@ -2,6 +2,11 @@ namespace VectorNNTPD.BackFillerService.Tests.Diagnostics { + /// + /// Serializes tests that install the ambient so parallel + /// suites cannot record into another test's installed instance. + /// + [Collection(nameof(ArticleWorkStageProfiler))] public sealed class StageLatencyHistogramTests { [Fact] @@ -76,100 +81,85 @@ public void Bucket_index_is_bounded() public void Disabled_profiler_records_nothing() { var profiler = new ArticleWorkStageProfiler(enabled: false); - ArticleWorkStageProfiler.Install(profiler); - try - { - Assert.False(ArticleWorkStageProfiler.IsEnabled); - profiler.BeginActive(); - profiler.Record(ArticleWorkStageId.HandlerTotal, 1); - profiler.EndActive(); - Assert.Equal(0, profiler[ArticleWorkStageId.HandlerTotal].Count); - Assert.Equal(0, profiler.Completed); - } - finally - { - ArticleWorkStageProfiler.Install(null); - } + WithInstalled(profiler, static () => Assert.False(ArticleWorkStageProfiler.IsEnabled)); + profiler.BeginActive(); + profiler.Record(ArticleWorkStageId.HandlerTotal, 1); + profiler.EndActive(); + Assert.Equal(0, profiler[ArticleWorkStageId.HandlerTotal].Count); + Assert.Equal(0, profiler.Completed); } [Fact] public void Enabled_profiler_tracks_active_and_stages() { var profiler = new ArticleWorkStageProfiler(enabled: true); - ArticleWorkStageProfiler.Install(profiler); - try - { - Assert.True(ArticleWorkStageProfiler.IsEnabled); - profiler.BeginActive(); - Assert.Equal(1, profiler.Active); - var started = System.Diagnostics.Stopwatch.GetTimestamp(); - profiler.Record(ArticleWorkStageId.Retain, started); - profiler.EndActive(failed: true); - Assert.Equal(0, profiler.Active); - Assert.Equal(1, profiler.PeakActive); - Assert.Equal(1, profiler.Completed); - Assert.Equal(1, profiler.Failed); - Assert.Equal(1, profiler[ArticleWorkStageId.Retain].Count); - var snap = profiler.CaptureSnapshot(); - Assert.Contains(snap.Stages, static s => s.Stage == nameof(ArticleWorkStageId.Retain) && s.Count == 1); - } - finally - { - ArticleWorkStageProfiler.Install(null); - } + WithInstalled(profiler, static () => Assert.True(ArticleWorkStageProfiler.IsEnabled)); + profiler.BeginActive(); + Assert.Equal(1, profiler.Active); + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + profiler.Record(ArticleWorkStageId.Retain, started); + profiler.EndActive(failed: true); + Assert.Equal(0, profiler.Active); + Assert.Equal(1, profiler.PeakActive); + Assert.Equal(1, profiler.Completed); + Assert.Equal(1, profiler.Failed); + Assert.Equal(1, profiler[ArticleWorkStageId.Retain].Count); + var snap = profiler.CaptureSnapshot(); + Assert.Contains(snap.Stages, static s => s.Stage == nameof(ArticleWorkStageId.Retain) && s.Count == 1); } [Fact] public void Publisher_sub_stages_aggregate_and_appear_in_snapshot() { var profiler = new ArticleWorkStageProfiler(enabled: true); - ArticleWorkStageProfiler.Install(profiler); - try - { - profiler[ArticleWorkStageId.ReplySerialize].RecordMicroseconds(100); - profiler[ArticleWorkStageId.ReplyPublishGateWait].RecordMicroseconds(20_000); - profiler[ArticleWorkStageId.ReplyPublishConfirm].RecordMicroseconds(800); - profiler[ArticleWorkStageId.ReplyPublish].RecordMicroseconds(21_000); + profiler[ArticleWorkStageId.ReplySerialize].RecordMicroseconds(100); + profiler[ArticleWorkStageId.ReplyPublishGateWait].RecordMicroseconds(20_000); + profiler[ArticleWorkStageId.ReplyPublishConfirm].RecordMicroseconds(800); + profiler[ArticleWorkStageId.ReplyPublish].RecordMicroseconds(21_000); - Assert.Equal(1, profiler[ArticleWorkStageId.ReplyPublishGateWait].Count); - Assert.Equal(20.0, profiler[ArticleWorkStageId.ReplyPublishGateWait].MeanMilliseconds, precision: 3); - Assert.Equal(0.8, profiler[ArticleWorkStageId.ReplyPublishConfirm].MeanMilliseconds, precision: 3); + Assert.Equal(1, profiler[ArticleWorkStageId.ReplyPublishGateWait].Count); + Assert.Equal(20.0, profiler[ArticleWorkStageId.ReplyPublishGateWait].MeanMilliseconds, precision: 3); + Assert.Equal(0.8, profiler[ArticleWorkStageId.ReplyPublishConfirm].MeanMilliseconds, precision: 3); - var snap = profiler.CaptureSnapshot(); - Assert.Contains( - snap.Stages, - static s => s.Stage == nameof(ArticleWorkStageId.ReplyPublishGateWait) - && s.Count == 1 - && s.Nesting.Contains("_publishGate", StringComparison.Ordinal) - && s.Nesting.Contains("MaxConcurrentConfirmedPublications", StringComparison.Ordinal)); - Assert.Contains( - snap.Stages, - static s => s.Stage == nameof(ArticleWorkStageId.ReplyPublishConfirm) - && s.Nesting.Contains("BasicPublishAsync", StringComparison.Ordinal)); - } - finally - { - ArticleWorkStageProfiler.Install(null); - } + var snap = profiler.CaptureSnapshot(); + Assert.Contains( + snap.Stages, + static s => s.Stage == nameof(ArticleWorkStageId.ReplyPublishGateWait) + && s.Count == 1 + && s.Nesting.Contains("_publishGate", StringComparison.Ordinal) + && s.Nesting.Contains("MaxConcurrentConfirmedPublications", StringComparison.Ordinal)); + Assert.Contains( + snap.Stages, + static s => s.Stage == nameof(ArticleWorkStageId.ReplyPublishConfirm) + && s.Nesting.Contains("BasicPublishAsync", StringComparison.Ordinal)); } [Fact] public void Concurrent_publisher_stage_records_do_not_lose_counts() { var profiler = new ArticleWorkStageProfiler(enabled: true); + Parallel.For( + 0, + 500, + _ => + { + profiler.Record(ArticleWorkStageId.ReplyPublishGateWait, System.Diagnostics.Stopwatch.GetTimestamp()); + profiler.Record(ArticleWorkStageId.ReplyPublishConfirm, System.Diagnostics.Stopwatch.GetTimestamp()); + }); + Assert.Equal(500, profiler[ArticleWorkStageId.ReplyPublishGateWait].Count); + Assert.Equal(500, profiler[ArticleWorkStageId.ReplyPublishConfirm].Count); + } + + /// + /// Installs only for the duration of so ambient + /// hot-path hooks from parallel tests cannot mutate the instance under assertion. + /// + private static void WithInstalled(ArticleWorkStageProfiler profiler, Action action) + { ArticleWorkStageProfiler.Install(profiler); try { - Parallel.For( - 0, - 500, - _ => - { - profiler.Record(ArticleWorkStageId.ReplyPublishGateWait, System.Diagnostics.Stopwatch.GetTimestamp()); - profiler.Record(ArticleWorkStageId.ReplyPublishConfirm, System.Diagnostics.Stopwatch.GetTimestamp()); - }); - Assert.Equal(500, profiler[ArticleWorkStageId.ReplyPublishGateWait].Count); - Assert.Equal(500, profiler[ArticleWorkStageId.ReplyPublishConfirm].Count); + action(); } finally { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs index db6cf0ef..bfcfdc41 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs @@ -165,14 +165,16 @@ public void AddBackFillerHosting_production_defaults_isolate_lab_features_and_pi publisherConfirmationTrackingEnabled: true); Assert.Null(unboundedDefaults.OutstandingPublisherConfirmationsRateLimiter); + using var limiter = new ThrottlingRateLimiter( + RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations); var productionOptions = new CreateChannelOptions( publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true, - outstandingPublisherConfirmationsRateLimiter: new ThrottlingRateLimiter( - RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations)); + outstandingPublisherConfirmationsRateLimiter: limiter); Assert.True(productionOptions.PublisherConfirmationsEnabled); Assert.True(productionOptions.PublisherConfirmationTrackingEnabled); Assert.NotNull(productionOptions.OutstandingPublisherConfirmationsRateLimiter); + Assert.Same(limiter, productionOptions.OutstandingPublisherConfirmationsRateLimiter); } /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndex.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndex.cs index 7f2d2b53..929543f4 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndex.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndex.cs @@ -39,10 +39,66 @@ internal bool TryResolve(string messageId, out string absolutePath) return false; } - absolutePath = Path.Combine(CorpusRoot, relative.Replace('/', Path.DirectorySeparatorChar)); + if (!TryResolveUnderCorpusRoot(relative, out absolutePath)) + { + return false; + } + return File.Exists(absolutePath); } + /// + /// Resolves under , rejecting rooted paths, + /// empty segments, and any result that escapes the corpus root. + /// + /// Corpus-relative path using / or platform separators. + /// Full path when resolution succeeds; otherwise empty. + /// when the path stays under the corpus root. + internal bool TryResolveUnderCorpusRoot(string relative, out string absolutePath) + { + absolutePath = ""; + if (string.IsNullOrWhiteSpace(relative)) + { + return false; + } + + var normalized = relative.Replace('\\', '/'); + if (Path.IsPathRooted(normalized) + || Path.IsPathRooted(relative) + || normalized.StartsWith('/') + || normalized.Contains("://", StringComparison.Ordinal)) + { + return false; + } + + var segments = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length == 0) + { + return false; + } + + foreach (var segment in segments) + { + if (segment is "." or "..") + { + return false; + } + } + + var candidate = Path.GetFullPath(Path.Join(CorpusRoot, Path.Join(segments))); + var rootWithSep = CorpusRoot.EndsWith(Path.DirectorySeparatorChar) + ? CorpusRoot + : CorpusRoot + Path.DirectorySeparatorChar; + if (!candidate.StartsWith(rootWithSep, StringComparison.OrdinalIgnoreCase) + && !string.Equals(candidate, CorpusRoot, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + absolutePath = candidate; + return true; + } + /// /// Builds an index by reading at most from each file under /// . Skips files without a Message-ID or header/body separator in the probe. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndexTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndexTests.cs new file mode 100644 index 00000000..1d0d239f --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/CorpusArticleIndexTests.cs @@ -0,0 +1,49 @@ +namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer +{ + /// Path safety for resolution under the corpus root. + public sealed class CorpusArticleIndexTests + { + [Fact] + public async Task TryResolve_rejects_rooted_and_traversal_relative_paths() + { + var dir = Path.Join(Path.GetTempPath(), "vnntp-corpus-path-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + const string messageId = ""; + await File.WriteAllTextAsync( + Path.Join(dir, "article1"), + $"Message-ID: {messageId}\r\nFrom: a@b\r\n\r\nbody\r\n"); + + var index = await CorpusArticleIndex.BuildAsync(dir); + Assert.True(index.TryResolve(messageId, out var safePath)); + Assert.True(File.Exists(safePath)); + Assert.StartsWith(index.CorpusRoot, safePath, StringComparison.OrdinalIgnoreCase); + + Assert.False(index.TryResolveUnderCorpusRoot("../outside", out _)); + Assert.False(index.TryResolveUnderCorpusRoot("..\\outside", out _)); + Assert.False(index.TryResolveUnderCorpusRoot("/etc/passwd", out _)); + Assert.False(index.TryResolveUnderCorpusRoot("", out _)); + Assert.False(index.TryResolveUnderCorpusRoot(".", out _)); + if (OperatingSystem.IsWindows()) + { + Assert.False(index.TryResolveUnderCorpusRoot("C:\\Windows\\system.ini", out _)); + } + + Assert.True(index.TryResolveUnderCorpusRoot("article1", out var under)); + Assert.Equal(Path.GetFullPath(Path.Join(index.CorpusRoot, "article1")), under); + } + finally + { + try + { + Directory.Delete(dir, recursive: true); + } + catch (IOException ex) + { + _ = ex.GetType(); + } + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs index 8be58a0b..bcd39bfb 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs @@ -96,6 +96,8 @@ public async ValueTask DisposeAsync() } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { + // Best-effort shutdown: listener may already be stopped. + _ = ex.GetType(); } try @@ -104,6 +106,8 @@ public async ValueTask DisposeAsync() } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { + // Accept loop may already be faulted or cancelled during dispose. + _ = ex.GetType(); } _cts.Dispose(); @@ -114,6 +118,7 @@ private async Task AcceptLoopAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { + // codeql[cs/missed-using-statement] Ownership transfers to the session task, which disposes in its finally. TcpClient client; try { @@ -271,7 +276,8 @@ await ServeArticleAsync(stream, line["ARTICLE ".Length..].Trim(), cancellationTo private async Task ServeArticleAsync(NetworkStream stream, string messageId, CancellationToken cancellationToken) { Metrics.ArticleBegin(); - var settled = false; + // True after a path has already updated article metrics so a later write failure does not double-count. + var metricsRecorded = false; try { if (_options.ArticleDelay > TimeSpan.Zero) @@ -285,7 +291,7 @@ private async Task ServeArticleAsync(NetworkStream stream, string messageId, Can { // Record before write so clients observing the status line cannot race the counter. Metrics.RecordArticleMissing(); - settled = true; + metricsRecorded = true; await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken) .ConfigureAwait(false); return; @@ -295,7 +301,6 @@ await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken await stream.WriteAsync(preloaded.WireResponse, cancellationToken).ConfigureAwait(false); await stream.FlushAsync(cancellationToken).ConfigureAwait(false); Metrics.RecordArticleSuccess(preloaded.LogicalByteCount, preloaded.WireResponse.Length); - settled = true; return; } @@ -303,7 +308,7 @@ await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken { // Record before write so clients observing the status line cannot race the counter. Metrics.RecordArticleMissing(); - settled = true; + metricsRecorded = true; await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken) .ConfigureAwait(false); return; @@ -311,6 +316,7 @@ await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken byte[] logical; byte[] framed; + var readFailed = false; try { (logical, framed) = await NntpWireFramer @@ -320,7 +326,14 @@ await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { Metrics.RecordArticleError(); - settled = true; + metricsRecorded = true; + readFailed = true; + logical = []; + framed = []; + } + + if (readFailed) + { await WriteAsciiAsync(stream, "403 Article unavailable\r\n", cancellationToken) .ConfigureAwait(false); return; @@ -332,11 +345,10 @@ await WriteAsciiAsync(stream, "403 Article unavailable\r\n", cancellationToken) await stream.WriteAsync(framed, cancellationToken).ConfigureAwait(false); await stream.FlushAsync(cancellationToken).ConfigureAwait(false); Metrics.RecordArticleSuccess(logical.Length, status.Length + framed.Length); - settled = true; } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { - if (!settled) + if (!metricsRecorded) { Metrics.RecordArticleError(); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs index f1fefa2f..fffe61f6 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs @@ -97,9 +97,7 @@ public async Task Peer_auth_flow_accepts_configured_credentials() Assert.StartsWith("281 ", await ReadLineAsync(stream)); await WriteAsync(stream, EncodingAscii($"ARTICLE {messageId}\r\n")); Assert.StartsWith("220 ", await ReadLineAsync(stream)); - while (await ReadLineAsync(stream) != ".") - { - } + await DrainMultilineUntilTerminatorAsync(stream); } [Fact] @@ -159,15 +157,11 @@ public async Task Peer_accepts_unauthenticated_flow_matching_lab_rewrite() Assert.StartsWith("200 ", await ReadLineAsync(stream)); await WriteAsync(stream, "CAPABILITIES\r\n"u8.ToArray()); Assert.StartsWith("101 ", await ReadLineAsync(stream)); - while (await ReadLineAsync(stream) != ".") - { - } + await DrainMultilineUntilTerminatorAsync(stream); await WriteAsync(stream, EncodingAscii($"ARTICLE {messageId}\r\n")); Assert.StartsWith("220 ", await ReadLineAsync(stream)); - while (await ReadLineAsync(stream) != ".") - { - } + await DrainMultilineUntilTerminatorAsync(stream); Assert.Equal(1, peer.Metrics.ArticleSuccess); Assert.Equal(0, peer.Metrics.ProtocolErrors); @@ -253,10 +247,10 @@ public async Task LabBinaryIdentity_capture_hashes_loaded_assemblies() private static string CreateFixtureCorpus(out string messageId) { messageId = $""; - var dir = Path.Combine(Path.GetTempPath(), "vnntp-local-peer-" + Guid.NewGuid().ToString("N")); + var dir = Path.Join(Path.GetTempPath(), "vnntp-local-peer-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); var body = $"Message-ID: {messageId}\r\nFrom: lab@test\r\n\r\nhello\r\n"; - File.WriteAllText(Path.Combine(dir, "article1"), body); + File.WriteAllText(Path.Join(dir, "article1"), body); return dir; } @@ -265,6 +259,15 @@ private static string CreateFixtureCorpus(out string messageId) private static async Task WriteAsync(Stream stream, byte[] bytes) => await stream.WriteAsync(bytes); + /// Consumes NNTP multiline response lines until the terminator .. + private static async Task DrainMultilineUntilTerminatorAsync(Stream stream) + { + while (await ReadLineAsync(stream) != ".") + { + // Intentionally discard article / capability lines. + } + } + private static async Task ReadLineAsync(Stream stream) { var buffer = new List(128); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerCorpusTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerCorpusTests.cs index 0855d9aa..c0714b4e 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerCorpusTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticlePeerCorpusTests.cs @@ -7,7 +7,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer public sealed class LocalArticlePeerCorpusTests { private static readonly string CorpusRoot = Path.GetFullPath( - Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "..", ".artifacts", "Articles")); + Path.Join(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "..", ".artifacts", "Articles")); [Fact] public async Task Real_corpus_indexes_and_serves_one_article_when_present() @@ -45,7 +45,7 @@ await session.ConnectAsync( Assert.True(result.Article!.Memory.Length > 0); var outDir = Path.GetFullPath( - Path.Combine( + Path.Join( AppContext.BaseDirectory, "..", "..", @@ -59,10 +59,10 @@ await session.ConnectAsync( "local-article-peer-benchmark-20261027")); Directory.CreateDirectory(outDir); await peer.Index.WriteManifestAsync( - Path.Combine(outDir, "corpus-fingerprint.json"), + Path.Join(outDir, "corpus-fingerprint.json"), CancellationToken.None); await LabBinaryIdentity.WriteAsync( - Path.Combine(outDir, "assembly-identity-correctness.json"), + Path.Join(outDir, "assembly-identity-correctness.json"), CancellationToken.None); } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGateTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGateTests.cs index 417f5e42..b2fc27d0 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGateTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalUpstreamIsolationGateTests.cs @@ -139,11 +139,11 @@ public void Accepts_fully_verified_local_only_configuration() [Fact] public async Task Allowlist_transport_blocks_external_and_allows_local_peer() { - var dir = Path.Combine(Path.GetTempPath(), "vnntp-allowlist-" + Guid.NewGuid().ToString("N")); + var dir = Path.Join(Path.GetTempPath(), "vnntp-allowlist-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); var messageId = $""; await File.WriteAllTextAsync( - Path.Combine(dir, "a1"), + Path.Join(dir, "a1"), $"Message-ID: {messageId}\r\nFrom: a@b\r\n\r\nbody\r\n"); await using var peer = await LocalArticleNntpPeer.StartAsync( diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs index 185f09cf..9416214e 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs @@ -78,6 +78,7 @@ internal static async Task LoadAsync( logicalBytes += logical.Length; wireBytes += wire.Length; } + // codeql[cs/catch-of-all-exceptions] Fail-closed preload must capture every article load failure and continue counting remaining files. catch (Exception ex) { failures++; diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs index faaf6d82..612699d8 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs @@ -183,12 +183,12 @@ public async Task BackFiller_client_downloads_from_memory_peer() private static string CreateFixtureCorpus(out string messageId, bool includeDotLine) { messageId = $""; - var dir = Path.Combine(Path.GetTempPath(), "vnntp-preload-" + Guid.NewGuid().ToString("N")); + var dir = Path.Join(Path.GetTempPath(), "vnntp-preload-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); var body = includeDotLine ? $"Message-ID: {messageId}\r\nFrom: lab@test\r\n\r\nhello\r\n.dotline\r\n" : $"Message-ID: {messageId}\r\nFrom: lab@test\r\n\r\nhello\r\n"; - File.WriteAllText(Path.Combine(dir, "article1"), body); + File.WriteAllText(Path.Join(dir, "article1"), body); return dir; } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Retention/ArticleRetentionMetricsTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Retention/ArticleRetentionMetricsTests.cs index 7b6b63b6..82782c81 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Retention/ArticleRetentionMetricsTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Retention/ArticleRetentionMetricsTests.cs @@ -119,7 +119,7 @@ public async Task Concurrent_retain_and_observe_stay_consistent_with_authority() { await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 16 * 1024 * 1024); using var metrics = new ArticleRetentionMetrics(authority); - var barrier = new Barrier(4); + using var barrier = new Barrier(4); var tasks = new Task[3]; for (var i = 0; i < tasks.Length; i++) { @@ -164,10 +164,11 @@ public async Task Concurrent_retain_and_observe_stay_consistent_with_authority() public async Task Dispose_is_idempotent_and_replacement_observes_again() { await using var authority = ArticleRetentionAuthorityTests.Create(new ManualTimeProvider(Start), maxBytes: 1024 * 1024); - var metrics = new ArticleRetentionMetrics(authority); var prepared = RetentionTestArticles.RetainPrepared(authority, ""); - metrics.Dispose(); - metrics.Dispose(); + using (var metrics = new ArticleRetentionMetrics(authority)) + { + metrics.Dispose(); + } using var replacement = new ArticleRetentionMetrics(authority); var sample = Observe(); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs index bcbbe3c2..55b6f112 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs @@ -916,6 +916,7 @@ public async Task PublishConfirmedAsync(RabbitMqConfirmedPublication publication while ((observed = Volatile.Read(ref _maxConfirmInFlight)) < inFlight && Interlocked.CompareExchange(ref _maxConfirmInFlight, inFlight, observed) != observed) { + // CAS retry: work occurs in the loop condition. } Enqueued?.TrySetResult(); From 8d8c491bbed216c4e306d62060d6366692b7426d Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 28 Oct 2026 13:51:47 +0200 Subject: [PATCH 4/8] Fix ARTICLE success accounting and Linux preload test race --- .../ArticleWorkStageProfilingHostedService.cs | 17 +++++--- .../RabbitMq/RabbitMqClientConnection.cs | 14 +++--- .../LocalArticlePeer/LocalArticleNntpPeer.cs | 43 ++++++++++++------- .../LocalArticleNntpPeerMetrics.cs | 29 +++++++++++++ .../LocalArticleNntpPeerTests.cs | 4 ++ .../LocalArticlePeer/PreloadedArticleStore.cs | 13 +++++- .../PreloadedArticleStoreTests.cs | 6 +++ 7 files changed, 99 insertions(+), 27 deletions(-) diff --git a/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilingHostedService.cs b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilingHostedService.cs index 30ec628e..7ebc3f7a 100644 --- a/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilingHostedService.cs +++ b/src/VectorNNTPD.BackFillerService/Diagnostics/ArticleWorkStageProfilingHostedService.cs @@ -73,10 +73,9 @@ public async Task StopAsync(CancellationToken cancellationToken) { _logger.LogTrace("Stage profiling export loop wait cancelled during stop."); } - // codeql[cs/catch-of-all-exceptions] Stop must observe export-loop failure without failing host shutdown. - catch (Exception ex) + catch (Exception ex) when (IsExportIoFailure(ex)) { - _logger.LogDebug(ex, "Stage profiling export loop ended with error."); + _logger.LogDebug(ex, "Stage profiling export loop ended with I/O error."); } } @@ -86,8 +85,7 @@ public async Task StopAsync(CancellationToken cancellationToken) { await WriteSnapshotAsync(CancellationToken.None).ConfigureAwait(false); } - // codeql[cs/catch-of-all-exceptions] Final snapshot is best-effort; shutdown must continue. - catch (Exception ex) + catch (Exception ex) when (IsExportIoFailure(ex)) { _logger.LogWarning(ex, "Failed to write final stage profiling snapshot."); } @@ -129,5 +127,14 @@ private Task WriteSnapshotAsync(CancellationToken cancellationToken) var path = Path.Join(_options.ExportDirectory!, "article-work-stage-profile.json"); return File.WriteAllTextAsync(path, _profiler.ToJson(), cancellationToken); } + + /// + /// True for expected snapshot export I/O failures. Programming errors and unexpected faults are not matched. + /// / are covered by . + /// + private static bool IsExportIoFailure(Exception ex) => + ex is IOException + or UnauthorizedAccessException + or System.Security.SecurityException; } } diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs index 32703871..373f75a5 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs @@ -148,8 +148,9 @@ public async Task CreatePublishChannelAsync( // Explicit limiter: the two-arg CreateChannelOptions leaves OutstandingPublisherConfirmationsRateLimiter // null (unlimited). Align with RabbitMqPublisherConfirmLimits / Article Work response publisher gate. - // On success the channel owns and disposes the limiter (RabbitMQ.Client 7.2.2 Channel.Dispose/DisposeAsync). - var limiter = new ThrottlingRateLimiter( + // On success the channel owns the limiter (RabbitMQ.Client 7.2.2 Channel.Dispose/DisposeAsync). + // Null the local after transfer so finally disposes only the failure path. + ThrottlingRateLimiter? limiter = new ThrottlingRateLimiter( RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations); try { @@ -160,12 +161,15 @@ public async Task CreatePublishChannelAsync( var channel = await _connection .CreateChannelAsync(options: options, cancellationToken: cancellationToken) .ConfigureAwait(false); + limiter = null; return new RabbitMqClientPublishChannel(channel, generation); } - catch + finally { - await limiter.DisposeAsync().ConfigureAwait(false); - throw; + if (limiter is not null) + { + await limiter.DisposeAsync().ConfigureAwait(false); + } } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs index bcd39bfb..3322749d 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs @@ -97,7 +97,8 @@ public async ValueTask DisposeAsync() catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { // Best-effort shutdown: listener may already be stopped. - _ = ex.GetType(); + Metrics.RecordProtocolError(); + _ = ex; } try @@ -107,7 +108,8 @@ public async ValueTask DisposeAsync() catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { // Accept loop may already be faulted or cancelled during dispose. - _ = ex.GetType(); + Metrics.RecordProtocolError(); + _ = ex; } _cts.Dispose(); @@ -118,7 +120,6 @@ private async Task AcceptLoopAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { - // codeql[cs/missed-using-statement] Ownership transfers to the session task, which disposes in its finally. TcpClient client; try { @@ -146,23 +147,31 @@ private async Task AcceptLoopAsync(CancellationToken cancellationToken) } Metrics.ConnectionAccepted(); + // Session task owns disposal via using; gate releases on every completion path. _ = Task.Run( - async () => - { - try - { - await HandleClientAsync(client, cancellationToken).ConfigureAwait(false); - } - finally - { - client.Dispose(); - _sessionGate.Release(); - } - }, + () => RunAcceptedSessionAsync(client, cancellationToken), CancellationToken.None); } } + /// + /// Owns for the session lifetime and always releases . + /// + private async Task RunAcceptedSessionAsync(TcpClient client, CancellationToken cancellationToken) + { + using (client) + { + try + { + await HandleClientAsync(client, cancellationToken).ConfigureAwait(false); + } + finally + { + _sessionGate.Release(); + } + } + } + private async Task HandleClientAsync(TcpClient client, CancellationToken cancellationToken) { Metrics.SessionStarted(); @@ -300,7 +309,9 @@ await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken // Shared immutable buffer: concurrent sessions may write the same array; none mutate it. await stream.WriteAsync(preloaded.WireResponse, cancellationToken).ConfigureAwait(false); await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + // After a successful write/flush: success means the ARTICLE response left the peer. Metrics.RecordArticleSuccess(preloaded.LogicalByteCount, preloaded.WireResponse.Length); + metricsRecorded = true; return; } @@ -344,7 +355,9 @@ await WriteAsciiAsync(stream, "403 Article unavailable\r\n", cancellationToken) await stream.WriteAsync(status, cancellationToken).ConfigureAwait(false); await stream.WriteAsync(framed, cancellationToken).ConfigureAwait(false); await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + // After a successful write/flush: success means the ARTICLE response left the peer. Metrics.RecordArticleSuccess(logical.Length, status.Length + framed.Length); + metricsRecorded = true; } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerMetrics.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerMetrics.cs index a0a11ac0..73e05e62 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerMetrics.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerMetrics.cs @@ -1,3 +1,5 @@ +using System.Diagnostics; + namespace VectorNNTPD.BackFillerService.Tests.Lab.LocalArticlePeer { /// Aggregate counters for one lifetime. @@ -20,6 +22,10 @@ internal sealed class LocalArticleNntpPeerMetrics internal long AcceptedConnections => Volatile.Read(ref _acceptedConnections); internal long RejectedConnections => Volatile.Read(ref _rejectedConnections); internal long ArticleRequests => Volatile.Read(ref _articleRequests); + /// + /// ARTICLE responses fully written and flushed to the client stream (220 + body). + /// Not incremented for lookup-only success before a write completes. + /// internal long ArticleSuccess => Volatile.Read(ref _articleSuccess); internal long ArticleMissing => Volatile.Read(ref _articleMissing); internal long ArticleErrors => Volatile.Read(ref _articleErrors); @@ -72,6 +78,29 @@ internal void RecordArticleError() internal void RecordProtocolError() => Interlocked.Increment(ref _protocolErrors); + /// + /// Waits until reaches and + /// is zero. Timeout fails (does not pass the caller). + /// Clients may finish reading before the peer records post-flush success. + /// + internal async Task WaitForArticleSuccessAsync(long expected, TimeSpan? timeout = null) + { + var limit = timeout ?? TimeSpan.FromSeconds(5); + var started = Stopwatch.GetTimestamp(); + while (ArticleSuccess < expected || OutstandingArticles != 0) + { + if (Stopwatch.GetElapsedTime(started) > limit) + { + throw new TimeoutException( + $"Timed out waiting for ArticleSuccess>={expected} with OutstandingArticles==0 " + + $"(success={ArticleSuccess}, outstanding={OutstandingArticles}, " + + $"missing={ArticleMissing}, errors={ArticleErrors})."); + } + + await Task.Yield(); + } + } + private static void UpdateMax(ref int location, int candidate) { var observed = Volatile.Read(ref location); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs index fffe61f6..1b0a3132 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeerTests.cs @@ -53,6 +53,7 @@ public async Task Peer_serves_greeting_capabilities_article_and_quit() await WriteAsync(stream, "QUIT\r\n"u8.ToArray()); Assert.StartsWith("205 ", await ReadLineAsync(stream)); + await peer.Metrics.WaitForArticleSuccessAsync(expected: 1); Assert.Equal(1, peer.Metrics.ArticleSuccess); Assert.Equal(0, peer.Metrics.ArticleMissing); Assert.Equal(0, peer.Metrics.OutstandingArticles); @@ -129,6 +130,7 @@ public async Task BackFiller_client_downloads_article_from_peer_without_tls_or_a Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, result.Kind); Assert.NotNull(result.Article); Assert.Contains("hello"u8, result.Article!.Memory.Span); + await peer.Metrics.WaitForArticleSuccessAsync(expected: 1); Assert.Equal(1, peer.Metrics.ArticleSuccess); } @@ -163,6 +165,7 @@ public async Task Peer_accepts_unauthenticated_flow_matching_lab_rewrite() Assert.StartsWith("220 ", await ReadLineAsync(stream)); await DrainMultilineUntilTerminatorAsync(stream); + await peer.Metrics.WaitForArticleSuccessAsync(expected: 1); Assert.Equal(1, peer.Metrics.ArticleSuccess); Assert.Equal(0, peer.Metrics.ProtocolErrors); } @@ -185,6 +188,7 @@ public async Task Concurrent_article_requests_complete() Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, result.Kind); }); await Task.WhenAll(tasks); + await peer.Metrics.WaitForArticleSuccessAsync(expected: 8); Assert.Equal(8, peer.Metrics.ArticleSuccess); Assert.Equal(0, peer.Metrics.OutstandingArticles); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs index 9416214e..a5fcdde9 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStore.cs @@ -78,8 +78,7 @@ internal static async Task LoadAsync( logicalBytes += logical.Length; wireBytes += wire.Length; } - // codeql[cs/catch-of-all-exceptions] Fail-closed preload must capture every article load failure and continue counting remaining files. - catch (Exception ex) + catch (Exception ex) when (IsExpectedArticleLoadFailure(ex)) { failures++; firstFailure ??= ex; @@ -110,6 +109,16 @@ internal static async Task LoadAsync( return new PreloadedArticleStore(map, metrics); } + + /// + /// File/content failures from that should be + /// aggregated into fail-closed preload. Unexpected programming faults are not matched. + /// + private static bool IsExpectedArticleLoadFailure(Exception ex) => + ex is IOException + or UnauthorizedAccessException + or InvalidOperationException + or ArgumentException; } /// Immutable precomputed ARTICLE wire response for one Message-ID. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs index 612699d8..b0d93e7b 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/PreloadedArticleStoreTests.cs @@ -42,6 +42,9 @@ public async Task Preload_matches_file_path_framing_byte_for_byte_including_term var memWire = await CaptureArticleResponseAsync(memPeer.Port, messageId); Assert.True(expectedWire.AsSpan().SequenceEqual(fileWire)); Assert.True(expectedWire.AsSpan().SequenceEqual(memWire)); + // Client read can finish before the peer records success after flush; wait on settlement. + await filePeer.Metrics.WaitForArticleSuccessAsync(expected: 1); + await memPeer.Metrics.WaitForArticleSuccessAsync(expected: 1); Assert.Equal(1, filePeer.Metrics.ArticleSuccess); Assert.Equal(1, memPeer.Metrics.ArticleSuccess); Assert.Equal(filePeer.Metrics.LogicalBytesTransferred, memPeer.Metrics.LogicalBytesTransferred); @@ -70,6 +73,7 @@ public async Task Concurrent_sessions_share_precomputed_buffers_without_corrupti Assert.True(expected.AsSpan().SequenceEqual(wire)); }); await Task.WhenAll(tasks); + await peer.Metrics.WaitForArticleSuccessAsync(expected: 16); Assert.Equal(16, peer.Metrics.ArticleSuccess); Assert.Equal(0, peer.Metrics.OutstandingArticles); Assert.True(peer.PreloadedStore.TryGet(messageId, out var again)); @@ -128,6 +132,7 @@ public async Task Memory_mode_counters_and_dispose_rebind() port = peer.Port; var wire = await CaptureArticleResponseAsync(peer.Port, messageId); Assert.True(EndsWithArticleTerminator(wire)); + await peer.Metrics.WaitForArticleSuccessAsync(expected: 1); Assert.Equal(1, peer.Metrics.ArticleRequests); Assert.Equal(1, peer.Metrics.ArticleSuccess); Assert.True(peer.Metrics.LogicalBytesTransferred > 0); @@ -177,6 +182,7 @@ public async Task BackFiller_client_downloads_from_memory_peer() using var result = await session.DownloadArticleAsync(messageId, CancellationToken.None); Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, result.Kind); Assert.Contains("hello"u8, result.Article!.Memory.Span); + await peer.Metrics.WaitForArticleSuccessAsync(expected: 1); Assert.Equal(1, peer.Metrics.ArticleSuccess); } From c50f8f33cd073eeaf42f0a9b81784a4ed6e2c418 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 28 Oct 2026 15:31:25 +0200 Subject: [PATCH 5/8] Fix CodeQL findings and clarify RabbitMQ limiter ownership --- .../RabbitMq/RabbitMqBorrowedRateLimiter.cs | 57 +++++++++++++++++++ .../RabbitMq/RabbitMqClientConnection.cs | 15 ++--- .../RabbitMq/RabbitMqClientPublishChannel.cs | 28 ++++++++- .../LocalArticlePeer/LocalArticleNntpPeer.cs | 4 +- .../RabbitMqBorrowedRateLimiterTests.cs | 45 +++++++++++++++ 5 files changed, 138 insertions(+), 11 deletions(-) create mode 100644 src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqBorrowedRateLimiter.cs create mode 100644 tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqBorrowedRateLimiter.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqBorrowedRateLimiter.cs new file mode 100644 index 00000000..5ac1f966 --- /dev/null +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqBorrowedRateLimiter.cs @@ -0,0 +1,57 @@ +using System.Threading.RateLimiting; + +namespace VectorNNTPD.Common.Messaging.RabbitMq +{ + /// + /// Forwards rate-limit operations to an inner without taking ownership. + /// + /// + /// RabbitMQ.Client 7.2.2 Channel stores and disposes + /// CreateChannelOptions.OutstandingPublisherConfirmationsRateLimiter. When production code + /// must retain sole disposal ownership of a ThrottlingRateLimiter, pass this facade into + /// CreateChannelOptions so the client dispose path only disposes the facade. + /// + internal sealed class RabbitMqBorrowedRateLimiter : RateLimiter + { + /// Limiter that performs acquires; not disposed by this facade. + private readonly RateLimiter _inner; + + /// Wraps without assuming disposal ownership. + /// Limiter used for acquire/statistics; owned by the caller. + internal RabbitMqBorrowedRateLimiter(RateLimiter inner) + { + ArgumentNullException.ThrowIfNull(inner); + _inner = inner; + } + + /// + public override TimeSpan? IdleDuration => _inner.IdleDuration; + + /// + public override RateLimiterStatistics? GetStatistics() => _inner.GetStatistics(); + + /// + protected override RateLimitLease AttemptAcquireCore(int permitCount) + => _inner.AttemptAcquire(permitCount); + + /// + protected override ValueTask AcquireAsyncCore( + int permitCount, + CancellationToken cancellationToken) + => _inner.AcquireAsync(permitCount, cancellationToken); + + /// + protected override void Dispose(bool disposing) + { + // Intentionally empty: the inner limiter is owned by the caller. + base.Dispose(disposing); + } + + /// + protected override ValueTask DisposeAsyncCore() + { + // Intentionally empty: the inner limiter is owned by the caller. + return default; + } + } +} diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs index 373f75a5..22ef6755 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs @@ -148,7 +148,8 @@ public async Task CreatePublishChannelAsync( // Explicit limiter: the two-arg CreateChannelOptions leaves OutstandingPublisherConfirmationsRateLimiter // null (unlimited). Align with RabbitMqPublisherConfirmLimits / Article Work response publisher gate. - // On success the channel owns the limiter (RabbitMQ.Client 7.2.2 Channel.Dispose/DisposeAsync). + // RabbitMQ.Client 7.2.2 Channel stores and disposes the options limiter, so pass a borrowed facade + // and transfer sole ownership of the real ThrottlingRateLimiter to RabbitMqClientPublishChannel. // Null the local after transfer so finally disposes only the failure path. ThrottlingRateLimiter? limiter = new ThrottlingRateLimiter( RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations); @@ -157,19 +158,19 @@ public async Task CreatePublishChannelAsync( var options = new CreateChannelOptions( publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true, - outstandingPublisherConfirmationsRateLimiter: limiter); + outstandingPublisherConfirmationsRateLimiter: new RabbitMqBorrowedRateLimiter(limiter)); var channel = await _connection .CreateChannelAsync(options: options, cancellationToken: cancellationToken) .ConfigureAwait(false); + var ownedLimiter = limiter; limiter = null; - return new RabbitMqClientPublishChannel(channel, generation); + return new RabbitMqClientPublishChannel(channel, generation, ownedLimiter); } finally { - if (limiter is not null) - { - await limiter.DisposeAsync().ConfigureAwait(false); - } + // ThrottlingRateLimiter (RateLimiter 8.0.0) only releases its ConcurrencyLimiter from Dispose(true). + // DisposeAsync ends in Dispose(false) and would leak the inner limiter. + limiter?.Dispose(); } } diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs index e6a9d635..0af0277c 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs @@ -1,3 +1,4 @@ +using System.Threading.RateLimiting; using RabbitMQ.Client; using RabbitMQ.Client.Events; using RabbitMQ.Client.Exceptions; @@ -21,17 +22,36 @@ internal sealed class RabbitMqClientPublishChannel : IRabbitMqPublishChannel /// private readonly IChannel _channel; + /// + /// Sole owner of the outstanding-confirm . + /// The RabbitMQ.Client channel receives only a facade. + /// + private readonly RateLimiter _outstandingConfirmationsRateLimiter; + /// 1 after the first . private int _disposed; /// 1 after an unexpected fault has been raised once for this channel. private int _faultRaised; - /// Initializes a new wrapper around an opened confirm-enabled channel. - internal RabbitMqClientPublishChannel(IChannel channel, long generation) + /// + /// Initializes a new wrapper around an opened confirm-enabled channel and the outstanding-confirm limiter + /// supplied at channel creation. + /// + /// Opened confirm-tracked RabbitMQ channel. + /// Connection generation for this channel. + /// + /// Owned outstanding-confirm limiter (not the borrowed facade passed to ). + /// + internal RabbitMqClientPublishChannel( + IChannel channel, + long generation, + RateLimiter outstandingConfirmationsRateLimiter) { ArgumentNullException.ThrowIfNull(channel); + ArgumentNullException.ThrowIfNull(outstandingConfirmationsRateLimiter); _channel = channel; + _outstandingConfirmationsRateLimiter = outstandingConfirmationsRateLimiter; Generation = generation; _channel.ChannelShutdownAsync += OnChannelShutdownAsync; } @@ -146,7 +166,11 @@ public async ValueTask DisposeAsync() { } + // Dispose the channel first so in-flight confirm tracking finishes; the channel only holds a + // borrowed facade and does not dispose this limiter. await _channel.DisposeAsync().ConfigureAwait(false); + // ThrottlingRateLimiter only releases managed state from Dispose(true); DisposeAsync uses Dispose(false). + _outstandingConfirmationsRateLimiter.Dispose(); } /// Raises once for an unexpected channel shutdown. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs index 3322749d..86c9fbfe 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Lab/LocalArticlePeer/LocalArticleNntpPeer.cs @@ -310,8 +310,8 @@ await WriteAsciiAsync(stream, "430 No Such Article Found\r\n", cancellationToken await stream.WriteAsync(preloaded.WireResponse, cancellationToken).ConfigureAwait(false); await stream.FlushAsync(cancellationToken).ConfigureAwait(false); // After a successful write/flush: success means the ARTICLE response left the peer. + // No metricsRecorded=true: this path returns immediately; a later catch cannot run. Metrics.RecordArticleSuccess(preloaded.LogicalByteCount, preloaded.WireResponse.Length); - metricsRecorded = true; return; } @@ -356,8 +356,8 @@ await WriteAsciiAsync(stream, "403 Article unavailable\r\n", cancellationToken) await stream.WriteAsync(framed, cancellationToken).ConfigureAwait(false); await stream.FlushAsync(cancellationToken).ConfigureAwait(false); // After a successful write/flush: success means the ARTICLE response left the peer. + // No metricsRecorded=true: try ends here; a later catch cannot run after this statement. Metrics.RecordArticleSuccess(logical.Length, status.Length + framed.Length); - metricsRecorded = true; } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs new file mode 100644 index 00000000..c630555b --- /dev/null +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs @@ -0,0 +1,45 @@ +using System.Threading.RateLimiting; +using RabbitMQ.Client; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.Common.Tests.Messaging.RabbitMq +{ + /// + /// Verifies the borrowed outstanding-confirm limiter facade does not assume disposal ownership. + /// + public sealed class RabbitMqBorrowedRateLimiterTests + { + /// + /// Disposing the facade must leave the inner usable so + /// remains the sole disposal owner. + /// + [Fact] + public async Task Dispose_of_borrowed_facade_does_not_dispose_inner_limiter() + { + using var inner = new ThrottlingRateLimiter(4); + var borrowed = new RabbitMqBorrowedRateLimiter(inner); + + await borrowed.DisposeAsync(); + borrowed.Dispose(); + + using RateLimitLease lease = await inner.AcquireAsync(1); + Assert.True(lease.IsAcquired); + } + + /// + /// After the owner disposes the inner limiter, acquires must fail closed. + /// + [Fact] + public async Task Dispose_of_inner_limiter_rejects_further_acquires() + { + var inner = new ThrottlingRateLimiter(4); + var borrowed = new RabbitMqBorrowedRateLimiter(inner); + inner.Dispose(); + + await Assert.ThrowsAsync(async () => + { + _ = await borrowed.AcquireAsync(1); + }); + } + } +} From 9da2f484f0df81383446d695664325fd36437ac6 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 28 Oct 2026 17:03:05 +0200 Subject: [PATCH 6/8] Fix CodeQL disposal findings in tests and RabbitMQ limiter --- .../RabbitMq/RabbitMqClientConnection.cs | 32 +++--------- .../RabbitMq/RabbitMqClientPublishChannel.cs | 50 ++++++++++++++++++- .../RabbitMqBorrowedRateLimiterTests.cs | 41 ++++++++++----- 3 files changed, 83 insertions(+), 40 deletions(-) diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs index 22ef6755..88447159 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientConnection.cs @@ -146,32 +146,12 @@ public async Task CreatePublishChannelAsync( throw new InvalidOperationException("RabbitMQ connection is not open for publication."); } - // Explicit limiter: the two-arg CreateChannelOptions leaves OutstandingPublisherConfirmationsRateLimiter - // null (unlimited). Align with RabbitMqPublisherConfirmLimits / Article Work response publisher gate. - // RabbitMQ.Client 7.2.2 Channel stores and disposes the options limiter, so pass a borrowed facade - // and transfer sole ownership of the real ThrottlingRateLimiter to RabbitMqClientPublishChannel. - // Null the local after transfer so finally disposes only the failure path. - ThrottlingRateLimiter? limiter = new ThrottlingRateLimiter( - RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations); - try - { - var options = new CreateChannelOptions( - publisherConfirmationsEnabled: true, - publisherConfirmationTrackingEnabled: true, - outstandingPublisherConfirmationsRateLimiter: new RabbitMqBorrowedRateLimiter(limiter)); - var channel = await _connection - .CreateChannelAsync(options: options, cancellationToken: cancellationToken) - .ConfigureAwait(false); - var ownedLimiter = limiter; - limiter = null; - return new RabbitMqClientPublishChannel(channel, generation, ownedLimiter); - } - finally - { - // ThrottlingRateLimiter (RateLimiter 8.0.0) only releases its ConcurrencyLimiter from Dispose(true). - // DisposeAsync ends in Dispose(false) and would leak the inner limiter. - limiter?.Dispose(); - } + // Confirm-tracked publish channels own an explicit outstanding-confirm limiter (see + // RabbitMqClientPublishChannel.CreateConfirmTrackedAsync). The two-arg CreateChannelOptions + // overload leaves OutstandingPublisherConfirmationsRateLimiter null (unlimited). + return await RabbitMqClientPublishChannel + .CreateConfirmTrackedAsync(_connection, generation, cancellationToken) + .ConfigureAwait(false); } /// diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs index 0af0277c..72c7669b 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs @@ -43,7 +43,7 @@ internal sealed class RabbitMqClientPublishChannel : IRabbitMqPublishChannel /// /// Owned outstanding-confirm limiter (not the borrowed facade passed to ). /// - internal RabbitMqClientPublishChannel( + private RabbitMqClientPublishChannel( IChannel channel, long generation, RateLimiter outstandingConfirmationsRateLimiter) @@ -56,6 +56,54 @@ internal RabbitMqClientPublishChannel( _channel.ChannelShutdownAsync += OnChannelShutdownAsync; } + /// + /// Opens a confirm-tracked channel with an explicit outstanding-confirm limiter owned by the returned wrapper. + /// + /// + /// + /// RabbitMQ.Client 7.2.2 disposes the limiter instance stored on , so the + /// client receives a facade. The real + /// is owned here and released with sync + /// from (RateLimiter's DisposeAsync ends in Dispose(false) and would + /// not release the inner ConcurrencyLimiter). + /// + /// + /// Construction uses / so + /// failure paths dispose via using / await using, and success transfers ownership into this + /// wrapper—matching . + /// + /// + /// Open RabbitMQ connection used to create the channel. + /// Connection generation stamped on the wrapper. + /// Cancels channel open. + /// Wrapper that owns both the channel and the outstanding-confirm limiter. + internal static async Task CreateConfirmTrackedAsync( + IConnection connection, + long generation, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(connection); + + using var limiterOwnership = new DisposableOwnership( + new ThrottlingRateLimiter(RabbitMqPublisherConfirmLimits.MaxOutstandingConfirmations)); + var options = new CreateChannelOptions( + publisherConfirmationsEnabled: true, + publisherConfirmationTrackingEnabled: true, + outstandingPublisherConfirmationsRateLimiter: new RabbitMqBorrowedRateLimiter( + limiterOwnership.Resource)); + var channel = await connection + .CreateChannelAsync(options: options, cancellationToken: cancellationToken) + .ConfigureAwait(false); + await using var channelOwnership = new AsyncDisposableOwnership(channel); + var publishChannel = new RabbitMqClientPublishChannel( + channelOwnership.Resource, + generation, + limiterOwnership.Resource); + channelOwnership.Transfer(); + limiterOwnership.Transfer(); + return publishChannel; + } + /// public long Generation { get; } diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs index c630555b..4e65f700 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs @@ -10,36 +10,51 @@ namespace VectorNNTPD.Common.Tests.Messaging.RabbitMq public sealed class RabbitMqBorrowedRateLimiterTests { /// - /// Disposing the facade must leave the inner usable so - /// remains the sole disposal owner. + /// Sync on the facade must leave the inner limiter usable. /// [Fact] - public async Task Dispose_of_borrowed_facade_does_not_dispose_inner_limiter() + public void Dispose_of_borrowed_facade_does_not_dispose_inner_limiter() { using var inner = new ThrottlingRateLimiter(4); - var borrowed = new RabbitMqBorrowedRateLimiter(inner); + using (var borrowed = new RabbitMqBorrowedRateLimiter(inner)) + { + } + + using RateLimitLease lease = inner.AttemptAcquire(1); + Assert.True(lease.IsAcquired); + } - await borrowed.DisposeAsync(); - borrowed.Dispose(); + /// + /// on the facade must leave the inner limiter usable. + /// + [Fact] + public async Task DisposeAsync_of_borrowed_facade_does_not_dispose_inner_limiter() + { + using var inner = new ThrottlingRateLimiter(4); + await using (var borrowed = new RabbitMqBorrowedRateLimiter(inner)) + { + } using RateLimitLease lease = await inner.AcquireAsync(1); Assert.True(lease.IsAcquired); } /// - /// After the owner disposes the inner limiter, acquires must fail closed. + /// After the owner disposes the inner limiter, acquires through the facade must fail closed. /// [Fact] public async Task Dispose_of_inner_limiter_rejects_further_acquires() { var inner = new ThrottlingRateLimiter(4); - var borrowed = new RabbitMqBorrowedRateLimiter(inner); - inner.Dispose(); - - await Assert.ThrowsAsync(async () => + await using (var borrowed = new RabbitMqBorrowedRateLimiter(inner)) { - _ = await borrowed.AcquireAsync(1); - }); + inner.Dispose(); + + await Assert.ThrowsAsync(async () => + { + _ = await borrowed.AcquireAsync(1); + }); + } } } } From 5715c73c4c2808c33bcab315945ba41f1335a327 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 28 Oct 2026 17:25:19 +0200 Subject: [PATCH 7/8] Strengthen borrowed limiter disposal tests for CodeQL --- .../RabbitMqBorrowedRateLimiterTests.cs | 63 ++++++++++++++----- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs index 4e65f700..91c60f20 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs @@ -10,33 +10,60 @@ namespace VectorNNTPD.Common.Tests.Messaging.RabbitMq public sealed class RabbitMqBorrowedRateLimiterTests { /// - /// Sync on the facade must leave the inner limiter usable. + /// Sync facade dispose must not dispose the inner limiter, and acquires must deplete inner permits. /// [Fact] public void Dispose_of_borrowed_facade_does_not_dispose_inner_limiter() { - using var inner = new ThrottlingRateLimiter(4); + using var inner = new ThrottlingRateLimiter(1); + RateLimitLease heldThroughFacade; using (var borrowed = new RabbitMqBorrowedRateLimiter(inner)) { + heldThroughFacade = borrowed.AttemptAcquire(1); + Assert.True(heldThroughFacade.IsAcquired); + + // Lease is held on the inner limiter: no further permit available. + using RateLimitLease blocked = inner.AttemptAcquire(1); + Assert.False(blocked.IsAcquired); + } + + // Facade disposed; the outstanding lease still belongs to the inner limiter. + using (heldThroughFacade) + { + using RateLimitLease stillBlocked = inner.AttemptAcquire(1); + Assert.False(stillBlocked.IsAcquired); } - using RateLimitLease lease = inner.AttemptAcquire(1); - Assert.True(lease.IsAcquired); + using RateLimitLease afterRelease = inner.AttemptAcquire(1); + Assert.True(afterRelease.IsAcquired); } /// - /// on the facade must leave the inner limiter usable. + /// Async facade dispose must not dispose the inner limiter, and async acquires must deplete inner permits. /// [Fact] public async Task DisposeAsync_of_borrowed_facade_does_not_dispose_inner_limiter() { - using var inner = new ThrottlingRateLimiter(4); + using var inner = new ThrottlingRateLimiter(1); + RateLimitLease heldThroughFacade; await using (var borrowed = new RabbitMqBorrowedRateLimiter(inner)) { + heldThroughFacade = await borrowed.AcquireAsync(1); + Assert.True(heldThroughFacade.IsAcquired); + + // AttemptAcquire (not AcquireAsync): exhausted permits queue async waits. + using RateLimitLease blocked = inner.AttemptAcquire(1); + Assert.False(blocked.IsAcquired); + } + + using (heldThroughFacade) + { + using RateLimitLease stillBlocked = inner.AttemptAcquire(1); + Assert.False(stillBlocked.IsAcquired); } - using RateLimitLease lease = await inner.AcquireAsync(1); - Assert.True(lease.IsAcquired); + using RateLimitLease afterRelease = await inner.AcquireAsync(1); + Assert.True(afterRelease.IsAcquired); } /// @@ -46,14 +73,22 @@ public async Task DisposeAsync_of_borrowed_facade_does_not_dispose_inner_limiter public async Task Dispose_of_inner_limiter_rejects_further_acquires() { var inner = new ThrottlingRateLimiter(4); - await using (var borrowed = new RabbitMqBorrowedRateLimiter(inner)) + try { - inner.Dispose(); - - await Assert.ThrowsAsync(async () => + await using (var borrowed = new RabbitMqBorrowedRateLimiter(inner)) { - _ = await borrowed.AcquireAsync(1); - }); + inner.Dispose(); + + await Assert.ThrowsAsync(async () => + { + _ = await borrowed.AcquireAsync(1); + }); + } + } + finally + { + // Idempotent if already disposed above; covers assertion-failure paths before Dispose. + inner.Dispose(); } } } From b284850387b0eb5de42f05676c8620c373f306bc Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Wed, 28 Oct 2026 18:48:16 +0200 Subject: [PATCH 8/8] Fix borrowed limiter test disposal ownership --- .../RabbitMqBorrowedRateLimiterTests.cs | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs index 91c60f20..0646d2ae 100644 --- a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqBorrowedRateLimiterTests.cs @@ -1,5 +1,6 @@ using System.Threading.RateLimiting; using RabbitMQ.Client; +using VectorNNTPD.Common.Core; using VectorNNTPD.Common.Messaging.RabbitMq; namespace VectorNNTPD.Common.Tests.Messaging.RabbitMq @@ -72,23 +73,18 @@ public async Task DisposeAsync_of_borrowed_facade_does_not_dispose_inner_limiter [Fact] public async Task Dispose_of_inner_limiter_rejects_further_acquires() { - var inner = new ThrottlingRateLimiter(4); - try + // DisposableOwnership makes mid-test owner disposal a using-friendly no-op at scope exit + // (atomic exchange clears the held resource before Dispose runs on the limiter). + using var ownership = new DisposableOwnership( + new ThrottlingRateLimiter(4)); + await using (var borrowed = new RabbitMqBorrowedRateLimiter(ownership.Resource)) { - await using (var borrowed = new RabbitMqBorrowedRateLimiter(inner)) - { - inner.Dispose(); + ownership.Dispose(); - await Assert.ThrowsAsync(async () => - { - _ = await borrowed.AcquireAsync(1); - }); - } - } - finally - { - // Idempotent if already disposed above; covers assertion-failure paths before Dispose. - inner.Dispose(); + await Assert.ThrowsAsync(async () => + { + _ = await borrowed.AcquireAsync(1); + }); } } }