From 8f878bf799b7abb5af5568c3ca9e76e1c1d8bd09 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Sun, 25 Oct 2026 19:02:54 +0200 Subject: [PATCH 1/3] fix(backfiller): correct settlement, ARTICLE reuse, and pool capacity ACK and reply ProviderFailure, UnexpectedFailure, and RetentionRejected so completed work is not redelivered without a client reply. Handle mandatory unroutable replies as terminal NACK without requeue. Keep NNTP sessions reusable for ARTICLE 403/500/501/503, map 451 to ArticleNotFound, and retain pool capacity until disposal completes. Add connectionId and retirement-cause diagnostics, the gated E2E Stress harness, regression tests, and related CodeQL remediations. --- docs/architecture.md | 12 +- docs/common/configuration.md | 2 +- .../ArticleWorkDeliveryPipeline.cs | 89 ++- .../ArticleWorkDispositionPlanner.cs | 15 +- .../ArticleWork/ArticleWorkLogMessages.cs | 51 +- .../ArticleWorkResponsePublisher.cs | 16 + .../ArticleWorkResponseWireProtocol.cs | 25 +- .../ArticleWork/ProviderArticleWorkHandler.cs | 69 +- .../Nntp/ArticleRetrievalKind.cs | 2 +- .../Nntp/ArticleRetrievalResult.cs | 15 + .../Nntp/NntpArticleRetriever.cs | 18 +- .../Nntp/NntpExceptionDiagnostics.cs | 137 ++++ .../Nntp/NntpLogMessages.cs | 24 +- .../Nntp/NntpProviderSession.cs | 78 +- .../Nntp/NntpSessionPool.cs | 293 ++++++-- .../Nntp/NntpSessionRetirementCause.cs | 26 + .../Nntp/NntpStatusCode.cs | 52 +- .../Messaging/RabbitMq/IRabbitMqRpcChannel.cs | 5 +- .../RabbitMq/RabbitMqClientPublishChannel.cs | 11 +- .../RabbitMq/RabbitMqClientRpcChannel.cs | 7 +- .../RabbitMqPublicationUnroutableException.cs | 32 + ...leFailedDiagnosticsIsolatedRuntimeTests.cs | 192 +++++ .../ArticleWork/ArticleProcessingLogTests.cs | 23 +- ...icleWorkArticlePipelineIntegrationTests.cs | 10 +- ...leWorkQuorumHealthAndDeliveryLimitTests.cs | 24 +- .../ArticleWorkResponsePublicationTests.cs | 162 ++++- .../ArticleWorkResponseWireProtocolTests.cs | 56 +- .../ArticleWork/ArticleWorkSettlementTests.cs | 52 +- .../ProviderFailureTerminalSettlementTests.cs | 134 ++++ ...expectedFailureRpcErrorDiagnosticsTests.cs | 331 +++++++++ .../LinuxServiceInstallContractTests.cs | 24 +- .../BackFillerConcurrencyIntegrationTests.cs | 18 +- .../Nntp/NntpArticleWorkIntegrationTests.cs | 51 +- .../Nntp/NntpCommandBudgetTests.cs | 5 +- .../NntpConnectionIdentityCorrelationTests.cs | 217 ++++++ .../Nntp/NntpExceptionDiagnosticsTests.cs | 122 ++++ .../Nntp/NntpProviderSessionTests.cs | 117 ++- .../Nntp/NntpSessionPoolTests.cs | 209 +++++- .../Nntp/NntpShutdownDeadlineTests.cs | 3 +- .../ArticleWorkRpcLifecycleContractTests.cs | 118 +++ .../ArticleWorkRpcOutstandingTracker.cs | 68 ++ .../ArticleWorkRpcOutstandingTrackerTests.cs | 77 ++ .../Stress/ArticleWorkStressClient.cs | 561 +++++++++++++++ ...eWorkStressClientRequestPropertiesTests.cs | 97 +++ .../Stress/BackFillerE2EStressBench.cs | 585 +++++++++++++++ .../Stress/BackFillerE2EStressHarness.cs | 678 ++++++++++++++++++ .../Stress/BackFillerNntpStressClient.cs | 269 +++++++ .../Stress/CorpusArticlePreflight.cs | 139 ++++ .../Stress/NewsLogMessageIdDiscovery.cs | 365 ++++++++++ .../Stress/NewsLogMessageIdDiscoveryTests.cs | 216 ++++++ .../Stress/NntpArticleResponseReader.cs | 330 +++++++++ .../Stress/NntpArticleResponseReaderTests.cs | 81 +++ .../Stress/StageLatencyStats.cs | 55 ++ .../Stress/StressAbortDecision.cs | 50 ++ .../Stress/StressAbortDecisionTests.cs | 74 ++ .../Stress/StressCorpusIntegrityTests.cs | 308 ++++++++ .../Stress/StressCorpusPaths.cs | 25 + .../Stress/StressHarnessOptions.cs | 207 ++++++ .../Stress/StressMetrics.cs | 503 +++++++++++++ .../Stress/StressPublishGateTests.cs | 80 +++ .../Stress/StressQueuePreflight.cs | 191 +++++ .../Stress/StressQueuePreflightTests.cs | 29 + .../Stress/StressWorkloadGuard.cs | 91 +++ .../Stress/WorkloadCorpusManifest.cs | 85 +++ .../TestDoubles/FakeBackFillerRabbitMq.cs | 2 +- ...itMqPublicationUnroutableExceptionTests.cs | 33 + 66 files changed, 7742 insertions(+), 304 deletions(-) create mode 100644 src/VectorNNTPD.BackFillerService/Nntp/NntpExceptionDiagnostics.cs create mode 100644 src/VectorNNTPD.BackFillerService/Nntp/NntpSessionRetirementCause.cs create mode 100644 src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqPublicationUnroutableException.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleFailedDiagnosticsIsolatedRuntimeTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ProviderFailureTerminalSettlementTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/UnexpectedFailureRpcErrorDiagnosticsTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpConnectionIdentityCorrelationTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpExceptionDiagnosticsTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcLifecycleContractTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTracker.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTrackerTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscovery.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscoveryTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReader.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReaderTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StageLatencyStats.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecision.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecisionTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusPaths.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressMetrics.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressPublishGateTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflight.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflightTests.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/StressWorkloadGuard.cs create mode 100644 tests/VectorNNTPD.BackFillerService.Tests/Stress/WorkloadCorpusManifest.cs create mode 100644 tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqPublicationUnroutableExceptionTests.cs diff --git a/docs/architecture.md b/docs/architecture.md index 615a178f..21174eca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -33,19 +33,19 @@ Provider queue type is derived from the configured unique `RabbitMQ:Hosts` count | Outcome | Broker action | Publish terminal response | |---|---|---| | Success | Basic.Ack | Yes | -| ArticleNotFound | Basic.Ack | Yes | +| ArticleNotFound | Basic.Ack | Yes. Definitive provider `430` or ARTICLE `451`; callers must not republish the same logical lookup to another backbone because of this outcome. | | InvalidArticle | Basic.Ack | Yes. Includes an article `ArticleWorkDeliveryFraming` cannot send. That rejection happens before retention, so the article is not stored. | | InvalidRequest | Basic.Ack | Yes when both `CorrelationId` and `ReplyTo` are present. Otherwise ACK without publication | -| ProviderFailure | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No | -| Cancelled | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No | -| UnexpectedFailure | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No | -| RetentionRejected | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No | +| ProviderFailure | Basic.Ack | Yes. Correlated failure reply (`outcome=ProviderFailure`). Must not NACK-requeue: classic queues lack enforceable delivery-count exhaustion and requeue without a reply caused unbounded redelivery plus client timeout republish. | +| Cancelled | Basic.Nack, requeue true while delivery count < limit; otherwise NACK without requeue | No. Cooperative cancel / shutdown may resume the delivery later. | +| UnexpectedFailure | Basic.Ack | Yes. Correlated failure reply (`outcome=UnexpectedFailure`). | +| RetentionRejected | Basic.Ack | Yes. Correlated failure reply (`outcome=RetentionRejected`). | If cancellation is already requested, the planner returns NACK (requeue while retries remain) and does not publish, including when the handler outcome was terminal. `ArticleWorkSettlementLease` is one settlement attempt for one delivery tag on the original consumer channel and connection generation. It sets the settled flag before the broker RPC. If that RPC throws, the flag stays set and the lease does not retry. If the channel is stale, closed, or not the original channel, `TrySettleAsync` returns false without ACK or NACK, and the broker may redeliver. Pipeline settlement calls use `CancellationToken.None`. -When the success publisher reports that it does not complete success publication, the pipeline NACK-requeues and still returns `ArticleWorkOutcome.Success`. A required terminal response that is cancelled or that fails NACK-requeues and returns `ArticleWorkOutcome.UnexpectedFailure`, including when the planned disposition was ACK. The lease still settles at most once. A failed NACK is not retried on that lease, and a stale or replacement channel is not settled. A confirm lost after the broker has accepted the publish is indistinguishable from a failed publish, so the requeue can publish a second reply with the same correlation id. That is at-least-once for the reply when the confirm is ambiguous, not exactly-once. +When the success publisher reports that it does not complete success publication, the pipeline NACK-requeues and still returns `ArticleWorkOutcome.Success`. A required terminal response that is cancelled or that fails for a **transient or ambiguous** reason NACK-requeues and returns `ArticleWorkOutcome.UnexpectedFailure`, including when the planned disposition was ACK. A **confirmed mandatory unroutable** reply (`RabbitMqPublicationUnroutableException` / broker basic.return on the ReplyTo queue) is terminal: NACK **without** requeue. Successful upstream article retrieval is not the same as successful RPC delivery; a permanently missing exclusive ReplyTo cannot be repaired by refetching. Because no dead-letter exchange is declared on Article Work queues, that terminal NACK **discards** the original work message at the broker. This is the same discard class as delivery-limit exhaustion. It is **not** a dead-letter. Operator visibility and recoverability for that path are the structured Warning `ReplyUnroutableTerminal` (request id, Message-ID, backbone, correlation id, ReplyTo, outcome) plus any article already retained from a successful upstream fetch. Introducing a durable failed-work queue or DLX is an explicit topology/ops decision and is not implied by the current settlement. The lease still settles at most once. A failed NACK is not retried on that lease, and a stale or replacement channel is not settled. A confirm lost after the broker has accepted the publish is indistinguishable from a failed publish, so the requeue can publish a second reply with the same correlation id. That is at-least-once for the reply when the confirm is ambiguous, not exactly-once. Publisher confirms (`BasicAcksAsync` / `BasicNacksAsync` on an async-confirm publish channel) are the broker's response to an outbound publish. They are not the consumer ACK of the inbound delivery. diff --git a/docs/common/configuration.md b/docs/common/configuration.md index 97e6c293..aa6beec1 100644 --- a/docs/common/configuration.md +++ b/docs/common/configuration.md @@ -42,7 +42,7 @@ There is no application-level RabbitMQ connection pool. `MinConnections` and `Ma | `PoolReconnectMaxDelayMs` | int | `30000` | **yes** | Application reconnect max delay (`50–300000`; ≥ base) | | `ChannelLeaseTimeoutSeconds` | int | `60` | **yes** | Validated; reserved for later channel work (`1–3600`; ≥ `RpcTimeoutSeconds`) | | `WorkRequestMaxPayloadBytes` | int | `1024` | **yes** | Maximum admitted Article Work request payload bytes (`1–4096`) | -| `ArticleWorkDeliveryLimit` | int | `10` | **yes** | Declared as `x-delivery-limit` on quorum Article Work queues and used as the application requeue cap when broker delivery-count metadata is available (`1–1000`). Not declared on classic single-host queues (broker does not accept that argument there). With count `0..(limit-1)` the app may requeue; at `limit` it rejects without requeue. No DLX is declared; exhausted messages are dropped | +| `ArticleWorkDeliveryLimit` | int | `10` | **yes** | Declared as `x-delivery-limit` on quorum Article Work queues and used as the application requeue cap when broker delivery-count metadata is available (`1–1000`). Not declared on classic single-host queues (broker does not accept that argument there). With count `0..(limit-1)` the app may requeue; at `limit` it rejects without requeue. No DLX is declared; exhausted messages are dropped. Confirmed mandatory-unroutable replies use the same discard class (NACK without requeue) with structured `ReplyUnroutableTerminal` logging — not a dead-letter | | `ChannelPoolSize` | int | `512` | **yes** | Validated; reserved for later consumer buffering (`1–8192`) | | `MinConnections` | int | `4` | **yes** | Validated; reserved (`1–512`; ≤ `MaxConnections`) | | `MaxConnections` | int | `16` | **yes** | Validated; reserved (`1–512`) | diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs index 9830ff08..97639748 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs @@ -1,4 +1,5 @@ +using VectorNNTPD.BackFillerService.Nntp; using VectorNNTPD.Common.Messaging.RabbitMq; using VectorNNTPD.Common.Core; @@ -38,6 +39,16 @@ internal sealed class ArticleWorkDeliveryPipeline /// internal bool TreatMissingRedeliveryCountAsExhausted => _treatMissingRedeliveryCountAsExhausted; + /// + /// Error text prepared for the most recent UnexpectedFailure caused by an escaped handler exception. + /// + /// + /// Cleared at the start of each call. When disposition publishes a terminal + /// reply, this is the error value passed to + /// publication. Formatted by . + /// + internal string? LastEscapedHandlerFailureError { get; private set; } + /// /// Initializes a new pipeline. /// @@ -92,8 +103,12 @@ internal ArticleWorkDeliveryPipeline( /// /// The classification for the delivery. Diagnostic parse failures are returned /// through the optional preParsed value and are not reparsed by diagnostic code. Returns - /// when a required publication is cancelled or fails; those paths - /// NACK requeue and do not ACK. A handler result of is treated as + /// when a required publication is cancelled or fails with a + /// transient or ambiguous error; those paths NACK-requeue (while the delivery limit allows) and do not ACK. + /// A confirmed mandatory unroutable reply () settles + /// NACK without requeue so a permanently missing ReplyTo cannot re-fetch indefinitely. Successful article + /// retrieval is not the same as successful RPC delivery in that case. + /// A handler result of is treated as /// . When /// is , Success is NACK-requeued without publication, and this method still returns /// . @@ -109,7 +124,11 @@ internal ArticleWorkDeliveryPipeline( /// is cancelled or fails NACK-requeues and returns , including when the planned /// disposition was ACK. The lease still attempts settlement once; a failed NACK is not retried on that lease. Any /// is disposed before settlement. Other handler exceptions become - /// and the error text is the exception type name. + /// ; the prepared error text is + /// (type and message when present). Handler + /// (and ProviderFailure / RetentionRejected) settle ACK with + /// a correlated terminal RPC reply under the current disposition map; that is distinct from a failed reply + /// publish, which still NACK-requeues via . /// SECURITY: The diagnostic reason is captured from the parse failure and returned directly. This ensures the configured payload /// limit is never bypassed by diagnostic code that might reparse the body with an unbounded limit. /// @@ -123,6 +142,7 @@ internal async Task ProcessAsync( { ArgumentException.ThrowIfNullOrWhiteSpace(consumingBackbone); ArgumentNullException.ThrowIfNull(channel); + LastEscapedHandlerFailureError = null; ArgumentNullException.ThrowIfNull(channelStillCurrent); var parsed = preParsed ?? ArticleWorkRequestParser.Parse(delivery, consumingBackbone, _maxPayloadBytes); @@ -213,7 +233,8 @@ internal async Task ProcessAsync( catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { outcome = ArticleWorkOutcome.UnexpectedFailure; - error = ex.GetType().Name; + error = NntpExceptionDiagnostics.FormatReason(ex); + LastEscapedHandlerFailureError = error; } var disposition = ArticleWorkDispositionPlanner.Create( @@ -297,20 +318,28 @@ await lease.TrySettleAsync( .ConfigureAwait(false); } - /// Builds the retry-or-discard disposition for the current delivery count. - private ArticleWorkDisposition CreateRetryDisposition(int? deliveryCount, bool redelivered) => - ArticleWorkDispositionPlanner.Create( - ArticleWorkOutcome.UnexpectedFailure, - replyable: false, - cancellationRequested: false, + /// + /// Builds NACK requeue/discard for publish-path retries (not handler ). + /// + /// + /// Must not use the planner's terminal UnexpectedFailure arm (ACK + reply). A failed or incomplete + /// reply publish must leave the delivery eligible for requeue while retries remain. + /// + private ArticleWorkDisposition CreateRetryDisposition(int? deliveryCount, bool redelivered) + { + var effectiveCount = ArticleWorkDispositionPlanner.ResolveEffectiveDeliveryCount( deliveryCount, redelivered, _deliveryLimit, _treatMissingRedeliveryCountAsExhausted); + return new ArticleWorkDisposition( + Acknowledge: false, + Requeue: effectiveCount < _deliveryLimit, + PublishResponse: false); + } /// - /// Settles after a publication attempt. A required publication that is cancelled or fails NACK-requeues - /// while retries remain. + /// Settles after a publication attempt. /// /// Result of the publication attempt. includes attempts that were not required. /// Plan produced before publication. Submitted only when is confirmed. @@ -322,11 +351,13 @@ private ArticleWorkDisposition CreateRetryDisposition(int? deliveryCount, bool r /// AMQP redelivered flag for missing-count handling. /// /// when the original disposition is submitted. - /// when publication was cancelled or failed. + /// when publication was cancelled, failed transiently, or was permanently unroutable. /// /// /// Broker RPCs use . A lost publisher confirm is reported as failure, so the /// requeue can publish another reply with the same correlation id. That is not exactly-once delivery. + /// A confirmed mandatory unroutable return is terminal (NACK without requeue): the ReplyTo queue is gone and + /// refetching cannot repair RPC delivery. Article retrieval may already have succeeded; that is not RPC delivery. /// private async Task CompleteAfterPublishAttemptAsync( ArticleWorkPublishAttempt published, @@ -338,6 +369,17 @@ private async Task CompleteAfterPublishAttemptAsync( int? deliveryCount, bool redelivered) { + if (published == ArticleWorkPublishAttempt.Unroutable) + { + var terminal = new ArticleWorkDisposition(Acknowledge: false, Requeue: false, PublishResponse: false); + await lease.TrySettleAsync( + terminal, + channelStillCurrent() && lease.IsOriginalChannel(channel), + CancellationToken.None) + .ConfigureAwait(false); + return ArticleWorkOutcome.UnexpectedFailure; + } + if (published != ArticleWorkPublishAttempt.Confirmed) { await SettleRetryableAsync(lease, channel, channelStillCurrent, deliveryCount, redelivered) @@ -371,6 +413,7 @@ await lease.TrySettleAsync( /// /// when publication is not required or completes. /// when it throws . + /// when it throws . /// when it throws any other exception. Exceptions are not propagated. /// private async Task TryPublishIfRequiredAsync( @@ -394,6 +437,10 @@ private async Task TryPublishIfRequiredAsync( try { + // Terminal failure replies require a non-empty error; Event 5321 may omit reason text. + var wireError = outcome == ArticleWorkOutcome.Success + ? error + : (string.IsNullOrWhiteSpace(error) ? outcome.ToString() : error); await _publisher.PublishAsync( new ArticleWorkResponseIntent( outcome, @@ -402,7 +449,7 @@ await _publisher.PublishAsync( backbone, correlationId, replyTo, - error, + wireError, outcome == ArticleWorkOutcome.Success ? fqdn : null, outcome == ArticleWorkOutcome.Success ? listenerPort : null, outcome == ArticleWorkOutcome.Success ? articleIdHex : null), @@ -414,6 +461,10 @@ await _publisher.PublishAsync( { return ArticleWorkPublishAttempt.Cancelled; } + catch (RabbitMqPublicationUnroutableException) + { + return ArticleWorkPublishAttempt.Unroutable; + } // codeql[cs/catch-of-all-exceptions] A delivery failure must be settled or logged by this pipeline boundary. catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { @@ -430,7 +481,15 @@ private enum ArticleWorkPublishAttempt /// The publisher threw . Cancelled, - /// The publisher threw an exception other than . + /// + /// The publisher threw (mandatory basic.return). + /// + Unroutable, + + /// + /// The publisher threw an exception other than cancellation or confirmed unroutable + /// (transient or ambiguous failure). + /// Failed, } } diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs index 6b6a6f81..f041f817 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs @@ -102,18 +102,21 @@ internal static ArticleWorkDisposition Create( return outcome switch { ArticleWorkOutcome.Success => new ArticleWorkDisposition(Acknowledge: true, Requeue: false, PublishResponse: true), - // Definitive absence / unusable article: processing completed successfully. - // ACK removes the message; NNTPD may publish a NEW request to another backbone. + // Definitive provider miss (430 or ARTICLE 451): ACK; do not requeue. + // Callers must treat NotFound as lookup-terminal (no cross-backbone republish). ArticleWorkOutcome.ArticleNotFound => new ArticleWorkDisposition(Acknowledge: true, Requeue: false, PublishResponse: true), + // Unusable article on this backbone: ACK + publish. Callers may try another backbone. ArticleWorkOutcome.InvalidArticle => new ArticleWorkDisposition(Acknowledge: true, Requeue: false, PublishResponse: true), ArticleWorkOutcome.InvalidRequest => new ArticleWorkDisposition(Acknowledge: true, Requeue: false, PublishResponse: replyable), - // Known retryable outcomes. The same settlement as an unrecognized value, and listed as a - // later change to the discard arm cannot retarget them. + // Provider/operation failures are terminal for this delivery: ACK + correlated failure reply. + // NACK-requeue without a reply caused unbounded classic-queue redelivery and client timeouts + // that published replacement work for an already-finished attempt. ArticleWorkOutcome.ProviderFailure - or ArticleWorkOutcome.Cancelled or ArticleWorkOutcome.UnexpectedFailure or ArticleWorkOutcome.RetentionRejected => - CreateRetryable(retriesExhausted), + new ArticleWorkDisposition(Acknowledge: true, Requeue: false, PublishResponse: true), + // Cooperative cancel / shutdown: requeue while delivery-limit retries remain so work can resume. + ArticleWorkOutcome.Cancelled => CreateRetryable(retriesExhausted), _ => CreateRetryable(retriesExhausted), }; } diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs index 443aa6d4..498e57d7 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkLogMessages.cs @@ -257,6 +257,41 @@ internal static partial void PublicationFailed( string outcome, string reason); + /// + /// Writes a Warning when a mandatory Success/terminal reply is returned as unroutable. + /// + /// + /// Article retrieval may already have succeeded. This event means RPC delivery to ReplyTo failed permanently; + /// the delivery pipeline settles NACK without requeue so the work is not re-fetched indefinitely. + /// No dead-letter exchange is declared on Article Work queues, so this settlement discards the broker message. + /// Structured fields (request id, Message-ID, backbone, correlation, ReplyTo, outcome) are the durable + /// diagnostic and operator-recovery trail for that intentional discard — not a claim that the broker + /// dead-lettered the delivery. + /// + /// Logger that receives the event. + /// Publish-channel generation that attempted the reply. + /// Request id in D format, or (none). + /// Work Message-ID when known, or (none). + /// Provider backbone label when known, or (none). + /// AMQP correlation id from the original work. + /// AMQP ReplyTo queue name that was missing. + /// Handler outcome that required the reply (typically Success). + /// Exception message from the unroutable classification. + [LoggerMessage( + EventId = 5330, + Level = LogLevel.Warning, + Message = "Article Work reply permanently unroutable generation={Generation} requestId={RequestId} messageId={MessageId} backbone={Backbone} correlationId={CorrelationId} replyTo={ReplyTo} outcome={Outcome}: {Reason}. Article may have been retrieved; RPC response was not delivered. Work will not be requeued for refetch; broker message is discarded without a dead-letter exchange.")] + internal static partial void ReplyUnroutableTerminal( + ILogger logger, + long generation, + string requestId, + string messageId, + string backbone, + string correlationId, + string replyTo, + string outcome, + string reason); + /// /// Writes a Debug event after the broker confirms a response, and the publication generation and channel are still current. /// @@ -338,8 +373,18 @@ internal static partial void ProviderTopologyDeclareFailed( /// Logger that receives the event. /// Request Message-ID, or (none) when it was not resolved. /// Request backbone, or (none) when it was not resolved. + /// + /// as a "D" GUID string, + /// or empty when no physical session was leased for this outcome. + /// /// Found, NotFound, ValidationFailed, or Failed. - /// Validation failure reason. Empty when is not ValidationFailed. + /// + /// Validation failure reason, or provider/retrieval diagnostic for Failed. Empty for Found + /// and typically empty for NotFound. + /// + /// + /// NNTP status as decimal text when the server produced one; empty otherwise. + /// /// /// Elapsed seconds from the ARTICLE send. Zero when ARTICLE was not sent. /// The message renders this value with three fractional digits (0.000). @@ -347,13 +392,15 @@ internal static partial void ProviderTopologyDeclareFailed( [LoggerMessage( EventId = 5321, Level = LogLevel.Information, - Message = "Article processed messageId={MessageId} backbone={Backbone} outcome={Outcome} reason={Reason} elapsed={Elapsed:0.000} s")] + Message = "Article processed messageId={MessageId} backbone={Backbone} connectionId={ConnectionId} outcome={Outcome} reason={Reason} statusCode={StatusCode} elapsed={Elapsed:0.000} s")] internal static partial void ArticleProcessed( ILogger logger, string messageId, string backbone, + string connectionId, string outcome, string reason, + string statusCode, double elapsed); /// diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs index 43511665..d42d79bb 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponsePublisher.cs @@ -360,6 +360,22 @@ await channel.PublishConfirmedAsync( outcome); } } + catch (RabbitMqPublicationUnroutableException ex) + { + // Permanent missing ReplyTo: article may already be retrieved; RPC delivery failed. + // NACK without requeue discards the work message (no DLX). Log identity fields for recovery. + ArticleWorkLogMessages.ReplyUnroutableTerminal( + _logger, + generation, + intent.RequestId?.ToString("D") ?? "(none)", + string.IsNullOrWhiteSpace(intent.MessageId) ? "(none)" : intent.MessageId, + string.IsNullOrWhiteSpace(intent.Backbone) ? "(none)" : intent.Backbone, + intent.CorrelationId, + intent.ReplyTo, + intent.Outcome.ToString(), + ex.Message); + throw; + } catch (Exception ex) when (ex is not OperationCanceledException) { ArticleWorkLogMessages.PublicationFailed( diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponseWireProtocol.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponseWireProtocol.cs index c34c729a..66d77352 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponseWireProtocol.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkResponseWireProtocol.cs @@ -22,9 +22,17 @@ internal static class ArticleWorkResponseWireProtocol private const string RequestIdHeaderName = "RequestId"; /// - /// Worker response TTL in milliseconds. Locked NNTPD architecture uses the same value - /// on requests and accepted responses. Old BackFiller publisher did not set Expiration. + /// AMQP per-message TTL in milliseconds for Article Work responses only. /// + /// + /// Applied when BackFiller publishes a terminal reply to the caller's exclusive ReplyTo queue. + /// Leftover replies should not linger if the caller has already abandoned the correlation. + /// This is RabbitMQ message TTL on the reply path, not a processing deadline for a work + /// delivery already handed to a consumer, and not a TTL for work requests waiting + /// in backfiller.{backbone}. Work requests must remain eligible for broker buffering + /// until a consumer delivers them; they must not carry this expiration. Queue declarations + /// must not set x-message-ttl that silently drops ready work. + /// internal const string ExpirationMilliseconds = "1000"; /// Serializes one validated v1 response. Does not embed article bytes. @@ -36,9 +44,8 @@ internal static class ArticleWorkResponseWireProtocol /// Thrown when is null. /// /// Thrown when is not a publishable v1 response. - /// , , - /// , and - /// are rejected. Success requires a canonical FQDN from , + /// is rejected (cancel settles by NACK-requeue without a terminal reply). + /// Success requires a canonical FQDN from , /// a port accepted by , and a 64-character lowercase /// hexadecimal article id, and must not include error. Other publishable outcomes require a /// non-empty error and must not include success fields. @@ -105,7 +112,7 @@ internal static byte[] SerializeV1(ArticleWorkResponseIntent intent) /// Returns the protocol outcome name written to the outcome property. /// Classified outcome to encode. /// - /// Success, ArticleNotFound, InvalidArticle, or InvalidRequest. + /// Wire outcome name for a publishable terminal response. /// /// /// Thrown when is not a publishable terminal response. @@ -117,6 +124,9 @@ internal static string OutcomeName(ArticleWorkOutcome outcome) => ArticleWorkOutcome.ArticleNotFound => "ArticleNotFound", ArticleWorkOutcome.InvalidArticle => "InvalidArticle", ArticleWorkOutcome.InvalidRequest => "InvalidRequest", + ArticleWorkOutcome.ProviderFailure => "ProviderFailure", + ArticleWorkOutcome.UnexpectedFailure => "UnexpectedFailure", + ArticleWorkOutcome.RetentionRejected => "RetentionRejected", _ => throw new InvalidOperationException($"Outcome '{outcome}' is not a publishable terminal response."), }; @@ -162,6 +172,9 @@ private static void Validate(ArticleWorkResponseIntent intent) case ArticleWorkOutcome.ArticleNotFound: case ArticleWorkOutcome.InvalidArticle: + case ArticleWorkOutcome.ProviderFailure: + case ArticleWorkOutcome.UnexpectedFailure: + case ArticleWorkOutcome.RetentionRejected: RequireIdentity(intent); if (intent.Fqdn is not null || intent.ListenerPort is not null) { diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ProviderArticleWorkHandler.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ProviderArticleWorkHandler.cs index bfcdbe06..969fbfa9 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ProviderArticleWorkHandler.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ProviderArticleWorkHandler.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Diagnostics; +using System.Globalization; using Microsoft.Extensions.Logging.Abstractions; using VectorNNTPD.BackFillerService.Configuration; using VectorNNTPD.BackFillerService.Nntp; @@ -276,7 +277,13 @@ public async ValueTask HandleAsync( if (cancellationToken.IsCancellationRequested) { LastKind = ArticleRetrievalKind.Cancelled; - LogArticleProcessed(item, OutcomeFailed, string.Empty, commandStartedTimestamp: 0); + LogArticleProcessed( + item, + OutcomeFailed, + string.Empty, + statusCode: null, + connectionId: Guid.Empty, + commandStartedTimestamp: 0); return new ArticleWorkHandlerResult(ArticleWorkOutcome.Cancelled, null); } @@ -326,7 +333,13 @@ public async ValueTask HandleAsync( } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { - LogArticleProcessed(item, OutcomeFailed, string.Empty, CommandTimestamp(ex)); + LogArticleProcessed( + item, + OutcomeFailed, + NntpExceptionDiagnostics.FormatReason(ex), + statusCode: null, + ConnectionIdFromException(ex), + CommandTimestamp(ex)); if (ex is OperationCanceledException && cancellationToken.IsCancellationRequested) { LastKind = ArticleRetrievalKind.Cancelled; @@ -343,14 +356,26 @@ public async ValueTask HandleAsync( { var mapped = MapRetrieval(retrieval); var (outcome, reason) = RetrievalLog(retrieval); - LogArticleProcessed(item, outcome, reason, retrieval.CommandStartedTimestamp); + LogArticleProcessed( + item, + outcome, + reason, + retrieval.StatusCode, + retrieval.ConnectionId, + retrieval.CommandStartedTimestamp); return mapped; } if (!consumed) { LastRetentionKind = ArticleRetentionKind.InvalidPayload; - LogArticleProcessed(item, OutcomeFailed, string.Empty, retrieval.CommandStartedTimestamp); + LogArticleProcessed( + item, + OutcomeFailed, + "Retrieved article payload could not be transferred into retention.", + retrieval.StatusCode, + retrieval.ConnectionId, + retrieval.CommandStartedTimestamp); return new ArticleWorkHandlerResult( ArticleWorkOutcome.RetentionRejected, "Retrieved article payload could not be transferred into retention."); @@ -367,6 +392,8 @@ public async ValueTask HandleAsync( item, OutcomeValidationFailed, rejected.Error ?? string.Empty, + retrieval.StatusCode, + retrieval.ConnectionId, retrieval.CommandStartedTimestamp); return rejected; } @@ -378,6 +405,8 @@ public async ValueTask HandleAsync( item, OutcomeValidationFailed, "MessageIdMismatch", + retrieval.StatusCode, + retrieval.ConnectionId, retrieval.CommandStartedTimestamp); return new ArticleWorkHandlerResult( ArticleWorkOutcome.InvalidArticle, @@ -390,13 +419,21 @@ public async ValueTask HandleAsync( item, OutcomeValidationFailed, "NotFrameable", + retrieval.StatusCode, + retrieval.ConnectionId, retrieval.CommandStartedTimestamp); return new ArticleWorkHandlerResult( ArticleWorkOutcome.InvalidArticle, "NotFrameable"); } - LogArticleProcessed(item, OutcomeFound, string.Empty, retrieval.CommandStartedTimestamp); + LogArticleProcessed( + item, + OutcomeFound, + string.Empty, + retrieval.StatusCode, + retrieval.ConnectionId, + retrieval.CommandStartedTimestamp); if (!System.Runtime.InteropServices.MemoryMarshal.TryGetArray(record.ArtData, out var segment) || segment.Array is null @@ -452,23 +489,37 @@ public async ValueTask HandleAsync( /// /// Work item whose Message-ID and backbone are logged. /// Terminal processing outcome. - /// Validation failure reason. Empty for every other outcome. + /// Validation or provider failure text. Empty for successful Found. + /// NNTP status when present. + /// Physical session identity, or when none was leased. /// ARTICLE send timestamp. Zero when the command was not sent. private void LogArticleProcessed( ArticleWorkItem item, string outcome, string reason, + int? statusCode, + Guid connectionId, long commandStartedTimestamp) { ArticleWorkLogMessages.ArticleProcessed( _logger, DisplayIdentity(item.Request.MessageId), DisplayIdentity(item.Request.Backbone), + FormatConnectionId(connectionId), outcome, reason, + statusCode is { } code ? code.ToString(CultureInfo.InvariantCulture) : string.Empty, ElapsedSeconds(commandStartedTimestamp)); } + /// Formats for Event 5321, or empty when none was leased. + private static string FormatConnectionId(Guid connectionId) => + connectionId == Guid.Empty ? string.Empty : connectionId.ToString("D"); + + /// Reads from . + private static Guid ConnectionIdFromException(Exception exception) => + exception.Data[ArticleRetrievalResult.ConnectionIdKey] is Guid id ? id : Guid.Empty; + /// /// Returns the preparation for , rebuilding it when that snapshot's /// article limit, site name, or DNS suffix differs from the published preparation. @@ -504,11 +555,11 @@ private SharedArticlePreparation ResolveSharedArticlePreparation() } } - /// Maps a non-retrieved download to the article-processing outcome and validation reason. + /// Maps a non-retrieved download to the article-processing outcome and log reason. /// Download that is not . /// /// NotFound for a miss, ValidationFailed with for an invalid article, - /// and Failed for every other kind. + /// and Failed with the retrieval diagnostic for every other kind. /// private static (string Outcome, string Reason) RetrievalLog(ArticleRetrievalResult retrieval) { @@ -516,7 +567,7 @@ private static (string Outcome, string Reason) RetrievalLog(ArticleRetrievalResu { ArticleRetrievalKind.ArticleNotFound => (OutcomeNotFound, string.Empty), ArticleRetrievalKind.InvalidArticle => (OutcomeValidationFailed, retrieval.Reason), - _ => (OutcomeFailed, string.Empty), + _ => (OutcomeFailed, retrieval.Reason), }; } diff --git a/src/VectorNNTPD.BackFillerService/Nntp/ArticleRetrievalKind.cs b/src/VectorNNTPD.BackFillerService/Nntp/ArticleRetrievalKind.cs index becb8093..b83ed7da 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/ArticleRetrievalKind.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/ArticleRetrievalKind.cs @@ -6,7 +6,7 @@ internal enum ArticleRetrievalKind /// ARTICLE succeeded and produced an owned destuffed payload. ArticleRetrieved = 0, - /// Provider reported that the article does not exist (430). + /// Provider reported that the article does not exist or is unavailable (430 or ARTICLE 451). ArticleNotFound = 1, /// diff --git a/src/VectorNNTPD.BackFillerService/Nntp/ArticleRetrievalResult.cs b/src/VectorNNTPD.BackFillerService/Nntp/ArticleRetrievalResult.cs index 930ebbe4..293610aa 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/ArticleRetrievalResult.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/ArticleRetrievalResult.cs @@ -27,6 +27,21 @@ internal sealed record ArticleRetrievalResult( /// Zero when the command was not sent. /// internal long CommandStartedTimestamp { get; set; } + + /// + /// Stable identity of the physical NNTP connection that performed this retrieval. + /// + /// + /// when no session was leased (cancelled before acquire, missing pool, or connect failure + /// before a session existed). Otherwise equals for that lease. + /// + internal Guid ConnectionId { get; set; } + + /// + /// Exception.Data key for when download throws after a session was leased. + /// + internal const string ConnectionIdKey = "VectorNNTPD.BackFillerService.NntpConnectionId"; + /// Disposes the owned when one is present. public void Dispose() { diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpArticleRetriever.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpArticleRetriever.cs index 0c46625f..19c97d0e 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpArticleRetriever.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpArticleRetriever.cs @@ -132,23 +132,29 @@ public async Task RetrieveAsync( } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { - RememberCommandTimestamp(ex, lease.Session.ArticleCommandStartedTimestamp); + RememberLeaseCorrelation(ex, lease.Session); throw; } } } /// - /// Copies a non-zero ARTICLE send timestamp onto when download did not return a result. + /// Copies ARTICLE send timestamp and onto + /// when download did not return a result. /// /// Exception propagating from . - /// Session timestamp. Zero means ARTICLE was not sent. - private static void RememberCommandTimestamp(Exception exception, long commandStartedTimestamp) + /// Leased session that owned the failed download. + private static void RememberLeaseCorrelation(Exception exception, NntpProviderSession session) { - if (commandStartedTimestamp != 0 + if (session.ArticleCommandStartedTimestamp != 0 && exception.Data[ArticleRetrievalResult.CommandStartedTimestampKey] is not long) { - exception.Data[ArticleRetrievalResult.CommandStartedTimestampKey] = commandStartedTimestamp; + exception.Data[ArticleRetrievalResult.CommandStartedTimestampKey] = session.ArticleCommandStartedTimestamp; + } + + if (exception.Data[ArticleRetrievalResult.ConnectionIdKey] is not Guid) + { + exception.Data[ArticleRetrievalResult.ConnectionIdKey] = session.ConnectionId; } } } diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpExceptionDiagnostics.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpExceptionDiagnostics.cs new file mode 100644 index 00000000..b71b5cc1 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpExceptionDiagnostics.cs @@ -0,0 +1,137 @@ +using System.Text; + +namespace VectorNNTPD.BackFillerService.Nntp +{ + /// + /// Formats exception text for , UnexpectedFailure pipeline errors, + /// and related failure diagnostics. + /// + /// + /// Reasons are operator-facing and must not embed credentials or article bodies. + /// Callers supply only transport/protocol or handler exceptions whose messages are already scrubbed of secrets. + /// Non-duplicate inner-exception type/message pairs may be appended after <- when they add context + /// (for example a blank outer message with a useful inner). + /// Typical wrappers already embed the socket text in ; + /// those inners are skipped to avoid repetition. + /// + internal static class NntpExceptionDiagnostics + { + /// Maximum characters in the returned reason, including type names and any truncation marker. + private const int MaxReasonLength = 256; + + /// Maximum inner exceptions considered after the outer exception. + private const int MaxInnerDepth = 3; + + /// Marker appended when the formatted reason is truncated to . + private const string TruncationMarker = "..."; + + /// + /// Builds a non-empty diagnostic reason from an exception type, message, and useful inner context. + /// + /// Caught transport, protocol, or handler exception. + /// + /// {TypeName}, {TypeName}: {message}, and optionally + /// <- {InnerType}: {inner message} segments when an inner exception adds non-duplicate text. + /// The full string is at most characters; longer text ends with .... + /// + internal static string FormatReason(Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + + var builder = new StringBuilder(capacity: 128); + AppendSegment(builder, exception); + + var seen = new HashSet(ReferenceEqualityComparer.Instance) { exception }; + var inner = exception.InnerException; + for (var depth = 0; inner is not null && depth < MaxInnerDepth; depth++) + { + if (!seen.Add(inner)) + { + break; + } + + if (ShouldAppendInner(builder, inner)) + { + builder.Append(" <- "); + AppendSegment(builder, inner); + } + + inner = inner.InnerException; + } + + return Truncate(builder.ToString()); + } + + /// Appends {Type} or {Type}: {trimmed message} for one exception. + private static void AppendSegment(StringBuilder builder, Exception exception) + { + builder.Append(exception.GetType().Name); + if (string.IsNullOrWhiteSpace(exception.Message)) + { + return; + } + + builder.Append(": "); + builder.Append(exception.Message.Trim()); + } + + /// + /// Returns whether adds type/message text not already present in . + /// + private static bool ShouldAppendInner(StringBuilder builder, Exception inner) + { + if (!string.IsNullOrWhiteSpace(inner.Message)) + { + var message = inner.Message.Trim(); + // Typical IOException wrappers already embed the socket/TLS detail in Message. + if (ContainsOrdinal(builder, message)) + { + return false; + } + + return true; + } + + // Blank inner message: only the type name could add information. + return !ContainsOrdinal(builder, inner.GetType().Name); + } + + /// Ordinal substring search over the current builder contents. + private static bool ContainsOrdinal(StringBuilder builder, string value) + { + if (value.Length == 0 || builder.Length < value.Length) + { + return false; + } + + return builder.ToString().Contains(value, StringComparison.Ordinal); + } + + /// Caps the reason at , appending when cut. + private static string Truncate(string reason) + { + if (reason.Length <= MaxReasonLength) + { + return reason; + } + + var keep = MaxReasonLength - TruncationMarker.Length; + if (keep <= 0) + { + return TruncationMarker[..MaxReasonLength]; + } + + return string.Concat(reason.AsSpan(0, keep), TruncationMarker); + } + + /// Reference equality for exception cycle detection along chains. + private sealed class ReferenceEqualityComparer : IEqualityComparer + { + internal static readonly ReferenceEqualityComparer Instance = new(); + + public bool Equals(Exception? x, Exception? y) => ReferenceEquals(x, y); + + public int GetHashCode(Exception obj) => System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj); + } + } +} diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpLogMessages.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpLogMessages.cs index 78859e61..24956eac 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpLogMessages.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpLogMessages.cs @@ -9,6 +9,7 @@ internal static partial class NntpLogMessages /// /// Logger that receives the information event. /// Provider backbone label. + /// of the ready physical connection. /// Upstream host from the provider definition. /// Upstream port from the provider definition. /// @@ -18,8 +19,14 @@ internal static partial class NntpLogMessages [LoggerMessage( EventId = 5400, Level = LogLevel.Information, - Message = "NNTP session ready backbone={Backbone} host={Host} port={Port} tls={UseTls}")] - internal static partial void SessionReady(ILogger logger, string Backbone, string Host, int Port, bool UseTls); + Message = "NNTP session ready backbone={Backbone} connectionId={ConnectionId} host={Host} port={Port} tls={UseTls}")] + internal static partial void SessionReady( + ILogger logger, + string Backbone, + Guid ConnectionId, + string Host, + int Port, + bool UseTls); /// /// Written when a live pooled session is removed for retirement, before that session is disposed. @@ -27,12 +34,19 @@ internal static partial class NntpLogMessages /// /// Logger that receives the warning. /// Backbone of the pool that owned the session. - /// Caller-supplied retirement text. This method does not throw. + /// of the retired physical connection. + /// Structured cause from . + /// Caller-supplied human-readable retirement text. This method does not throw. [LoggerMessage( EventId = 5401, Level = LogLevel.Warning, - Message = "NNTP session retired backbone={Backbone} reason={Reason}")] - internal static partial void SessionRetired(ILogger logger, string Backbone, string Reason); + Message = "NNTP session retired backbone={Backbone} connectionId={ConnectionId} cause={Cause} reason={Reason}")] + internal static partial void SessionRetired( + ILogger logger, + string Backbone, + Guid ConnectionId, + string Cause, + string Reason); /// /// Defined for a non-success download. The article retriever does not call it. diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpProviderSession.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpProviderSession.cs index c46b70da..da355f43 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpProviderSession.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpProviderSession.cs @@ -142,13 +142,26 @@ internal NntpProviderSession( _options = options; _logger = logger; ConnectionNumber = connectionNumber; + ConnectionId = Guid.NewGuid(); _wireIdentity = FormatWireIdentity(provider, connectionNumber); State = NntpSessionState.Created; } /// Gets the one-based pool slot captured at construction. + /// + /// Pool slot numbers are reused after retirement. Use to correlate one physical TCP session. + /// internal int ConnectionNumber { get; } + /// + /// Gets the process-unique identity assigned when this physical NNTP session object is constructed. + /// + /// + /// Stable for the lifetime of this connection. Distinct from (reusable pool slot), + /// Article Work request ids, Message-IDs, and RabbitMQ delivery tags. + /// + internal Guid ConnectionId { get; } + /// /// Gets the stable Debug wire-log prefix /// {Backbone}/{Account}[{ConnectionNumber:000}/{MaxSessions}]. @@ -315,7 +328,13 @@ internal Func? TlsStreamFactory } AssignState(NntpSessionState.Ready); - NntpLogMessages.SessionReady(_logger, _provider.Backbone, _provider.Host, _provider.Port, _provider.UseTls); + NntpLogMessages.SessionReady( + _logger, + _provider.Backbone, + ConnectionId, + _provider.Host, + _provider.Port, + _provider.UseTls); return null; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -328,7 +347,7 @@ internal Func? TlsStreamFactory } catch (Exception ex) when (ex is SocketException or IOException or ObjectDisposedException or InvalidOperationException or AuthenticationException or FormatException or InvalidDataException or TimeoutException) { - return FailClosed(ArticleRetrievalKind.ProviderFailure, null, ex.GetType().Name, reusable: false); + return FailClosed(ArticleRetrievalKind.ProviderFailure, null, NntpExceptionDiagnostics.FormatReason(ex), reusable: false); } } @@ -353,10 +372,16 @@ internal Func? TlsStreamFactory /// Status returns the destuffed payload when is true. /// With null, that payload is an owned . /// With not null, the result's article is null because the delegate already consumed the scratch. - /// Status is and the session stays reusable. + /// Status or + /// is and the session stays reusable. /// A 220 payload with no header/body separator is and stays reusable. /// A payload that reaches is and the session is marked unhealthy because the multiline response is no longer synchronized. - /// , , statuses 412, 420, and 423, and every other status mark the session unhealthy. + /// Single-line ARTICLE failures classified by + /// (403, 500, 501, 503) are and stay reusable: the + /// response does not desynchronize the session (RFC 3977 §§3.2, 3.2.1). + /// , , + /// , statuses 412, 420, and 423, empty or malformed + /// status lines, multiline desynchronization, and transport failures mark the session unhealthy. /// /// is null or whitespace. /// @@ -559,7 +584,7 @@ await WriteCommandAsync( return WithCommandTimestamp(ArticleRetrievalResult.Retrieved(code, text)); } - if (code == NntpStatusCode.NoArticleWithMessageId) + if (code is NntpStatusCode.NoArticleWithMessageId or NntpStatusCode.ArticleUnavailable) { AssignState(NntpSessionState.Ready); return WithCommandTimestamp(ArticleRetrievalResult.Failed( @@ -569,15 +594,27 @@ await WriteCommandAsync( sessionReusable: true)); } + if (NntpStatusCode.IsArticleLevelFailureReusable(code)) + { + // Single-line negative ARTICLE status: protocol remains synchronized; fail this article only. + AssignState(NntpSessionState.Ready); + return WithCommandTimestamp(ArticleRetrievalResult.Failed( + ArticleRetrievalKind.ProviderFailure, + code, + text, + sessionReusable: true)); + } + if (NntpStatusCode.IsAuthenticationFailure(code)) { return MarkUnhealthy(ArticleRetrievalKind.AuthenticationFailure, code, text); } - if (NntpStatusCode.IsCommandRejected(code) - || code is NntpStatusCode.NoNewsgroupSelected + if (code is NntpStatusCode.NoNewsgroupSelected or NntpStatusCode.CurrentArticleInvalid - or NntpStatusCode.NoArticleWithNumber) + or NntpStatusCode.NoArticleWithNumber + or NntpStatusCode.CommandNotPermitted + or NntpStatusCode.ServiceTemporarilyUnavailable) { return MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, code, text); } @@ -598,7 +635,7 @@ or NntpStatusCode.CurrentArticleInvalid } catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException or AuthenticationException or FormatException or InvalidDataException or TimeoutException) { - return MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, null, ex.GetType().Name); + return MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, null, NntpExceptionDiagnostics.FormatReason(ex)); } finally { @@ -734,7 +771,7 @@ await WriteCommandAsync("DATE", NntpProtocolIo.DateCommand, cancellationToken) } catch (Exception ex) when (ex is IOException or ObjectDisposedException or InvalidOperationException or AuthenticationException or FormatException or InvalidDataException or TimeoutException) { - _ = MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, null, ex.GetType().Name); + _ = MarkUnhealthy(ArticleRetrievalKind.ProviderFailure, null, NntpExceptionDiagnostics.FormatReason(ex)); return false; } finally @@ -1168,14 +1205,14 @@ await WriteCommandAsync("STARTTLS", NntpProtocolIo.StartTlsCommand, cancellation return ArticleRetrievalResult.Failed( ArticleRetrievalKind.ProviderFailure, null, - $"TLS handshake failed ({ex.GetType().Name}).", + "TLS handshake failed: " + NntpExceptionDiagnostics.FormatReason(ex), sessionReusable: false); } return FailClosed( ArticleRetrievalKind.ProviderFailure, null, - $"TLS handshake failed ({ex.GetType().Name}).", + "TLS handshake failed: " + NntpExceptionDiagnostics.FormatReason(ex), reusable: false); } @@ -1584,27 +1621,34 @@ private ArticleRetrievalResult MarkUnhealthy(ArticleRetrievalKind kind, int? cod } /// - /// Copies onto . - /// Zero means ARTICLE was not sent. + /// Copies and onto . + /// Zero command timestamp means ARTICLE was not sent. /// /// Retrieval result about to be returned. - /// The same result, with the command timestamp stored. + /// The same result, with session correlation fields stored. private ArticleRetrievalResult WithCommandTimestamp(ArticleRetrievalResult result) { result.CommandStartedTimestamp = ArticleCommandStartedTimestamp; + result.ConnectionId = ConnectionId; return result; } /// - /// Stores on when ARTICLE was sent. + /// Stores and on + /// for handler correlation when download throws. /// - /// Exception that leaves after the command timestamp is known. + /// Exception that leaves after the session is known. private void RememberCommandTimestamp(Exception exception) { if (ArticleCommandStartedTimestamp != 0) { exception.Data[ArticleRetrievalResult.CommandStartedTimestampKey] = ArticleCommandStartedTimestamp; } + + if (exception.Data[ArticleRetrievalResult.ConnectionIdKey] is not Guid) + { + exception.Data[ArticleRetrievalResult.ConnectionIdKey] = ConnectionId; + } } /// diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionPool.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionPool.cs index c59e43f2..a4ac444c 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionPool.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionPool.cs @@ -144,7 +144,8 @@ internal sealed class NntpSessionPool : IAsyncDisposable /// /// Test gate invoked after a retired session has left the live set and before that session is disposed. - /// Null in production. The delegate runs outside and must not be required for accounting to stay committed. + /// Null in production. The delegate runs outside . + /// Created-count and connection-number capacity are still held while this gate runs; they are released only after dispose completes. /// private Func? _beforeDetachedSessionDispose; @@ -240,11 +241,25 @@ internal sealed class NntpSessionPool : IAsyncDisposable private readonly SemaphoreSlim _fillGate = new(1, 1); /// - /// Sessions counted against the current max. - /// Incremented before a connection number is reserved and decremented exactly once when that attempt is abandoned or the live session is retired. + /// Sessions counted against the current max, including sessions whose transport is still disposing after leaving . + /// Incremented before a connection number is reserved. + /// Decremented exactly once when an unregistered reservation is abandoned, or when a registered session has finished + /// after winning retirement. /// private int _created; + /// + /// Serializes replacement of so acquire waiters observe every capacity or idle-admission pulse. + /// Not held across await. + /// + private readonly Lock _capacityGate = new(); + + /// + /// Completes when created-count capacity is released after dispose or when a session is admitted to . + /// Replaced under before the prior source is completed so a waiter cannot miss the next pulse. + /// + private TaskCompletionSource _capacitySignal = new(TaskCreationOptions.RunContinuationsAsynchronously); + /// /// Leases whose issue slot has reserved and has not released. /// Transitions are made under together with . @@ -562,7 +577,7 @@ internal Func? BeforeKeepAliveReusabilityCheck /// /// Pauses retirement after the session has left and before . /// Null unless a test is holding that window. The delegate runs outside . - /// Accounting for the retired session is already committed. Replacement fill runs after the pause and the dispose. + /// Created-count and connection-number capacity remain reserved until dispose finishes; replacement create cannot use that slot while this pause holds. /// internal Func? BeforeDetachedSessionDispose { @@ -665,7 +680,8 @@ internal async Task EnsureDesiredSessionsAsync(CancellationToken cancellationTok { ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); StartPeriodicReplenish(); - var missing = Math.Max(0, _provider.MaxSessions - _live.Count); + // Use _created (includes still-disposing sessions), not _live, so fill does not open TCP above MaxSessions. + var missing = Math.Max(0, _provider.MaxSessions - Volatile.Read(ref _created)); if (missing == 0) { NotifyActiveSessionCountChanged(); @@ -687,7 +703,7 @@ internal async Task EnsureDesiredSessionsAsync(CancellationToken cancellationTok /// A lease that owns one ready session. Disposing the lease returns or retires it. /// The pool is disposed before the permit is taken or before a session is issued. /// The caller token or cancels the wait. - /// The created-session count is already at . + /// A created session was retired before it could be issued. /// /// The new session did not become . It is retired without replenish. /// @@ -721,6 +737,7 @@ internal async Task AcquireAsync(CancellationToken cancellatio { ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); var retryWithoutPermit = false; + var waitForCapacity = false; while (TryTakeIdleSession(out var keepalive) is { } idle) { taken = idle; @@ -744,7 +761,11 @@ internal async Task AcquireAsync(CancellationToken cancellatio break; } - await RetireSessionAsync(idle, "idle session was not reusable").ConfigureAwait(false); + await RetireSessionAsync( + idle, + NntpSessionRetirementCause.SessionNotReusable, + "idle session was not reusable") + .ConfigureAwait(false); taken = null; } @@ -774,42 +795,54 @@ internal async Task AcquireAsync(CancellationToken cancellatio break; } - await RetireSessionAsync(idle, "idle session was not reusable").ConfigureAwait(false); + await RetireSessionAsync( + idle, + NntpSessionRetirementCause.SessionNotReusable, + "idle session was not reusable") + .ConfigureAwait(false); taken = null; } if (!retryWithoutPermit) { + // Capacity includes sessions still disposing after leaving _live. Wait rather than + // opening another TCP (or throwing) while that reservation is held. if (Volatile.Read(ref _created) >= _provider.MaxSessions) { - throw new InvalidOperationException("NNTP session pool exceeded MaxSessions."); + waitForCapacity = true; } - - var created = await CreateReadySessionAsync(linked.Token(cancellationToken, _shutdown.Token)).ConfigureAwait(false); - taken = created; - if (_beforeCreatedSessionIssue is { } beforeIssue) + else { - await beforeIssue().ConfigureAwait(false); - } + var created = await CreateReadySessionAsync(linked.Token(cancellationToken, _shutdown.Token)).ConfigureAwait(false); + taken = created; + if (_beforeCreatedSessionIssue is { } beforeIssue) + { + await beforeIssue().ConfigureAwait(false); + } - var issued = await TryIssueHeldSessionAsync(created).ConfigureAwait(false); - if (issued.Lease is not null) - { - return issued.Lease; - } + var issued = await TryIssueHeldSessionAsync(created).ConfigureAwait(false); + if (issued.Lease is not null) + { + return issued.Lease; + } - if (issued.RejectedForCapacity) - { - permitHeld = false; - taken = null; - retryWithoutPermit = true; - } - else - { - await RetireSessionAsync(created, "created session was not reusable").ConfigureAwait(false); - taken = null; - ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); - throw new InvalidOperationException("NNTP session was retired before it could be issued."); + if (issued.RejectedForCapacity) + { + permitHeld = false; + taken = null; + retryWithoutPermit = true; + } + else + { + await RetireSessionAsync( + created, + NntpSessionRetirementCause.EstablishmentFailed, + "created session was not reusable") + .ConfigureAwait(false); + taken = null; + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) == 1, this); + throw new InvalidOperationException("NNTP session was retired before it could be issued."); + } } } } @@ -829,6 +862,13 @@ internal async Task AcquireAsync(CancellationToken cancellatio { continue; } + + if (waitForCapacity) + { + await WaitForCapacityChangedAsync(linked.Token(cancellationToken, _shutdown.Token)) + .ConfigureAwait(false); + continue; + } } catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex)) { @@ -840,7 +880,11 @@ internal async Task AcquireAsync(CancellationToken cancellatio try { // The DATE loop retires only a session that is still idle. This one was already removed. - await RetireSessionAsync(taken, "idle session was not reusable").ConfigureAwait(false); + await RetireSessionAsync( + taken, + NntpSessionRetirementCause.SessionNotReusable, + "idle session was not reusable") + .ConfigureAwait(false); } catch (Exception cleanup) when (IsExpectedSessionCleanupFailure(cleanup)) { @@ -912,7 +956,12 @@ internal async Task ReturnAsync(NntpProviderSession session, bool retire) { if (retire || Volatile.Read(ref _disposed) == 1 || _shutdown.IsCancellationRequested) { - await RetireSessionAsync(session, retire ? "lease requested retirement" : "pool is shutting down") + await RetireSessionAsync( + session, + retire + ? NntpSessionRetirementCause.LeaseRequested + : NntpSessionRetirementCause.PoolShuttingDown, + retire ? "lease requested retirement" : "pool is shutting down") .ConfigureAwait(false); } else @@ -925,13 +974,18 @@ await RetireSessionAsync(session, retire ? "lease requested retirement" : "pool case IdleAdmission.OverBound: await RetireSessionAsync( session, + NntpSessionRetirementCause.MaxSessionsShrink, "max sessions shrink", replenish: false, liveAlreadyDetached: true) .ConfigureAwait(false); break; default: - await RetireSessionAsync(session, "session is no longer reusable").ConfigureAwait(false); + await RetireSessionAsync( + session, + NntpSessionRetirementCause.SessionNotReusable, + "session is no longer reusable") + .ConfigureAwait(false); break; } } @@ -998,7 +1052,12 @@ internal async Task ShrinkToBoundAsync(CancellationToken cancellationToken) while (_live.Count > bound && TryTakeIdleSession(out var keepalive) is { } idle) { CancelKeepAliveRegistration(keepalive); - await RetireSessionAsync(idle, "max sessions shrink", replenish: false).ConfigureAwait(false); + await RetireSessionAsync( + idle, + NntpSessionRetirementCause.MaxSessionsShrink, + "max sessions shrink", + replenish: false) + .ConfigureAwait(false); } } finally @@ -1345,6 +1404,7 @@ private Task RetireRemainingSessionsAsync(NntpShutdownDeadline deadline, Cancell sessions, session => RetireSessionAsync( session, + NntpSessionRetirementCause.PoolShuttingDown, "pool dispose", replenish: false, deadline: deadline, @@ -1686,13 +1746,18 @@ private async Task RestoreUnissuedSessionAsync(NntpProviderSession session) case IdleAdmission.OverBound: await RetireSessionAsync( session, + NntpSessionRetirementCause.MaxSessionsShrink, "max sessions shrink", replenish: false, liveAlreadyDetached: true) .ConfigureAwait(false); break; default: - await RetireSessionAsync(session, "session is no longer reusable").ConfigureAwait(false); + await RetireSessionAsync( + session, + NntpSessionRetirementCause.SessionNotReusable, + "session is no longer reusable") + .ConfigureAwait(false); break; } } @@ -1781,7 +1846,8 @@ private async Task CreateReadySessionAsync(CancellationToke { NntpLogMessages.SessionEstablishmentCompleted(_logger, _provider.Backbone, connectionNumber); NotifyActiveSessionCountChanged(); - if (session.State == NntpSessionState.Ready && _live.ContainsKey(session)) + // State remains Ready on this thread; re-check live-set membership for concurrent retirement. + if (_live.ContainsKey(session)) { return session; } @@ -1837,7 +1903,7 @@ await AbandonFailedCreationAsync( throw new NntpProviderConnectException( ArticleRetrievalKind.ProviderFailure, null, - ex.GetType().Name); + NntpExceptionDiagnostics.FormatReason(ex)); } throw; @@ -1861,10 +1927,11 @@ await AbandonFailedCreationAsync( /// /// Retirement text used when is . /// - /// A registered session is removed through , which decrements - /// and frees the connection number only when this call wins TryRemove. + /// A registered session is removed through , which disposes the transport and then + /// decrements and frees the connection number only when this call wins TryRemove. /// A concurrent retire or dispose that already removed the session leaves the counters unchanged. - /// An unregistered reservation is released by before the session is disposed. + /// An unregistered reservation is released by before the session is disposed + /// (that path never opened a provider TCP). /// Expected disposal failures are logged and do not propagate. Replenishment is not requested. /// The periodic loop retries on its existing interval. /// @@ -1876,7 +1943,12 @@ private async Task AbandonFailedCreationAsync( { if (registered) { - await RetireSessionAsync(session, reason, replenish: false).ConfigureAwait(false); + await RetireSessionAsync( + session, + NntpSessionRetirementCause.EstablishmentFailed, + reason, + replenish: false) + .ConfigureAwait(false); return; } @@ -1894,7 +1966,8 @@ await DisposeDetachedSessionAsync( /// /// The connect-failure message when is . /// NNTP connect was cancelled when cancelled an . - /// Otherwise the exception type name, which matches failure reasons. + /// Otherwise the exception type name for unexpected escapes (session connect failures already use + /// on the path). /// private static string CreationFailureReason(Exception failure, CancellationToken cancellationToken) { @@ -2096,6 +2169,11 @@ private IdleAdmission TryAdmitIdle(NntpProviderSession session, bool enforceBoun } } + if (admission == IdleAdmission.Admitted) + { + NotifyCapacityChanged(); + } + return admission; } @@ -2352,6 +2430,7 @@ private async Task RetireFromKeepAliveAsync( await RetireSessionAsync( session, + NntpSessionRetirementCause.DateKeepaliveFailed, reason, onlyIfIdle: true, actingRegistration: registration) @@ -2409,10 +2488,11 @@ private async Task AwaitKeepAlivesAsync(NntpShutdownDeadline deadline, Cancellat } /// - /// Stops keepalive, frees the connection number, disposes , and optionally asks the pool to replace it. + /// Stops keepalive, disposes , frees the connection number after that dispose, and optionally asks the pool to replace it. /// /// Live session to drop. A session that is already absent is left alone. - /// Text passed to . + /// Structured cause from . + /// Human-readable text passed to . /// schedules after a session was actually removed. /// /// when the caller is the DATE loop. The session is disposed only if it is still in @@ -2436,18 +2516,21 @@ private async Task AwaitKeepAlivesAsync(NntpShutdownDeadline deadline, Cancellat /// for an decision. Skips the membership lock and finishes disposal only. /// /// - /// TryRemove wins the decrement and connection-number release. A second retire, dispose, or creation rollback - /// observes the loss and does not change the counters again. Idle membership is removed under - /// before those releases, and before the retirement log, the active-count notification, and disposal. A retired + /// TryRemove wins disposal and the later created-count / connection-number release. A second retire, dispose, + /// or creation rollback observes the loss and does not change the counters again. Idle membership is removed under + /// before the retirement log, the active-count notification, and disposal. A retired /// session therefore does not stay reachable from until a later . - /// An issued session stays in when is set, so - /// is the one retire or re-enqueue. Observer failures are contained by - /// so they cannot skip disposal, decrement twice, or recurse through another retire. Expected disposal failures - /// are logged. Any other disposal exception still requests replenishment when is - /// , then propagates. + /// Created-count and connection-number capacity stay reserved until returns + /// (including when disposal throws), so a replacement cannot open another provider TCP while this transport is still + /// closing. An issued session stays in when is set, so + /// is the one retire or re-enqueue. Observer failures are contained by + /// so they cannot skip disposal, decrement twice, or recurse through another retire. + /// Expected disposal failures are logged. Any other disposal exception still releases capacity exactly once and + /// requests replenishment when is , then propagates. /// private async Task RetireSessionAsync( NntpProviderSession session, + string cause, string reason, bool replenish = true, bool onlyIfIdle = false, @@ -2516,9 +2599,7 @@ private async Task RetireSessionAsync( } CancelKeepAliveRegistration(keepalive); - Interlocked.Decrement(ref _created); - ReleaseConnectionNumber(session.ConnectionNumber); - SignalSessionRetired(session, reason); + SignalSessionRetired(session, cause, reason); try { if (_beforeDetachedSessionDispose is { } beforeDispose) @@ -2535,6 +2616,9 @@ await DisposeDetachedSessionAsync(session, session.ConnectionNumber, deadline, c } finally { + Interlocked.Decrement(ref _created); + ReleaseConnectionNumber(session.ConnectionNumber); + NotifyCapacityChanged(); if (replenish) { RequestReplenish(); @@ -2547,22 +2631,29 @@ await DisposeDetachedSessionAsync(session, session.ConnectionNumber, deadline, c /// Logs retirement and notifies after the session has left . /// /// Session already removed from . Its connection number is used only in a diagnostic. - /// Text passed to . + /// Structured cause from . + /// Human-readable text passed to . /// /// Both calls are observers. Ordinary failures are contained so still disposes /// the session once and does not enter retirement again. Process-integrity failures propagate. - /// runs after this notification and before that dispose. A fault from the gate still disposes the session. + /// Created-count capacity is still held: and dispose run after this + /// notification, and the reservation is released only after dispose returns. A fault from the gate still disposes the session. /// A logger that fails is not asked to record its own failure. A subscriber failure is recorded when the logger still works. /// - private void SignalSessionRetired(NntpProviderSession session, string reason) + private void SignalSessionRetired(NntpProviderSession session, string cause, string reason) { try { - NntpLogMessages.SessionRetired(_logger, _provider.Backbone, reason); + NntpLogMessages.SessionRetired( + _logger, + _provider.Backbone, + session.ConnectionId, + cause, + reason); } catch (Exception diagnostic) when (!IsProcessIntegrityFailure(diagnostic)) { - // Accounting is already committed. The failing logger cannot record this diagnostic. Disposal still runs. + // Live membership is already removed; capacity is released after dispose. Disposal still runs. } try @@ -2576,10 +2667,10 @@ private void SignalSessionRetired(NntpProviderSession session, string reason) } /// - /// Disposes a session whose created-count and connection number have already been released. + /// Disposes a session that has left but whose created-count and connection number are still reserved. /// /// Detached session. - /// Connection number already returned to the free set. Used only in the cleanup diagnostic. + /// Connection number still reserved for this session. Used in the cleanup diagnostic. /// Shared shutdown instant. Forwarded unchanged so this session does not receive a new QUIT budget. /// Caller cancellation forwarded only to the QUIT write. /// @@ -2587,7 +2678,8 @@ private void SignalSessionRetired(NntpProviderSession session, string reason) /// are logged and do not propagate. /// is expected when pool dispose and session dispose close the same /// transport, and already treats it as shutdown. Other exceptions propagate so a - /// programming error is not hidden. This method does not change pool accounting. + /// programming error is not hidden. This method does not change pool accounting; the caller releases the reservation + /// after this method returns or throws. /// private async Task DisposeDetachedSessionAsync( NntpProviderSession session, @@ -2605,6 +2697,68 @@ private async Task DisposeDetachedSessionAsync( } } + /// + /// Wakes acquire waiters that are blocked because is at . + /// + /// + /// Called after a retiring session releases its reservation, and when a session becomes idle so a waiter can take it + /// instead of opening a new connection. The prior signal is completed only after a replacement source is published. + /// + private void NotifyCapacityChanged() + { + TaskCompletionSource completed; + lock (_capacityGate) + { + completed = _capacitySignal; + _capacitySignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + completed.TrySetResult(); + } + + /// + /// Waits until created-count capacity may allow a new session, an idle session is available, the pool is shutting down, + /// or a capacity pulse arrives. + /// + /// Caller or shutdown cancellation for the wait. + /// + /// The fill gate is not held. After this returns, rechecks idle membership and . + /// Idle is observed before awaiting so an admission that raced with this call is not missed by waiting on the + /// replacement alone. A single pulse returns to the acquire loop; that loop waits again + /// if the pool is still at capacity with an empty idle queue. + /// + private Task WaitForCapacityChangedAsync(CancellationToken cancellationToken) + { + Task wait; + lock (_capacityGate) + { + if (Volatile.Read(ref _created) < _provider.MaxSessions + || Volatile.Read(ref _disposed) == 1 + || _shutdown.IsCancellationRequested) + { + return Task.CompletedTask; + } + + wait = _capacitySignal.Task; + } + + if (IdleSessionAvailable()) + { + return Task.CompletedTask; + } + + return wait.WaitAsync(cancellationToken); + } + + /// Whether has a session a capacity waiter can take instead of creating. + private bool IdleSessionAvailable() + { + lock (_idleGate) + { + return _idle.Count > 0; + } + } + /// Reports whether is an expected failure while disposing a detached session. /// Exception thrown by session disposal or another cleanup call on that session. /// @@ -2742,7 +2896,8 @@ private async Task FillDeficitAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested && Volatile.Read(ref _disposed) == 0 - && _live.Count < _provider.MaxSessions) + && _live.Count < _provider.MaxSessions + && Volatile.Read(ref _created) < _provider.MaxSessions) { if (!await TryCreateIdleSessionAsync(cancellationToken).ConfigureAwait(false)) { @@ -2780,7 +2935,11 @@ private async Task TryCreateIdleSessionAsync(CancellationToken cancellatio var session = await CreateReadySessionAsync(cancellationToken).ConfigureAwait(false); if (TryAdmitIdle(session, enforceBound: false) != IdleAdmission.Admitted) { - await RetireSessionAsync(session, "session is no longer reusable").ConfigureAwait(false); + await RetireSessionAsync( + session, + NntpSessionRetirementCause.SessionNotReusable, + "session is no longer reusable") + .ConfigureAwait(false); return false; } diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionRetirementCause.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionRetirementCause.cs new file mode 100644 index 00000000..1d946754 --- /dev/null +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionRetirementCause.cs @@ -0,0 +1,26 @@ +namespace VectorNNTPD.BackFillerService.Nntp +{ + /// + /// Structured causes for . Distinct from the human-readable reason text. + /// + internal static class NntpSessionRetirementCause + { + /// Lease dispose requested retirement ( or non-reusable session). + internal const string LeaseRequested = "lease_requested"; + + /// Pool dispose or shutdown cancelled outstanding idle/leased sessions. + internal const string PoolShuttingDown = "pool_shutting_down"; + + /// Idle return found live count above the bound max after a MaxSessions shrink. + internal const string MaxSessionsShrink = "max_sessions_shrink"; + + /// Idle admission rejected the session as not reusable (unhealthy, wrong state, or missing stream). + internal const string SessionNotReusable = "session_not_reusable"; + + /// DATE keepalive observed a non-reusable or failed session while idle. + internal const string DateKeepaliveFailed = "date_keepalive_failed"; + + /// Connect/publication path retired a session that never became Ready for issue. + internal const string EstablishmentFailed = "establishment_failed"; + } +} diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpStatusCode.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpStatusCode.cs index a5ed0c59..b1827159 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpStatusCode.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpStatusCode.cs @@ -42,6 +42,26 @@ internal static class NntpStatusCode /// RFC 3977 ARTICLE by Message-ID: no such article. internal const int NoArticleWithMessageId = 430; + /// + /// Provider ARTICLE response treated as a definitive miss for Article Work lookup. + /// + /// + /// Not an RFC 3977 ARTICLE success/failure code for Message-ID retrieval. Some upstream providers + /// return 451 when the article is unavailable; BackFiller maps that status to + /// (same settlement as 430), not provider failure. + /// + internal const int ArticleUnavailable = 451; + + /// + /// RFC 3977 §3.2.1: server terminates the connection after this response. + /// + internal const int ServiceTemporarilyUnavailable = 400; + + /// + /// RFC 3977 §3.2.1: internal fault preventing the command (single-line; connection stays open). + /// + internal const int InternalFault = 403; + /// RFC 3977/4643: authentication required for the command. private const int AuthenticationRequired = 480; @@ -52,16 +72,19 @@ internal static class NntpStatusCode private const int AuthenticationOutOfSequence = 482; /// RFC 3977: command unknown. - private const int CommandUnknown = 500; + internal const int CommandUnknown = 500; /// RFC 3977: command syntax error. - private const int CommandSyntaxError = 501; + internal const int CommandSyntaxError = 501; - /// RFC 3977: command not permitted / service permanently unavailable. - private const int CommandNotPermitted = 502; + /// + /// RFC 3977 §3.2.1: command not permitted on this connection; a new connection with + /// appropriate authority is required before the facility can be used. + /// + internal const int CommandNotPermitted = 502; /// RFC 3977: program fault / temporary unavailability. - private const int ProgramFault = 503; + internal const int ProgramFault = 503; /// Returns whether is a usable NNTP greeting. /// NNTP status code. @@ -88,5 +111,24 @@ internal static bool IsAuthenticationFailure(int code) => /// internal static bool IsCommandRejected(int code) => code is CommandUnknown or CommandSyntaxError or CommandNotPermitted or ProgramFault; + + /// + /// Returns whether an ARTICLE status is a single-line command failure that leaves the session synchronized + /// and reusable for a later ARTICLE (RFC 3977 §§3.2, 3.2.1). + /// + /// NNTP status from the ARTICLE status line. + /// + /// for , , + /// , or . + /// + /// + /// ARTICLE is a mandatory READER command; RFC 3977 §3.2.1 forbids 500 for mandatory commands. + /// Providers may still return 500 (for example Giganews server error) as a per-request fault. + /// Unexpected 4xx/5xx are command failures (RFC 3977 §3.2); a single-line response does not desynchronize + /// the session. and are + /// excluded: 502 requires a new connection with appropriate authority, and 400 closes the connection. + /// + internal static bool IsArticleLevelFailureReusable(int code) => + code is InternalFault or CommandUnknown or CommandSyntaxError or ProgramFault; } } diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqRpcChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqRpcChannel.cs index 5148a9cd..d8aec4fa 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqRpcChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/IRabbitMqRpcChannel.cs @@ -37,7 +37,10 @@ Task QueueDeclareAsync( /// Logical lookup UUID carried as the AMQP RequestId property. /// NNTPD reply-queue name. /// AMQP content type. Article-work uses application/json. - /// AMQP expiration in milliseconds. Article-work uses 1000. + /// + /// AMQP expiration in milliseconds, or empty/whitespace to omit the property. + /// Article-work requests omit expiration so ready work may buffer; other RPCs may supply a TTL. + /// /// Application payload bytes. /// Token used to cancel the publish. Task PublishAsync( diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs index ef4cb831..e6a9d635 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientPublishChannel.cs @@ -107,10 +107,15 @@ await _channel.BasicPublishAsync( } catch (PublishException ex) { + // With publisherConfirmationTrackingEnabled, BasicPublishAsync completes only after + // confirm or return; IsReturn is the correlated mandatory-unroutable outcome. + if (ex.IsReturn) + { + throw new RabbitMqPublicationUnroutableException(ex); + } + throw new InvalidOperationException( - ex.IsReturn - ? "RabbitMQ returned the publication as unroutable." - : "RabbitMQ negatively acknowledged the publication.", + "RabbitMQ negatively acknowledged the publication.", ex); } diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientRpcChannel.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientRpcChannel.cs index 808dd86e..654ee9a2 100644 --- a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientRpcChannel.cs +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqClientRpcChannel.cs @@ -66,16 +66,19 @@ public async Task PublishAsync( ArgumentException.ThrowIfNullOrWhiteSpace(requestId); ArgumentException.ThrowIfNullOrWhiteSpace(replyTo); ArgumentException.ThrowIfNullOrWhiteSpace(contentType); - ArgumentException.ThrowIfNullOrWhiteSpace(expiration); + ArgumentNullException.ThrowIfNull(expiration); var properties = new BasicProperties { CorrelationId = correlationId, ReplyTo = replyTo, ContentType = contentType, - Expiration = expiration, Headers = CreateRequestIdHeaders(requestId), }; + if (!string.IsNullOrWhiteSpace(expiration)) + { + properties.Expiration = expiration; + } await _channel.BasicPublishAsync( exchange, diff --git a/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqPublicationUnroutableException.cs b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqPublicationUnroutableException.cs new file mode 100644 index 00000000..a63c0cdf --- /dev/null +++ b/src/VectorNNTPD.Common/Messaging/RabbitMq/RabbitMqPublicationUnroutableException.cs @@ -0,0 +1,32 @@ +namespace VectorNNTPD.Common.Messaging.RabbitMq +{ + /// + /// The broker returned a mandatory publication as unroutable (basic.return / confirm return). + /// + /// + /// Distinct from a publisher-confirm NACK or a connection/channel failure. Callers must not + /// treat every publish failure as unroutable. + /// + public sealed class RabbitMqPublicationUnroutableException : InvalidOperationException + { + /// Stable message used when the broker returns a mandatory publication. + public const string DefaultMessage = "RabbitMQ returned the publication as unroutable."; + + /// + /// Initializes a new instance with . + /// + public RabbitMqPublicationUnroutableException() + : base(DefaultMessage) + { + } + + /// + /// Initializes a new instance with and an inner exception. + /// + /// Broker client exception that reported the return. + public RabbitMqPublicationUnroutableException(Exception? innerException) + : base(DefaultMessage, innerException) + { + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleFailedDiagnosticsIsolatedRuntimeTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleFailedDiagnosticsIsolatedRuntimeTests.cs new file mode 100644 index 00000000..044cd5a7 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleFailedDiagnosticsIsolatedRuntimeTests.cs @@ -0,0 +1,192 @@ +using System.Globalization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Serilog.Extensions.Logging; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.Retention; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork +{ + /// + /// Isolated verification that provider failures still emit Information Event 5321 and preserve + /// useful text after A5/A6 cleanup. + /// + public sealed class ArticleFailedDiagnosticsIsolatedRuntimeTests + { + /// + /// Scripted ARTICLE 400 → ProviderFailure result error retains status text; Event 5321 is Information Failed. + /// + [Fact] + public async Task Isolated_scripted_nntp_provider_failure_emits_serilog_failed_event_5321() + { + const string statusText = "Service unavailable"; + const int statusCode = 400; + var sink = new ListSink(); + using var serilog = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sink).CreateLogger(); + using var factory = new SerilogLoggerFactory(serilog, dispose: false); + var handlerLogger = factory.CreateLogger(); + + var nntpFactory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + server.Respond(_ => $"{statusCode} {statusText}\r\n"); + nntpFactory.Enqueue(server); + await using var registry = new NntpProviderRegistry( + new StaticBackFillerProviderCatalog( + [ + new BackFillerProviderDefinition("Giganews", "127.0.0.1", 119, false, null, null, 0, 1), + ]), + nntpFactory, + NntpSessionOptions.Default with + { + ConnectTimeout = TimeSpan.FromSeconds(2), + CommandTimeout = TimeSpan.FromSeconds(2), + ReceiveTimeout = TimeSpan.FromSeconds(2), + }, + TimeSpan.FromSeconds(2), + NullLogger.Instance); + await using var retention = CreateRetention(); + var handler = new ProviderArticleWorkHandler( + new NntpArticleRetriever(registry), + retention, + new NntpArticleParser("backfiller.test"), + sharedConfiguration: null, + serverId: 0, + handlerLogger); + + var result = await handler.HandleAsync( + Item(ArticleWorkTestDeliveries.CanonicalMessageId, "Giganews"), + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.ProviderFailure, result.Outcome); + Assert.Equal(statusText, result.Error); + Assert.Equal(ArticleRetrievalKind.ProviderFailure, handler.LastKind); + + var entry = Assert.Single(sink.Events, static e => e.Level == LogEventLevel.Information); + var rendered = entry.RenderMessage(CultureInfo.InvariantCulture); + Assert.Contains("Article processed", rendered, StringComparison.Ordinal); + Assert.Contains("outcome=\"Failed\"", rendered, StringComparison.Ordinal); + Assert.Null(entry.Exception); + } + + /// + /// Connect-time IOException message survives into handler result.Error (FormatReason), not type-name-only. + /// + [Fact] + public async Task Isolated_connect_io_exception_preserves_message_in_handler_error() + { + const string transportMessage = "connection reset by peer"; + var sink = new ListSink(); + using var serilog = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sink).CreateLogger(); + using var factory = new SerilogLoggerFactory(serilog, dispose: false); + var handlerLogger = factory.CreateLogger(); + + var nntpFactory = new ScriptedNntpTransportFactory + { + ConnectException = new IOException(transportMessage), + }; + await using var registry = new NntpProviderRegistry( + new StaticBackFillerProviderCatalog( + [ + new BackFillerProviderDefinition("Giganews", "127.0.0.1", 119, false, null, null, 0, 1), + ]), + nntpFactory, + NntpSessionOptions.Default with + { + ConnectTimeout = TimeSpan.FromSeconds(2), + CommandTimeout = TimeSpan.FromSeconds(2), + ReceiveTimeout = TimeSpan.FromSeconds(2), + }, + TimeSpan.FromSeconds(2), + NullLogger.Instance); + await using var retention = CreateRetention(); + var handler = new ProviderArticleWorkHandler( + new NntpArticleRetriever(registry), + retention, + new NntpArticleParser("backfiller.test"), + sharedConfiguration: null, + serverId: 0, + handlerLogger); + + var result = await handler.HandleAsync( + Item(ArticleWorkTestDeliveries.CanonicalMessageId, "Giganews"), + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.ProviderFailure, result.Outcome); + Assert.Equal("IOException: " + transportMessage, result.Error); + var entry = Assert.Single(sink.Events, static e => e.Level == LogEventLevel.Information); + Assert.Contains("outcome=\"Failed\"", entry.RenderMessage(CultureInfo.InvariantCulture), StringComparison.Ordinal); + } + + /// + /// A retrieve exception still emits Failed Event 5321 before rethrow (HEAD behaviour: no attached exception). + /// + [Fact] + public async Task Isolated_retrieve_exception_emits_failed_event_then_rethrows() + { + var sink = new ListSink(); + using var serilog = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sink).CreateLogger(); + using var factory = new SerilogLoggerFactory(serilog, dispose: false); + var handlerLogger = factory.CreateLogger(); + await using var retention = CreateRetention(); + var thrown = new IOException("isolated-runtime socket closed"); + var handler = new ProviderArticleWorkHandler( + new ThrowingRetriever(thrown), + retention, + new NntpArticleParser("backfiller.test"), + sharedConfiguration: null, + serverId: 0, + handlerLogger); + + var ex = await Assert.ThrowsAsync( + () => handler.HandleAsync( + Item(ArticleWorkTestDeliveries.CanonicalMessageId, "Giganews"), + CancellationToken.None).AsTask()); + + Assert.Same(thrown, ex); + var entry = Assert.Single(sink.Events, static e => e.Level == LogEventLevel.Information); + Assert.Contains("outcome=\"Failed\"", entry.RenderMessage(CultureInfo.InvariantCulture), StringComparison.Ordinal); + Assert.Null(entry.Exception); + } + + private static ArticleWorkItem Item(string messageId, string backbone) => + new( + new ArticleWorkRequest(1, Guid.Parse(ArticleWorkTestDeliveries.CanonicalRequestId), messageId, backbone), + "corr", + "reply", + new ArticleWorkSettlementLease(new FakeBackFillerRabbitMqChannel(1), 1, 1)); + + private static ArticleRetentionAuthority CreateRetention() => + new( + new BackFillerArticleRetentionRuntimeOptions(1024 * 1024, TimeSpan.FromHours(1), TimeSpan.FromMinutes(1), 256), + "backfiller.test", + 563, + TimeProvider.System, + NullLogger.Instance); + + private sealed class ThrowingRetriever(Exception exception) : INntpArticleRetriever + { + public Task RetrieveAsync( + ArticleWorkItem item, + int maxArticleBytes, + Func, ArticleRecordCreateResult> consumePayload, + CancellationToken cancellationToken) => + Task.FromException(exception); + } + + private sealed class ListSink : ILogEventSink + { + public List Events { get; } = new(); + + public void Emit(LogEvent logEvent) => Events.Add(logEvent); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleProcessingLogTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleProcessingLogTests.cs index 360a1226..b698b05c 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleProcessingLogTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleProcessingLogTests.cs @@ -103,11 +103,12 @@ public async Task ValidationFailure_logs_one_information_event_with_the_parser_r } /// - /// Retryable provider transport failure logs outcome Failed without echoing exception text or validation wording. + /// Provider transport failure logs outcome Failed with the retrieval diagnostic reason and connection id. /// [Fact] public async Task TransportFailure_logs_one_information_event_with_failed() { + var connectionId = Guid.Parse("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"); var logger = new CollectingLogger(); await using var retention = CreateRetention(); var handler = CreateHandler( @@ -115,15 +116,18 @@ public async Task TransportFailure_logs_one_information_event_with_failed() retention, new ScriptedRetriever( payload: null, - ArticleRetrievalResult.Failed(ArticleRetrievalKind.ProviderFailure, null, "connection reset", sessionReusable: false), - Stopwatch.GetTimestamp())); + ArticleRetrievalResult.Failed(ArticleRetrievalKind.ProviderFailure, 400, "connection reset", sessionReusable: false), + Stopwatch.GetTimestamp(), + connectionId)); var result = await handler.HandleAsync(Item(ArticleWorkTestDeliveries.CanonicalMessageId, "Giganews"), CancellationToken.None); Assert.Equal(ArticleWorkOutcome.ProviderFailure, result.Outcome); var entry = AssertSingleCompletion(logger); AssertCompletion(entry, ArticleWorkTestDeliveries.CanonicalMessageId, "Giganews", "Failed"); - Assert.DoesNotContain("connection reset", entry.Message, StringComparison.Ordinal); + Assert.Contains("reason=connection reset", entry.Message, StringComparison.Ordinal); + Assert.Contains("statusCode=400", entry.Message, StringComparison.Ordinal); + Assert.Contains($"connectionId={connectionId:D}", entry.Message, StringComparison.Ordinal); Assert.DoesNotContain("ValidationFailed", entry.Message, StringComparison.Ordinal); } @@ -472,11 +476,19 @@ private sealed class ScriptedRetriever : INntpArticleRetriever /// Article bytes for the callback, or null to skip consumption (failures). /// Base retrieval result (kind, code, session reuse). /// Value assigned to on return. - internal ScriptedRetriever(byte[]? payload, ArticleRetrievalResult result, long commandStartedTimestamp) + /// Optional physical connection identity stamped onto the returned result. + private readonly Guid _connectionId; + + internal ScriptedRetriever( + byte[]? payload, + ArticleRetrievalResult result, + long commandStartedTimestamp, + Guid connectionId = default) { _payload = payload; _result = result; _commandStartedTimestamp = commandStartedTimestamp; + _connectionId = connectionId; } /// @@ -501,6 +513,7 @@ public Task RetrieveAsync( } _result.CommandStartedTimestamp = _commandStartedTimestamp; + _result.ConnectionId = _connectionId; return Task.FromResult(_result); } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkArticlePipelineIntegrationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkArticlePipelineIntegrationTests.cs index f252322c..16910ca6 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkArticlePipelineIntegrationTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkArticlePipelineIntegrationTests.cs @@ -143,10 +143,10 @@ public async Task Article_message_id_mismatch_acks_without_requeue_retain_or_suc } /// - /// Provider connect failure is retryable: nack with requeue, no terminal response publication, no retention. + /// Provider connect failure is terminal for this delivery: ACK, correlated ProviderFailure reply, no retention. /// [Fact] - public async Task Provider_network_failure_nacks_with_requeue() + public async Task Provider_network_failure_acks_with_failure_reply() { await using var harness = await BackFillerPipelineHarness.StartAsync(); harness.Nntp.ConnectException = new IOException("down"); @@ -157,9 +157,9 @@ public async Task Provider_network_failure_nacks_with_requeue() Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); Assert.Equal(0, harness.Retention.RetainedCount); var settlement = Assert.Single(channel.Settlements); - Assert.False(settlement.Acknowledge); - Assert.True(settlement.Requeue); - Assert.Empty(harness.PublishChannel.Publications); + Assert.True(settlement.Acknowledge); + Assert.False(settlement.Requeue); + Assert.Single(harness.PublishChannel.Publications); } /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs index 6a692110..5517afae 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs @@ -136,7 +136,7 @@ public async Task First_delivery_without_count_requeues_on_retryable_failure() { var channel = new FakeBackFillerRabbitMqChannel(1); var pipeline = new ArticleWorkDeliveryPipeline( - new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure }, + new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Cancelled }, new RecordingArticleWorkResponsePublisher(), maxPayloadBytes: 1024, deliveryLimit: 3); @@ -148,7 +148,7 @@ public async Task First_delivery_without_count_requeues_on_retryable_failure() static () => true, CancellationToken.None); - Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + Assert.Equal(ArticleWorkOutcome.Cancelled, outcome); var settlement = Assert.Single(channel.Settlements); Assert.False(settlement.Acknowledge); Assert.True(settlement.Requeue); @@ -162,7 +162,7 @@ public async Task Final_permitted_retry_still_requeues() { var channel = new FakeBackFillerRabbitMqChannel(1); var pipeline = new ArticleWorkDeliveryPipeline( - new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure }, + new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Cancelled }, new RecordingArticleWorkResponsePublisher(), maxPayloadBytes: 1024, deliveryLimit: 5); @@ -174,7 +174,7 @@ public async Task Final_permitted_retry_still_requeues() static () => true, CancellationToken.None); - Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + Assert.Equal(ArticleWorkOutcome.Cancelled, outcome); var settlement = Assert.Single(channel.Settlements); Assert.False(settlement.Acknowledge); Assert.True(settlement.Requeue); @@ -395,7 +395,7 @@ public async Task Retryable_failure_requeues_within_delivery_limit() { var channel = new FakeBackFillerRabbitMqChannel(1); var pipeline = new ArticleWorkDeliveryPipeline( - new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure }, + new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Cancelled }, new RecordingArticleWorkResponsePublisher(), maxPayloadBytes: 1024, deliveryLimit: 10); @@ -407,7 +407,7 @@ public async Task Retryable_failure_requeues_within_delivery_limit() static () => true, CancellationToken.None); - Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + Assert.Equal(ArticleWorkOutcome.Cancelled, outcome); var settlement = Assert.Single(channel.Settlements); Assert.False(settlement.Acknowledge); Assert.True(settlement.Requeue); @@ -421,7 +421,7 @@ public async Task Exhausted_delivery_limit_nacks_without_requeue() { var channel = new FakeBackFillerRabbitMqChannel(1); var pipeline = new ArticleWorkDeliveryPipeline( - new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure }, + new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Cancelled }, new RecordingArticleWorkResponsePublisher(), maxPayloadBytes: 1024, deliveryLimit: 5); @@ -433,7 +433,7 @@ public async Task Exhausted_delivery_limit_nacks_without_requeue() static () => true, CancellationToken.None); - Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + Assert.Equal(ArticleWorkOutcome.Cancelled, outcome); var settlement = Assert.Single(channel.Settlements); Assert.False(settlement.Acknowledge); Assert.False(settlement.Requeue); @@ -534,7 +534,7 @@ public async Task Quorum_redelivery_without_delivery_count_does_not_requeue() { var channel = new FakeBackFillerRabbitMqChannel(1); var pipeline = new ArticleWorkDeliveryPipeline( - new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure }, + new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Cancelled }, new RecordingArticleWorkResponsePublisher(), maxPayloadBytes: 1024, deliveryLimit: 5, @@ -553,7 +553,7 @@ public async Task Quorum_redelivery_without_delivery_count_does_not_requeue() static () => true, CancellationToken.None); - Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + Assert.Equal(ArticleWorkOutcome.Cancelled, outcome); var settlement = Assert.Single(channel.Settlements); Assert.False(settlement.Acknowledge); Assert.False(settlement.Requeue); @@ -567,7 +567,7 @@ public async Task Classic_redelivery_without_delivery_count_still_requeues() { var channel = new FakeBackFillerRabbitMqChannel(1); var pipeline = new ArticleWorkDeliveryPipeline( - new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.ProviderFailure }, + new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Cancelled }, new RecordingArticleWorkResponsePublisher(), maxPayloadBytes: 1024, deliveryLimit: 5, @@ -586,7 +586,7 @@ public async Task Classic_redelivery_without_delivery_count_still_requeues() static () => true, CancellationToken.None); - Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + Assert.Equal(ArticleWorkOutcome.Cancelled, outcome); var settlement = Assert.Single(channel.Settlements); Assert.False(settlement.Acknowledge); Assert.True(settlement.Requeue); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs index 984bf945..02839be3 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponsePublicationTests.cs @@ -2,6 +2,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.RabbitMq; using VectorNNTPD.BackFillerService.Tests.Fixtures; using VectorNNTPD.BackFillerService.Tests.RabbitMq; using VectorNNTPD.BackFillerService.Tests.TestDoubles; @@ -78,7 +80,8 @@ public async Task Redelivered_success_may_publish_again_but_each_delivery_settle } /// - /// Publish/confirm failure modes for success NACK-requeue once and do not ACK (no local retry loop). + /// Transient/ambiguous publish failures NACK-requeue once and do not ACK (no local retry loop). + /// Confirmed unroutable is covered separately as terminal without requeue. /// [Fact] public async Task Success_publication_failures_nack_requeue_and_do_not_ack() @@ -88,7 +91,6 @@ public async Task Success_publication_failures_nack_requeue_and_do_not_ack() FakePublishConfirmBehavior.ThrowOnPublish, FakePublishConfirmBehavior.Nack, FakePublishConfirmBehavior.CloseChannel, - FakePublishConfirmBehavior.Unroutable, ]; foreach (var behavior in behaviors) @@ -370,29 +372,21 @@ public async Task InvalidRequest_publish_failure_nacks_with_requeue() } /// - /// Retryable handler outcomes nack with requeue and emit no response JSON. + /// Cancelled (including cancel override) nacks with requeue and emits no response JSON. /// - /// Retryable outcome name. - [Theory] - [InlineData(nameof(ArticleWorkOutcome.ProviderFailure))] - [InlineData(nameof(ArticleWorkOutcome.Cancelled))] - [InlineData(nameof(ArticleWorkOutcome.UnexpectedFailure))] - public async Task Retryable_outcomes_do_not_publish(string handlerOutcomeName) + [Fact] + public async Task Cancelled_outcomes_do_not_publish() { - var handlerOutcome = Enum.Parse(handlerOutcomeName); await using var context = await PublicationContext.StartAsync(FakePublishConfirmBehavior.Confirm); var handler = new ControllableArticleWorkHandler { - Outcome = handlerOutcome, + Outcome = ArticleWorkOutcome.Cancelled, Error = "retry", }; var channel = new FakeBackFillerRabbitMqChannel(1); var pipeline = new ArticleWorkDeliveryPipeline(handler, context.Publisher, 1024); using var cts = new CancellationTokenSource(); - if (handlerOutcome == ArticleWorkOutcome.Cancelled) - { - cts.Cancel(); - } + cts.Cancel(); var outcome = await pipeline.ProcessAsync( ArticleWorkTestDeliveries.Canonical(), @@ -401,7 +395,7 @@ public async Task Retryable_outcomes_do_not_publish(string handlerOutcomeName) static () => true, cts.Token); - Assert.Equal(handlerOutcome, outcome); + Assert.Equal(ArticleWorkOutcome.Cancelled, outcome); Assert.Empty(context.PublishChannel.Publications); Assert.True(Assert.Single(channel.Settlements).Requeue); Assert.False(Assert.Single(channel.Settlements).Acknowledge); @@ -588,10 +582,10 @@ await pipeline.ProcessAsync( } /// - /// An unroutable confirm NACK-requeues once. The pipeline does not replay the handler on that same call. + /// Confirmed mandatory unroutable Success is terminal: NACK without requeue, one handler invocation. /// [Fact] - public async Task Unroutable_success_response_nacks_once_without_a_local_execution_loop() + public async Task Unroutable_success_response_nacks_without_requeue_and_without_a_local_execution_loop() { await using var context = await PublicationContext.StartAsync(FakePublishConfirmBehavior.Unroutable); var handler = SuccessHandler(); @@ -609,11 +603,11 @@ public async Task Unroutable_success_response_nacks_once_without_a_local_executi Assert.Equal(1, handler.HandleCount); var settlement = Assert.Single(channel.Settlements); Assert.False(settlement.Acknowledge); - Assert.True(settlement.Requeue); + Assert.False(settlement.Requeue); } /// - /// Unroutable success publication logs request id, outcome, and unroutable hint, then NACK-requeues. + /// Unroutable success publication logs request id, correlation, ReplyTo, and outcome, then terminal NACK. /// [Fact] public async Task Unroutable_success_publication_is_logged_with_request_id_and_outcome() @@ -633,14 +627,136 @@ public async Task Unroutable_success_publication_is_logged_with_request_id_and_o CancellationToken.None); Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); - Assert.False(Assert.Single(channel.Settlements).Acknowledge); - Assert.True(Assert.Single(channel.Settlements).Requeue); + var settlement = Assert.Single(channel.Settlements); + Assert.False(settlement.Acknowledge); + Assert.False(settlement.Requeue); Assert.Contains( logger.Messages, static message => message.Contains(ArticleWorkTestDeliveries.CanonicalRequestId, StringComparison.Ordinal) + && message.Contains(ArticleWorkTestDeliveries.CanonicalMessageId, StringComparison.Ordinal) + && message.Contains("Giganews", StringComparison.Ordinal) && message.Contains("Success", StringComparison.Ordinal) - && message.Contains("unroutable", StringComparison.OrdinalIgnoreCase)); + && message.Contains("unroutable", StringComparison.OrdinalIgnoreCase) + && message.Contains("not be requeued", StringComparison.OrdinalIgnoreCase) + && message.Contains("dead-letter", StringComparison.OrdinalIgnoreCase)); + } + + /// + /// Terminal unroutable settlement discards without requeue and does not claim a dead-letter destination. + /// + [Fact] + public async Task Unroutable_terminal_settlement_is_discard_without_requeue_not_dead_letter_claim() + { + await using var context = await PublicationContext.StartAsync(FakePublishConfirmBehavior.Unroutable); + var channel = new FakeBackFillerRabbitMqChannel(1); + var pipeline = new ArticleWorkDeliveryPipeline(SuccessHandler(), context.Publisher, 1024); + + _ = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + var settlement = Assert.Single(channel.Settlements); + Assert.False(settlement.Acknowledge); + Assert.False(settlement.Requeue); + // Architecture: no DLX on Article Work queues; this is broker discard, not dead-letter. + Assert.False( + BackFillerArticleWorkTopology.BuildQueueArguments( + new BackFillerRabbitMqRuntimeOptions( + WorkRequestMaxPayloadBytes: 1024, + PublishConfirmTimeoutSeconds: 10, + ConsumerPrefetchCount: 1, + QueueTopology: new ArticleWorkQueueTopology( + ArticleWorkQueueKind.Classic, + ConfiguredHostCount: 1, + QuorumInitialGroupSize: null), + DeliveryLimit: 10)) + .ContainsKey("x-dead-letter-exchange")); + } + + /// + /// Ambiguous/non-unroutable publish exceptions remain retryable (NACK requeue). + /// + [Fact] + public async Task Ambiguous_publish_exception_is_not_classified_as_unroutable() + { + var publisher = new RecordingArticleWorkResponsePublisher + { + CompletesSuccessPublication = true, + PublishException = new InvalidOperationException("some other publish failure"), + }; + var handler = SuccessHandler(); + var channel = new FakeBackFillerRabbitMqChannel(1); + var pipeline = new ArticleWorkDeliveryPipeline(handler, publisher, 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + var settlement = Assert.Single(channel.Settlements); + Assert.False(settlement.Acknowledge); + Assert.True(settlement.Requeue); + } + + /// + /// Typed is terminal even when thrown by a test double. + /// + [Fact] + public async Task Typed_unroutable_exception_from_publisher_nacks_without_requeue() + { + var publisher = new RecordingArticleWorkResponsePublisher + { + CompletesSuccessPublication = true, + PublishException = new RabbitMqPublicationUnroutableException(), + }; + var handler = SuccessHandler(); + var channel = new FakeBackFillerRabbitMqChannel(1); + var pipeline = new ArticleWorkDeliveryPipeline(handler, publisher, 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + Assert.Equal(1, handler.HandleCount); + var settlement = Assert.Single(channel.Settlements); + Assert.False(settlement.Acknowledge); + Assert.False(settlement.Requeue); + } + + /// + /// Publisher-confirm NACK (non-return) remains retryable and must not be treated as unroutable. + /// + [Fact] + public async Task Publisher_confirm_nack_remains_retryable() + { + await using var context = await PublicationContext.StartAsync(FakePublishConfirmBehavior.Nack); + var handler = SuccessHandler(); + var channel = new FakeBackFillerRabbitMqChannel(1); + var pipeline = new ArticleWorkDeliveryPipeline(handler, context.Publisher, 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + Assert.Equal(1, handler.HandleCount); + var settlement = Assert.Single(channel.Settlements); + Assert.False(settlement.Acknowledge); + Assert.True(settlement.Requeue); } /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponseWireProtocolTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponseWireProtocolTests.cs index 08088827..e3856893 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponseWireProtocolTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponseWireProtocolTests.cs @@ -10,6 +10,23 @@ namespace VectorNNTPD.BackFillerService.Tests.ArticleWork /// public sealed class ArticleWorkResponseWireProtocolTests { + /// + /// Response AMQP Expiration remains the locked 1-second reply-path TTL (not a work-request TTL). + /// + [Fact] + public void Response_expiration_milliseconds_is_exactly_1000() + { + Assert.Equal("1000", ArticleWorkResponseWireProtocol.ExpirationMilliseconds); + } + + [Fact] + public void Response_expiration_is_distinct_from_omitted_work_request_ttl() + { + // Work requests must not inherit the response TTL constant; buffering depends on omission. + Assert.False(string.IsNullOrWhiteSpace(ArticleWorkResponseWireProtocol.ExpirationMilliseconds)); + Assert.Equal("1000", ArticleWorkResponseWireProtocol.ExpirationMilliseconds); + } + /// /// Success serialization matches the canonical protocol bytes and excludes uri/error/payload fields. /// @@ -155,27 +172,46 @@ public void Success_article_id_round_trips_exactly() } /// - /// Retryable/non-terminal outcomes must not produce v1 terminal response JSON (serializer throws). + /// Cancelled is not a publishable terminal reply (NACK-requeue without RPC response). + /// + [Fact] + public void Cancelled_is_not_a_serializable_terminal_response() + { + var intent = new ArticleWorkResponseIntent( + ArticleWorkOutcome.Cancelled, + Guid.Parse(ArticleWorkTestDeliveries.CanonicalRequestId), + ArticleWorkTestDeliveries.CanonicalMessageId, + "Giganews", + ArticleWorkTestDeliveries.CanonicalCorrelationId, + ArticleWorkTestDeliveries.CanonicalReplyTo, + "cancelled"); + + Assert.Throws(() => ArticleWorkResponseWireProtocol.SerializeV1(intent)); + } + + /// + /// ProviderFailure / UnexpectedFailure / RetentionRejected serialize as terminal failure replies with error text. /// - /// Non-terminal outcome name. [Theory] - [InlineData(nameof(ArticleWorkOutcome.ProviderFailure))] - [InlineData(nameof(ArticleWorkOutcome.Cancelled))] - [InlineData(nameof(ArticleWorkOutcome.UnexpectedFailure))] - [InlineData(nameof(ArticleWorkOutcome.RetentionRejected))] - public void Retryable_outcomes_are_not_serializable_terminal_responses(string outcomeName) + [InlineData(nameof(ArticleWorkOutcome.ProviderFailure), "ProviderFailure")] + [InlineData(nameof(ArticleWorkOutcome.UnexpectedFailure), "UnexpectedFailure")] + [InlineData(nameof(ArticleWorkOutcome.RetentionRejected), "RetentionRejected")] + public void Terminal_provider_operation_failures_serialize_with_error(string outcomeName, string wireName) { var outcome = Enum.Parse(outcomeName); - var intent = new ArticleWorkResponseIntent( + var json = ArticleWorkResponseWireProtocol.SerializeV1(new ArticleWorkResponseIntent( outcome, Guid.Parse(ArticleWorkTestDeliveries.CanonicalRequestId), ArticleWorkTestDeliveries.CanonicalMessageId, "Giganews", ArticleWorkTestDeliveries.CanonicalCorrelationId, ArticleWorkTestDeliveries.CanonicalReplyTo, - "retry"); + "upstream fault")); - Assert.Throws(() => ArticleWorkResponseWireProtocol.SerializeV1(intent)); + using var document = JsonDocument.Parse(json); + Assert.Equal(wireName, document.RootElement.GetProperty("outcome").GetString()); + Assert.Equal("upstream fault", document.RootElement.GetProperty("error").GetString()); + Assert.False(document.RootElement.TryGetProperty("fqdn", out _)); } /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkSettlementTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkSettlementTests.cs index e0effb8a..9a799af7 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkSettlementTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkSettlementTests.cs @@ -23,10 +23,10 @@ public sealed class ArticleWorkSettlementTests [InlineData(nameof(ArticleWorkOutcome.ArticleNotFound), true, false, true)] [InlineData(nameof(ArticleWorkOutcome.InvalidArticle), true, false, true)] [InlineData(nameof(ArticleWorkOutcome.InvalidRequest), true, false, true)] - [InlineData(nameof(ArticleWorkOutcome.ProviderFailure), false, true, false)] + [InlineData(nameof(ArticleWorkOutcome.ProviderFailure), true, false, true)] + [InlineData(nameof(ArticleWorkOutcome.UnexpectedFailure), true, false, true)] + [InlineData(nameof(ArticleWorkOutcome.RetentionRejected), true, false, true)] [InlineData(nameof(ArticleWorkOutcome.Cancelled), false, true, false)] - [InlineData(nameof(ArticleWorkOutcome.UnexpectedFailure), false, true, false)] - [InlineData(nameof(ArticleWorkOutcome.RetentionRejected), false, true, false)] public void Planner_maps_terminal_and_retryable_outcomes( string outcomeName, bool acknowledge, @@ -85,13 +85,13 @@ public void Cancellation_overrides_a_terminal_outcome() } /// - /// When broker delivery count reaches the configured limit, retryable outcomes NACK without requeue. + /// When broker delivery count reaches the configured limit, cancel/retryable settlements NACK without requeue. /// [Fact] public void Delivery_limit_exhaustion_nacks_without_requeue() { var disposition = ArticleWorkDispositionPlanner.Create( - ArticleWorkOutcome.ProviderFailure, + ArticleWorkOutcome.Cancelled, replyable: true, cancellationRequested: false, deliveryCount: 10, @@ -103,6 +103,25 @@ public void Delivery_limit_exhaustion_nacks_without_requeue() Assert.False(disposition.PublishResponse); } + /// + /// ProviderFailure is terminal regardless of delivery count (ACK + publish; no classic unbounded requeue). + /// + [Fact] + public void ProviderFailure_is_terminal_even_when_delivery_count_is_high() + { + var disposition = ArticleWorkDispositionPlanner.Create( + ArticleWorkOutcome.ProviderFailure, + replyable: true, + cancellationRequested: false, + deliveryCount: 10, + redelivered: true, + deliveryLimit: 10); + + Assert.True(disposition.Acknowledge); + Assert.False(disposition.Requeue); + Assert.True(disposition.PublishResponse); + } + /// /// Cancellation that has already exhausted the delivery limit also discards without requeue. /// @@ -123,7 +142,7 @@ public void Cancellation_with_exhausted_delivery_limit_does_not_requeue() } /// - /// Deferred handler (not completed) surfaces provider failure with requeue and no published terminal JSON. + /// Deferred handler surfaces provider failure as ACK + correlated failure reply (not Success, not requeue). /// [Fact] public async Task Valid_work_is_admitted_as_provider_failure_without_a_fake_success() @@ -143,10 +162,13 @@ public async Task Valid_work_is_admitted_as_provider_failure_without_a_fake_succ CancellationToken.None); Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); - Assert.Empty(publisher.Published); + var intent = Assert.Single(publisher.Published); + Assert.Equal(ArticleWorkOutcome.ProviderFailure, intent.Outcome); + Assert.False(string.IsNullOrWhiteSpace(intent.Error)); + Assert.NotEqual(ArticleWorkOutcome.Success, intent.Outcome); var settlement = Assert.Single(channel.Settlements); - Assert.False(settlement.Acknowledge); - Assert.True(settlement.Requeue); + Assert.True(settlement.Acknowledge); + Assert.False(settlement.Requeue); Assert.Equal(7UL, settlement.DeliveryTag); } @@ -217,10 +239,10 @@ public async Task Terminal_handler_outcomes_publish_and_settle_exactly_once( } /// - /// Handler exception maps to UnexpectedFailure with nack/requeue and empty publisher log. + /// Handler exception maps to UnexpectedFailure with ACK, correlated failure reply, and prepared type+message error. /// [Fact] - public async Task Retryable_failures_nack_requeue_without_a_terminal_response() + public async Task Unexpected_handler_failure_acks_with_terminal_failure_reply() { var channel = new FakeBackFillerRabbitMqChannel(1); var publisher = new RecordingArticleWorkResponsePublisher(); @@ -235,8 +257,12 @@ public async Task Retryable_failures_nack_requeue_without_a_terminal_response() CancellationToken.None); Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); - Assert.Empty(publisher.Published); - Assert.True(Assert.Single(channel.Settlements).Requeue); + Assert.Equal("InvalidOperationException: boom", pipeline.LastEscapedHandlerFailureError); + var intent = Assert.Single(publisher.Published); + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, intent.Outcome); + Assert.Equal("InvalidOperationException: boom", intent.Error); + Assert.True(Assert.Single(channel.Settlements).Acknowledge); + Assert.False(Assert.Single(channel.Settlements).Requeue); } /// diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ProviderFailureTerminalSettlementTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ProviderFailureTerminalSettlementTests.cs new file mode 100644 index 00000000..dbb0108a --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ProviderFailureTerminalSettlementTests.cs @@ -0,0 +1,134 @@ +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork +{ + /// + /// Focused regression: ProviderFailure / peer terminal failures ACK with a correlated RPC reply and never requeue. + /// + public sealed class ProviderFailureTerminalSettlementTests + { + /// + /// Planner disposition for ProviderFailure is backbone-agnostic (same ACK/publish contract). + /// + [Fact] + public void ProviderFailure_planner_disposition_is_identical_for_any_caller() + { + var first = ArticleWorkDispositionPlanner.Create( + ArticleWorkOutcome.ProviderFailure, + replyable: true, + cancellationRequested: false, + deliveryCount: null, + redelivered: false, + deliveryLimit: 10); + var second = ArticleWorkDispositionPlanner.Create( + ArticleWorkOutcome.ProviderFailure, + replyable: true, + cancellationRequested: false, + deliveryCount: null, + redelivered: false, + deliveryLimit: 10); + Assert.Equal(first, second); + Assert.True(first.Acknowledge); + Assert.False(first.Requeue); + Assert.True(first.PublishResponse); + } + + /// + /// Pipeline publishes ProviderFailure with the handler error and ACKs without requeue. + /// + [Fact] + public async Task ProviderFailure_pipeline_acks_and_publishes_failure_reply() + { + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ProviderFailure, + Error = "IOException: connection reset", + }, + publisher, + 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + var intent = Assert.Single(publisher.Published); + Assert.Equal(ArticleWorkOutcome.ProviderFailure, intent.Outcome); + Assert.Equal("IOException: connection reset", intent.Error); + var settlement = Assert.Single(channel.Settlements); + Assert.True(settlement.Acknowledge); + Assert.False(settlement.Requeue); + } + + /// + /// Repeated ProviderFailure deliveries ACK each time (no broker requeue feedback loop). + /// + [Fact] + public async Task ProviderFailure_does_not_requeue_across_repeated_deliveries() + { + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ProviderFailure, + Error = "fault", + }, + publisher, + 1024); + + for (var i = 0; i < 5; i++) + { + var channel = new FakeBackFillerRabbitMqChannel(1); + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + Assert.True(Assert.Single(channel.Settlements).Acknowledge); + Assert.False(Assert.Single(channel.Settlements).Requeue); + } + + Assert.Equal(5, publisher.Published.Count); + } + + /// + /// Found / NotFound / InvalidArticle remain ACK + publish; Cancelled remains NACK-requeue without publish. + /// + [Fact] + public void Established_terminal_and_cancel_dispositions_unchanged() + { + AssertDisposition(ArticleWorkOutcome.Success, acknowledge: true, requeue: false, publish: true); + AssertDisposition(ArticleWorkOutcome.ArticleNotFound, acknowledge: true, requeue: false, publish: true); + AssertDisposition(ArticleWorkOutcome.InvalidArticle, acknowledge: true, requeue: false, publish: true); + AssertDisposition(ArticleWorkOutcome.Cancelled, acknowledge: false, requeue: true, publish: false); + } + + private static void AssertDisposition( + ArticleWorkOutcome outcome, + bool acknowledge, + bool requeue, + bool publish) + { + var disposition = ArticleWorkDispositionPlanner.Create( + outcome, + replyable: true, + cancellationRequested: false, + deliveryCount: null, + redelivered: false, + deliveryLimit: 10); + Assert.Equal(acknowledge, disposition.Acknowledge); + Assert.Equal(requeue, disposition.Requeue); + Assert.Equal(publish, disposition.PublishResponse); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/UnexpectedFailureRpcErrorDiagnosticsTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/UnexpectedFailureRpcErrorDiagnosticsTests.cs new file mode 100644 index 00000000..b2be500e --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/UnexpectedFailureRpcErrorDiagnosticsTests.cs @@ -0,0 +1,331 @@ +using System.Net.Sockets; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.Retention; +using VectorNNTPD.BackFillerService.Tests.Fixtures; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; + +namespace VectorNNTPD.BackFillerService.Tests.ArticleWork +{ + /// + /// Regression for UnexpectedFailure error text when a handler exception escapes classification. + /// Handler UnexpectedFailure ACKs and publishes a correlated failure reply (not broker requeue). + /// + public sealed class UnexpectedFailureRpcErrorDiagnosticsTests + { + /// + /// A thrown exception with a useful message yields type+message on the published UnexpectedFailure reply. + /// + [Fact] + public async Task Escaped_handler_exception_preserves_type_and_message_in_prepared_error() + { + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler { Throw = new IOException("socket closed during ARTICLE") }, + publisher, + 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + Assert.Equal("IOException: socket closed during ARTICLE", pipeline.LastEscapedHandlerFailureError); + var intent = Assert.Single(publisher.Published); + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, intent.Outcome); + Assert.Equal("IOException: socket closed during ARTICLE", intent.Error); + var settlement = Assert.Single(channel.Settlements); + Assert.True(settlement.Acknowledge); + Assert.False(settlement.Requeue); + } + + /// + /// Whitespace-only exception messages fall back to a non-empty type-name reason on the reply. + /// + [Fact] + public async Task Escaped_handler_exception_with_blank_message_uses_type_name_fallback() + { + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler { Throw = new InvalidOperationException(" ") }, + publisher, + 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + Assert.Equal(nameof(InvalidOperationException), pipeline.LastEscapedHandlerFailureError); + Assert.False(string.IsNullOrWhiteSpace(pipeline.LastEscapedHandlerFailureError)); + Assert.Equal(nameof(InvalidOperationException), Assert.Single(publisher.Published).Error); + } + + /// + /// A useful inner exception is preserved on the published UnexpectedFailure error. + /// + [Fact] + public async Task Escaped_handler_exception_preserves_useful_inner_in_prepared_error() + { + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var inner = new SocketException((int)SocketError.ConnectionReset); + var outer = new IOException("transport reset", inner); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler { Throw = outer }, + publisher, + 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + Assert.Equal( + "IOException: transport reset <- " + nameof(SocketException) + ": " + inner.Message, + pipeline.LastEscapedHandlerFailureError); + Assert.Equal(pipeline.LastEscapedHandlerFailureError, Assert.Single(publisher.Published).Error); + Assert.True(Assert.Single(channel.Settlements).Acknowledge); + Assert.False(Assert.Single(channel.Settlements).Requeue); + } + + /// + /// When the outer message already embeds the inner text, the prepared error does not repeat it. + /// + [Fact] + public async Task Escaped_handler_exception_does_not_repeat_embedded_inner_message() + { + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var inner = new SocketException((int)SocketError.ConnectionReset); + var outer = new IOException("Unable to read data from the transport connection: " + inner.Message, inner); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler { Throw = outer }, + publisher, + 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + Assert.Equal( + "IOException: Unable to read data from the transport connection: " + inner.Message, + pipeline.LastEscapedHandlerFailureError); + Assert.DoesNotContain(" <- ", pipeline.LastEscapedHandlerFailureError, StringComparison.Ordinal); + Assert.Equal(pipeline.LastEscapedHandlerFailureError, Assert.Single(publisher.Published).Error); + } + + /// + /// Host cancellation remains Cancelled with null prepared UnexpectedFailure error and no publication. + /// + [Fact] + public async Task Cancellation_does_not_enter_unexpected_failure_error_path() + { + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler { Outcome = ArticleWorkOutcome.Success }, + publisher, + 1024); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + cts.Token); + + Assert.Equal(ArticleWorkOutcome.Cancelled, outcome); + Assert.Null(pipeline.LastEscapedHandlerFailureError); + Assert.Empty(publisher.Published); + Assert.True(Assert.Single(channel.Settlements).Requeue); + } + + /// + /// ProviderFailure keeps its handler error, does not use the escaped-exception path, ACKs, and publishes a failure reply. + /// + [Fact] + public async Task Provider_failure_acks_and_publishes_correlated_failure_without_escaped_path() + { + const string providerError = "IOException: connection reset by peer"; + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.ProviderFailure, + Error = providerError, + }, + publisher, + 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); + Assert.Null(pipeline.LastEscapedHandlerFailureError); + var intent = Assert.Single(publisher.Published); + Assert.Equal(ArticleWorkOutcome.ProviderFailure, intent.Outcome); + Assert.Equal(providerError, intent.Error); + var settlement = Assert.Single(channel.Settlements); + Assert.True(settlement.Acknowledge); + Assert.False(settlement.Requeue); + } + + /// + /// Publishable terminal failures still serialize the handler error into the confirmed RPC JSON body. + /// + [Fact] + public async Task InvalidArticle_published_rpc_preserves_handler_error_text() + { + const string errorText = "ProviderFailure status=400: Service unavailable"; + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline( + new ControllableArticleWorkHandler + { + Outcome = ArticleWorkOutcome.InvalidArticle, + Error = errorText, + }, + publisher, + 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.InvalidArticle, outcome); + var intent = Assert.Single(publisher.Published); + Assert.Equal(errorText, intent.Error); + using var document = JsonDocument.Parse(ArticleWorkResponseWireProtocol.SerializeV1(intent)); + Assert.Equal("InvalidArticle", document.RootElement.GetProperty("outcome").GetString()); + Assert.Equal(errorText, document.RootElement.GetProperty("error").GetString()); + Assert.True(Assert.Single(channel.Settlements).Acknowledge); + } + + /// + /// Production handler rethrow through the delivery pipeline preserves FormatReason on the prepared error. + /// + [Fact] + public async Task Isolated_pipeline_retrieve_exception_prepares_format_reason_error() + { + await using var retention = CreateRetention(); + var thrown = new IOException("isolated pipeline socket closed"); + var handler = new ProviderArticleWorkHandler( + new ThrowingRetriever(thrown), + retention, + new NntpArticleParser("backfiller.test"), + sharedConfiguration: null, + serverId: 0, + NullLogger.Instance); + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline(handler, publisher, 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + Assert.Equal("IOException: isolated pipeline socket closed", pipeline.LastEscapedHandlerFailureError); + Assert.Equal(pipeline.LastEscapedHandlerFailureError, Assert.Single(publisher.Published).Error); + Assert.True(Assert.Single(channel.Settlements).Acknowledge); + Assert.False(Assert.Single(channel.Settlements).Requeue); + } + + /// + /// Production retrieve throw with a distinct inner exception preserves that context on the prepared error. + /// + [Fact] + public async Task Isolated_pipeline_retrieve_exception_preserves_distinct_inner() + { + await using var retention = CreateRetention(); + var inner = new SocketException((int)SocketError.TimedOut); + var thrown = new IOException("isolated retrieve failed", inner); + var handler = new ProviderArticleWorkHandler( + new ThrowingRetriever(thrown), + retention, + new NntpArticleParser("backfiller.test"), + sharedConfiguration: null, + serverId: 0, + NullLogger.Instance); + var channel = new FakeBackFillerRabbitMqChannel(1); + var publisher = new RecordingArticleWorkResponsePublisher(); + var pipeline = new ArticleWorkDeliveryPipeline(handler, publisher, 1024); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.UnexpectedFailure, outcome); + Assert.Equal( + "IOException: isolated retrieve failed <- " + nameof(SocketException) + ": " + inner.Message, + pipeline.LastEscapedHandlerFailureError); + Assert.Equal(pipeline.LastEscapedHandlerFailureError, Assert.Single(publisher.Published).Error); + } + + private static ArticleRetentionAuthority CreateRetention() => + new( + new BackFillerArticleRetentionRuntimeOptions(1024 * 1024, TimeSpan.FromHours(1), TimeSpan.FromMinutes(1), 256), + "backfiller.test", + 563, + TimeProvider.System, + NullLogger.Instance); + + private sealed class ThrowingRetriever : INntpArticleRetriever + { + private readonly Exception _exception; + + internal ThrowingRetriever(Exception exception) + { + _exception = exception; + } + + public Task RetrieveAsync( + ArticleWorkItem item, + int maxArticleBytes, + Func, ArticleRecordCreateResult> consumePayload, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(item); + ArgumentNullException.ThrowIfNull(consumePayload); + throw _exception; + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LinuxServiceInstallContractTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LinuxServiceInstallContractTests.cs index 62a72781..d57e24e2 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LinuxServiceInstallContractTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/LinuxServiceInstallContractTests.cs @@ -169,22 +169,11 @@ public void Windows_release_validation_rejects_the_linux_environment_example() /// Unit and environment-file lines that systemd treats as settings. /// File text. /// Non-empty lines whose first non-space character is not a comment marker. - private static List ActiveLines(string text) - { - var lines = new List(); - foreach (var raw in text.Split('\n')) - { - var line = raw.TrimEnd('\r').Trim(); - if (line.Length == 0 || line[0] is '#' or ';') - { - continue; - } - - lines.Add(line); - } - - return lines; - } + private static List ActiveLines(string text) => + text.Split('\n') + .Select(static raw => raw.TrimEnd('\r').Trim()) + .Where(static line => line.Length > 0 && line[0] is not '#' and not ';') + .ToList(); /// Markdown fenced blocks, without the fence markers. /// Markdown document. @@ -194,9 +183,8 @@ private static List FencedBlocks(string text) var blocks = new List(); var current = new System.Text.StringBuilder(); var inside = false; - foreach (var raw in text.Split('\n')) + foreach (var line in text.Split('\n').Select(static raw => raw.TrimEnd('\r'))) { - var line = raw.TrimEnd('\r'); if (line.TrimStart().StartsWith("```", StringComparison.Ordinal)) { if (inside) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Integration/BackFillerConcurrencyIntegrationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Integration/BackFillerConcurrencyIntegrationTests.cs index 13295c49..2ddd630a 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Integration/BackFillerConcurrencyIntegrationTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Integration/BackFillerConcurrencyIntegrationTests.cs @@ -13,10 +13,10 @@ public sealed class BackFillerConcurrencyIntegrationTests { /// Canonical valid article bytes used across concurrency scenarios. private static readonly byte[] Payload = ArticleWorkTestArticles.Valid(); - /// Provider connect failure requeues without ack; successful redelivery acks only the second delivery tag. + /// Provider connect failure ACKs with a failure reply; a later success ACKs a distinct delivery tag. [Fact] - public async Task Retryable_failures_then_redelivery_acks_only_the_second_attempt() + public async Task Provider_failure_acks_then_later_success_acks_second_delivery() { await using var harness = await BackFillerPipelineHarness.StartAsync(); var channel = new FakeBackFillerRabbitMqChannel(1); @@ -39,8 +39,8 @@ public async Task Retryable_failures_then_redelivery_acks_only_the_second_attemp static () => true, CancellationToken.None); Assert.Equal(ArticleWorkOutcome.ProviderFailure, providerFailure); - Assert.False(Assert.Single(channel.Settlements).Acknowledge); - Assert.True(channel.Settlements[0].Requeue); + Assert.True(Assert.Single(channel.Settlements).Acknowledge); + Assert.False(channel.Settlements[0].Requeue); harness.EnqueueArticle(Payload); var success = await harness.ProcessCanonicalAsync(channel, deliveryTag: 8); @@ -51,10 +51,10 @@ public async Task Retryable_failures_then_redelivery_acks_only_the_second_attemp Assert.Equal(7UL, channel.Settlements[0].DeliveryTag); Assert.Equal(1, harness.Retention.RetainedCount); } - /// Oversized retention requeues; a publish confirm failure after ingest NACK-requeues and does not ACK. + /// Oversized retention ACKs with RetentionRejected reply; a publish confirm failure after Success still NACK-requeues. [Fact] - public async Task Retention_rejection_requeues_and_publish_failure_after_success_nacks() + public async Task Retention_rejection_acks_and_publish_failure_after_success_nacks() { await using var tiny = await BackFillerPipelineHarness.StartAsync(); var oversized = ArticleWorkTestArticles.OversizedForOneMegabyteRetention(); @@ -62,9 +62,9 @@ public async Task Retention_rejection_requeues_and_publish_failure_after_success var rejectChannel = new FakeBackFillerRabbitMqChannel(1); var rejected = await tiny.ProcessCanonicalAsync(rejectChannel); Assert.Equal(ArticleWorkOutcome.RetentionRejected, rejected); - Assert.True(Assert.Single(rejectChannel.Settlements).Requeue); - Assert.False(rejectChannel.Settlements[0].Acknowledge); - Assert.Empty(tiny.PublishChannel.Publications); + Assert.False(Assert.Single(rejectChannel.Settlements).Requeue); + Assert.True(rejectChannel.Settlements[0].Acknowledge); + Assert.Single(tiny.PublishChannel.Publications); await using var publishFail = await BackFillerPipelineHarness.StartAsync(FakePublishConfirmBehavior.Nack); publishFail.EnqueueArticle(Payload); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpArticleWorkIntegrationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpArticleWorkIntegrationTests.cs index d4b5b1e4..5bae4364 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpArticleWorkIntegrationTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpArticleWorkIntegrationTests.cs @@ -137,6 +137,35 @@ public async Task ArticleNotFound_maps_to_ack_without_requeue() Assert.False(settlement.Requeue); } + /// + /// ARTICLE 451 follows the same terminal NotFound path as 430: ACK, RPC reply, no requeue. + /// + [Fact] + public async Task Article_451_maps_to_not_found_ack_without_requeue() + { + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + server.Respond(static _ => "451 article unavailable\r\n"); + factory.Enqueue(server); + var (_, publisher, pipeline, _) = CreatePipeline(factory); + var channel = new FakeBackFillerRabbitMqChannel(1); + + var outcome = await pipeline.ProcessAsync( + ArticleWorkTestDeliveries.Canonical(), + "Giganews", + channel, + static () => true, + CancellationToken.None); + + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, outcome); + Assert.NotEqual(ArticleWorkOutcome.ProviderFailure, outcome); + var intent = Assert.Single(publisher.Published); + Assert.Equal(ArticleWorkOutcome.ArticleNotFound, intent.Outcome); + var settlement = Assert.Single(channel.Settlements); + Assert.True(settlement.Acknowledge); + Assert.False(settlement.Requeue); + } + /// /// Connect/auth failures requeue without publishing a terminal Article Work response. /// @@ -170,10 +199,12 @@ public async Task Provider_and_auth_failures_requeue_without_terminal_response() Assert.Equal(ArticleWorkOutcome.ProviderFailure, downOutcome); Assert.Equal(ArticleWorkOutcome.ProviderFailure, authOutcome); - Assert.Empty(downPipeline.Publisher.Published); - Assert.Empty(authPipeline.Publisher.Published); - Assert.True(Assert.Single(downChannel.Settlements).Requeue); - Assert.True(Assert.Single(authChannel.Settlements).Requeue); + Assert.Equal(ArticleWorkOutcome.ProviderFailure, Assert.Single(downPipeline.Publisher.Published).Outcome); + Assert.Equal(ArticleWorkOutcome.ProviderFailure, Assert.Single(authPipeline.Publisher.Published).Outcome); + Assert.True(Assert.Single(downChannel.Settlements).Acknowledge); + Assert.False(Assert.Single(downChannel.Settlements).Requeue); + Assert.True(Assert.Single(authChannel.Settlements).Acknowledge); + Assert.False(Assert.Single(authChannel.Settlements).Requeue); } /// @@ -272,13 +303,13 @@ NntpSessionOptions.Default with Assert.Equal(ArticleWorkOutcome.RetentionRejected, outcome); Assert.Equal(ArticleRetentionKind.PayloadExceedsCapacity, handler.LastRetentionKind); Assert.NotEqual(ArticleWorkOutcome.ArticleNotFound, outcome); - Assert.Empty(publisher.Published); - Assert.False(Assert.Single(channel.Settlements).Acknowledge); - Assert.True(Assert.Single(channel.Settlements).Requeue); + Assert.Equal(ArticleWorkOutcome.RetentionRejected, Assert.Single(publisher.Published).Outcome); + Assert.True(Assert.Single(channel.Settlements).Acknowledge); + Assert.False(Assert.Single(channel.Settlements).Requeue); } /// - /// Unknown backbone name yields ProviderFailure and requeue. + /// Unknown backbone name yields ProviderFailure with ACK and a correlated failure reply. /// [Fact] @@ -306,7 +337,9 @@ public async Task Missing_provider_is_provider_failure() CancellationToken.None); Assert.Equal(ArticleWorkOutcome.ProviderFailure, outcome); - Assert.True(Assert.Single(channel.Settlements).Requeue); + Assert.Equal(ArticleWorkOutcome.ProviderFailure, Assert.Single(publisher.Published).Outcome); + Assert.True(Assert.Single(channel.Settlements).Acknowledge); + Assert.False(Assert.Single(channel.Settlements).Requeue); } /// Wires Giganews on loopback with scripted transport, retention, handler, publisher, and a 1024-byte delivery pipeline. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandBudgetTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandBudgetTests.cs index 5598db27..19a1457d 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandBudgetTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandBudgetTests.cs @@ -77,7 +77,7 @@ public async Task Stalled_write_is_command_timeout_and_not_reusable() } } - /// A stalled status read is reported as TimeoutException and retires the session. + /// A stalled status read is reported as TimeoutException with FormatReason text and retires the session. [Fact] public async Task Stalled_status_read_is_timeout_exception_and_not_reusable() { @@ -95,7 +95,8 @@ public async Task Stalled_status_read_is_timeout_exception_and_not_reusable() await statusCopied.Task.WaitAsync(TimeSpan.FromSeconds(5)); using var result = await download.WaitAsync(TimeSpan.FromSeconds(5)); Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); - Assert.Equal(nameof(TimeoutException), result.Reason); + Assert.StartsWith(nameof(TimeoutException) + ":", result.Reason, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(result.Reason[(nameof(TimeoutException) + ": ").Length..])); Assert.False(result.SessionReusable); } finally diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpConnectionIdentityCorrelationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpConnectionIdentityCorrelationTests.cs new file mode 100644 index 00000000..8c0b70ff --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpConnectionIdentityCorrelationTests.cs @@ -0,0 +1,217 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.BackFillerService.Nntp; +using VectorNNTPD.BackFillerService.Tests.TestDoubles; + +namespace VectorNNTPD.BackFillerService.Tests.Nntp +{ + /// + /// Stable physical assignment and retirement correlation. + /// + public sealed class NntpConnectionIdentityCorrelationTests + { + /// A newly constructed session receives a non-empty connection id that remains stable. + [Fact] + public void New_session_receives_stable_connection_id() + { + var session = new NntpProviderSession( + new BackFillerProviderDefinition("Giganews", "127.0.0.1", 119, false, null, null, 0, 1), + NntpSessionOptions.Default, + NullLogger.Instance, + connectionNumber: 1); + + Assert.NotEqual(Guid.Empty, session.ConnectionId); + Assert.Equal(session.ConnectionId, session.ConnectionId); + } + + /// Distinct physical session objects receive distinct connection ids even with the same pool slot. + [Fact] + public void Distinct_sessions_receive_distinct_connection_ids() + { + var provider = new BackFillerProviderDefinition("Eweka", "127.0.0.1", 119, false, null, null, 0, 2); + var first = new NntpProviderSession(provider, NntpSessionOptions.Default, NullLogger.Instance, 1); + var second = new NntpProviderSession(provider, NntpSessionOptions.Default, NullLogger.Instance, 1); + + Assert.NotEqual(first.ConnectionId, second.ConnectionId); + } + + /// + /// ARTICLE 430 keeps the session reusable; lease dispose does not emit lease-requested retirement. + /// + [Fact] + public async Task Article_430_does_not_retire_session() + { + var logger = new CollectingLogger(); + var factory = new ScriptedNntpTransportFactory(); + NntpSessionPoolTests.EnqueueReadyServers(factory, count: 1); + await using var pool = NntpSessionPoolTests.CreatePool(factory, max: 1, logger: logger); + await using var lease = await pool.AcquireAsync(CancellationToken.None); + var connectionId = lease.Session.ConnectionId; + using var result = await lease.Session.DownloadArticleAsync("", CancellationToken.None); + + Assert.Equal(ArticleRetrievalKind.ArticleNotFound, result.Kind); + Assert.True(result.SessionReusable); + Assert.Equal(connectionId, result.ConnectionId); + await lease.DisposeAsync(); + + Assert.DoesNotContain( + logger.Messages, + static m => m.Contains("cause=lease_requested", StringComparison.Ordinal)); + Assert.Equal(1, pool.LiveSessionCount); + } + + /// + /// ARTICLE 451 follows the same reusable NotFound path as 430 on Eweka (same lifecycle rules). + /// + [Fact] + public async Task Article_451_does_not_retire_session() + { + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + server.Respond(static command => + command.StartsWith("AUTHINFO", StringComparison.OrdinalIgnoreCase) + ? "281 authentication accepted\r\n" + : "451 article unavailable\r\n"); + factory.Enqueue(server); + await using var pool = NntpSessionPoolTests.CreatePool(factory, max: 1, backbone: "Eweka"); + await using var lease = await pool.AcquireAsync(CancellationToken.None); + using var result = await lease.Session.DownloadArticleAsync("", CancellationToken.None); + + Assert.Equal(ArticleRetrievalKind.ArticleNotFound, result.Kind); + Assert.Equal(NntpStatusCode.ArticleUnavailable, result.StatusCode); + Assert.True(result.SessionReusable); + Assert.Equal(lease.Session.ConnectionId, result.ConnectionId); + await lease.DisposeAsync(); + Assert.Equal(1, pool.LiveSessionCount); + } + + /// + /// ARTICLE 400 ProviderFailure marks the session non-reusable; lease dispose retires with matching connectionId. + /// Replacement does not exceed MaxSessions. + /// + [Fact] + public async Task Provider_failure_400_retires_with_matching_connection_id() + { + var logger = new CollectingLogger(); + var factory = new ScriptedNntpTransportFactory(); + var first = new ScriptedNntpServer(); + first.Respond(static command => + command.StartsWith("AUTHINFO", StringComparison.OrdinalIgnoreCase) + ? "281 authentication accepted\r\n" + : "400 Service unavailable\r\n"); + factory.Enqueue(first); + NntpSessionPoolTests.EnqueueReadyServers(factory, count: 1); + + await using var pool = NntpSessionPoolTests.CreatePool(factory, max: 1, logger: logger); + Guid connectionId; + await using (var lease = await pool.AcquireAsync(CancellationToken.None)) + { + connectionId = lease.Session.ConnectionId; + using var result = await lease.Session.DownloadArticleAsync("", CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); + Assert.Equal(NntpStatusCode.ServiceTemporarilyUnavailable, result.StatusCode); + Assert.False(result.SessionReusable); + Assert.Equal(connectionId, result.ConnectionId); + lease.Retire(); + } + + Assert.Contains( + logger.Messages, + m => m.Contains("NNTP session retired", StringComparison.Ordinal) + && m.Contains($"connectionId={connectionId:D}", StringComparison.Ordinal) + && m.Contains("cause=lease_requested", StringComparison.Ordinal) + && m.Contains("reason=lease requested retirement", StringComparison.Ordinal)); + + var replacement = await pool.AcquireAsync(CancellationToken.None); + Assert.NotEqual(connectionId, replacement.Session.ConnectionId); + Assert.True(pool.LiveSessionCount <= 1); + await replacement.DisposeAsync(); + } + + /// + /// Giganews-style ARTICLE 500 server error is ProviderFailure without lease retirement; pool capacity is unchanged. + /// + [Fact] + public async Task Article_500_server_error_does_not_retire_session() + { + var logger = new CollectingLogger(); + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + server.Respond(static command => + command.StartsWith("AUTHINFO", StringComparison.OrdinalIgnoreCase) + ? "281 authentication accepted\r\n" + : "500 server error\r\n"); + factory.Enqueue(server); + + await using var pool = NntpSessionPoolTests.CreatePool(factory, max: 1, logger: logger); + await using var lease = await pool.AcquireAsync(CancellationToken.None); + var connectionId = lease.Session.ConnectionId; + using var result = await lease.Session.DownloadArticleAsync("", CancellationToken.None); + + Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); + Assert.Equal(NntpStatusCode.CommandUnknown, result.StatusCode); + Assert.Equal("server error", result.Reason); + Assert.True(result.SessionReusable); + Assert.Equal(connectionId, result.ConnectionId); + await lease.DisposeAsync(); + + Assert.DoesNotContain( + logger.Messages, + static m => m.Contains("cause=lease_requested", StringComparison.Ordinal)); + Assert.Equal(1, pool.LiveSessionCount); + } + + /// + /// Transport IOException during ARTICLE marks the session non-reusable and retires on lease dispose. + /// + [Fact] + public async Task Transport_failure_retires_session() + { + var logger = new CollectingLogger(); + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer { CompleteAfterResponse = true }; + server.Respond(static command => + command.StartsWith("AUTHINFO", StringComparison.OrdinalIgnoreCase) + ? "281 authentication accepted\r\n" + : "220 article follows\r\nFrom: a@b\r\n\r\nbody\r\n"); + factory.Enqueue(server); + NntpSessionPoolTests.EnqueueReadyServers(factory, count: 1); + + await using var pool = NntpSessionPoolTests.CreatePool(factory, max: 1, logger: logger); + Guid connectionId; + await using (var lease = await pool.AcquireAsync(CancellationToken.None)) + { + connectionId = lease.Session.ConnectionId; + using var result = await lease.Session.DownloadArticleAsync("", CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); + Assert.False(result.SessionReusable); + Assert.Equal(connectionId, result.ConnectionId); + lease.Retire(); + } + + Assert.Contains( + logger.Messages, + m => m.Contains("cause=lease_requested", StringComparison.Ordinal) + && m.Contains($"connectionId={connectionId:D}", StringComparison.Ordinal)); + Assert.True(pool.LiveSessionCount <= 1); + } + + /// Pool dispose retires remaining sessions under pool_shutting_down without exceeding capacity. + [Fact] + public async Task Shutdown_retires_with_pool_shutting_down_cause() + { + var logger = new CollectingLogger(); + var factory = new ScriptedNntpTransportFactory(); + NntpSessionPoolTests.EnqueueReadyServers(factory, count: 1); + var pool = NntpSessionPoolTests.CreatePool(factory, max: 1, backbone: "Eweka", logger: logger); + await pool.EnsureDesiredSessionsAsync(CancellationToken.None); + Assert.Equal(1, pool.LiveSessionCount); + await pool.DisposeAsync(); + + Assert.Contains( + logger.Messages, + static m => m.Contains("cause=pool_shutting_down", StringComparison.Ordinal) + && m.Contains("NNTP session retired", StringComparison.Ordinal)); + Assert.Equal(0, pool.LiveSessionCount); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpExceptionDiagnosticsTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpExceptionDiagnosticsTests.cs new file mode 100644 index 00000000..cfd58fd0 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpExceptionDiagnosticsTests.cs @@ -0,0 +1,122 @@ +using System.Net.Sockets; +using VectorNNTPD.BackFillerService.Nntp; + +namespace VectorNNTPD.BackFillerService.Tests.Nntp +{ + /// + /// Unit coverage for bounds, inners, and fallbacks. + /// + public sealed class NntpExceptionDiagnosticsTests + { + /// + /// Non-empty messages become {Type}: {message} with no inner. + /// + [Fact] + public void FormatReason_includes_type_and_trimmed_message() + { + var reason = NntpExceptionDiagnostics.FormatReason(new IOException(" connection reset ")); + Assert.Equal("IOException: connection reset", reason); + } + + /// + /// Blank messages fall back to the type name alone when no useful inner is present. + /// + [Theory] + [InlineData("")] + [InlineData(" ")] + public void FormatReason_blank_message_returns_type_name(string message) + { + Assert.Equal(nameof(BlankMessageException), NntpExceptionDiagnostics.FormatReason(new BlankMessageException(message))); + } + + /// + /// A useful inner is appended when the outer message does not already contain it. + /// + [Fact] + public void FormatReason_appends_useful_inner_exception() + { + var inner = new SocketException((int)SocketError.TimedOut); + var reason = NntpExceptionDiagnostics.FormatReason(new IOException("transport failed", inner)); + Assert.Equal("IOException: transport failed <- " + nameof(SocketException) + ": " + inner.Message, reason); + } + + /// + /// When the outer message already embeds the inner text (typical transport wrappers), the inner is not repeated. + /// + [Fact] + public void FormatReason_skips_inner_when_message_already_embedded() + { + var inner = new SocketException((int)SocketError.ConnectionReset); + var outer = new IOException("Unable to read data from the transport connection: " + inner.Message, inner); + var reason = NntpExceptionDiagnostics.FormatReason(outer); + Assert.Equal("IOException: Unable to read data from the transport connection: " + inner.Message, reason); + Assert.DoesNotContain(" <- ", reason, StringComparison.Ordinal); + } + + /// + /// An empty outer message still surfaces a useful inner type and message. + /// + [Fact] + public void FormatReason_blank_outer_preserves_useful_inner() + { + var inner = new SocketException((int)SocketError.ConnectionRefused); + var reason = NntpExceptionDiagnostics.FormatReason(new IOException(string.Empty, inner)); + Assert.Equal("IOException <- " + nameof(SocketException) + ": " + inner.Message, reason); + } + + /// + /// Chains that exceed the 256-character budget are truncated with a trailing marker. + /// + [Fact] + public void FormatReason_truncates_long_chain_at_256_characters_with_marker() + { + var inner = new InvalidOperationException(new string('y', 200)); + var outer = new IOException(new string('x', 200), inner); + var reason = NntpExceptionDiagnostics.FormatReason(outer); + Assert.Equal(256, reason.Length); + Assert.EndsWith("...", reason, StringComparison.Ordinal); + Assert.StartsWith("IOException: ", reason, StringComparison.Ordinal); + Assert.Contains(" <- ", reason, StringComparison.Ordinal); + } + + /// + /// A long outer-only message is truncated to 256 characters with a marker. + /// + [Fact] + public void FormatReason_truncates_long_messages_at_256_characters() + { + var longMessage = new string('x', 400); + var reason = NntpExceptionDiagnostics.FormatReason(new IOException(longMessage)); + Assert.Equal(256, reason.Length); + Assert.StartsWith("IOException: ", reason, StringComparison.Ordinal); + Assert.EndsWith("...", reason, StringComparison.Ordinal); + } + + /// + /// Deep chains stop after three inners so formatting stays bounded. + /// + [Fact] + public void FormatReason_limits_inner_depth() + { + Exception chain = new InvalidOperationException("level-5"); + chain = new InvalidOperationException("level-4", chain); + chain = new InvalidOperationException("level-3", chain); + chain = new InvalidOperationException("level-2", chain); + chain = new InvalidOperationException("level-1", chain); + + var reason = NntpExceptionDiagnostics.FormatReason(chain); + Assert.Equal( + "InvalidOperationException: level-1 <- InvalidOperationException: level-2 <- InvalidOperationException: level-3 <- InvalidOperationException: level-4", + reason); + Assert.DoesNotContain("level-5", reason, StringComparison.Ordinal); + } + + private sealed class BlankMessageException : Exception + { + public BlankMessageException(string message) + : base(message) + { + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs index 8ec39e4f..609c27f5 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs @@ -94,11 +94,12 @@ public async Task Connect_failure_and_timeout_are_provider_failures() var result = await session.ConnectAsync(factory, CancellationToken.None); Assert.Equal(ArticleRetrievalKind.ProviderFailure, result!.Kind); + Assert.Equal("IOException: refused", result.Reason); await session.DisposeAsync(); } /// - /// A real from connect is a non-reusable provider failure whose reason is the exception type name. + /// A real from connect is a non-reusable provider failure whose reason includes type and message. /// [Fact] public async Task Connect_socket_exception_is_a_non_reusable_provider_failure() @@ -113,7 +114,8 @@ public async Task Connect_socket_exception_is_a_non_reusable_provider_failure() Assert.NotNull(result); Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); Assert.False(result.SessionReusable); - Assert.Equal(nameof(SocketException), result.Reason); + Assert.StartsWith(nameof(SocketException) + ":", result.Reason, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(result.Reason[(nameof(SocketException) + ": ").Length..])); Assert.Equal(NntpSessionState.Retiring, session.State); await session.DisposeAsync(); } @@ -212,7 +214,7 @@ public async Task Authinfo_pass_write_fault_still_clears_the_buffer() { var result = await session.ConnectAsync(factory, CancellationToken.None); Assert.Equal(ArticleRetrievalKind.ProviderFailure, result!.Kind); - Assert.Equal(nameof(IOException), result.Reason); + Assert.Equal("IOException: AUTHINFO PASS write fault", result.Reason); Assert.DoesNotContain(Password, result.Reason, StringComparison.Ordinal); Assert.Equal("AUTHINFO USER acct", server.Commands[1]); Assert.DoesNotContain( @@ -682,7 +684,7 @@ public async Task Starttls_log_failure_before_handshake_disposes_the_plaintext_t Assert.NotNull(result); Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); Assert.False(result.SessionReusable); - Assert.Equal("InvalidOperationException", result.Reason); + Assert.Equal("InvalidOperationException: TLS handshake starting", result.Reason); Assert.Null(SessionField(session, "_stream")); Assert.Null(SessionField(session, "_reader")); Assert.Equal(1, server.ClientStreamDisposeCount); @@ -721,7 +723,7 @@ public async Task Starttls_cleanup_failure_preserves_the_pre_handshake_exception Assert.NotNull(result); Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); Assert.False(result.SessionReusable); - Assert.Equal("InvalidOperationException", result.Reason); + Assert.Equal("InvalidOperationException: TLS handshake starting", result.Reason); Assert.Null(SessionField(session, "_stream")); Assert.Null(SessionField(session, "_reader")); Assert.Equal(1, server.ClientStreamDisposeCount); @@ -878,6 +880,28 @@ public async Task Article_not_found_is_reusable() using var result = await session.DownloadArticleAsync("", CancellationToken.None); Assert.Equal(ArticleRetrievalKind.ArticleNotFound, result.Kind); + Assert.Equal(430, result.StatusCode); + Assert.True(result.SessionReusable); + await session.DisposeAsync(); + } + + /// + /// ARTICLE 451 is classified as ArticleNotFound (not ProviderFailure) and keeps the session reusable. + /// + [Fact] + public async Task Article_451_is_article_not_found_and_reusable() + { + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + server.Respond(static _ => "451 article unavailable\r\n"); + factory.Enqueue(server); + var session = CreateSession(); + Assert.Null(await session.ConnectAsync(factory, CancellationToken.None)); + + using var result = await session.DownloadArticleAsync("", CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ArticleNotFound, result.Kind); + Assert.Equal(NntpStatusCode.ArticleUnavailable, result.StatusCode); + Assert.NotEqual(ArticleRetrievalKind.ProviderFailure, result.Kind); Assert.True(result.SessionReusable); await session.DisposeAsync(); } @@ -1238,15 +1262,12 @@ NntpSessionOptions.Default with } /// - /// 4xx/5xx ARTICLE statuses are ProviderFailure, not ArticleNotFound, and retire the session. + /// Connection-terminating or authority ARTICLE statuses are ProviderFailure (not NotFound) and retire the session. /// - [Theory] [InlineData("400 service unavailable")] - [InlineData("503 program fault")] [InlineData("502 not permitted")] - [InlineData("500 unknown command")] - public async Task Provider_error_status_is_not_article_not_found(string status) + public async Task Provider_terminating_status_is_not_article_not_found_and_retires(string status) { var factory = new ScriptedNntpTransportFactory(); var server = new ScriptedNntpServer(); @@ -1262,6 +1283,76 @@ public async Task Provider_error_status_is_not_article_not_found(string status) await session.DisposeAsync(); } + /// + /// Single-line ARTICLE 403/500/501/503 are ProviderFailure (not NotFound) and keep the session reusable. + /// + [Theory] + [InlineData("403 internal fault", NntpStatusCode.InternalFault)] + [InlineData("500 server error", NntpStatusCode.CommandUnknown)] + [InlineData("500 unknown command", NntpStatusCode.CommandUnknown)] + [InlineData("501 syntax error", NntpStatusCode.CommandSyntaxError)] + [InlineData("503 program fault", NntpStatusCode.ProgramFault)] + public async Task Article_level_5xx_is_provider_failure_and_reusable(string status, int expectedCode) + { + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + server.Respond(_ => status + "\r\n"); + factory.Enqueue(server); + var session = CreateSession(); + Assert.Null(await session.ConnectAsync(factory, CancellationToken.None)); + + using var result = await session.DownloadArticleAsync("", CancellationToken.None); + Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); + Assert.NotEqual(ArticleRetrievalKind.ArticleNotFound, result.Kind); + Assert.Equal(expectedCode, result.StatusCode); + Assert.True(result.SessionReusable); + Assert.True(session.IsReusable); + await session.DisposeAsync(); + } + + /// + /// Giganews-style ARTICLE 500 server error leaves the connection synchronized for a later Found ARTICLE. + /// + [Fact] + public async Task Giganews_500_server_error_allows_subsequent_article_on_same_session() + { + var factory = new ScriptedNntpTransportFactory(); + var server = new ScriptedNntpServer(); + var articleOrdinal = 0; + server.Respond(_ => + { + var n = Interlocked.Increment(ref articleOrdinal); + return n == 1 + ? "500 server error\r\n" + : "220 0 article follows\r\nFrom: a@b\r\n\r\nbody\r\n.\r\n"; + }); + factory.Enqueue(server); + var session = CreateSession(); + Assert.Null(await session.ConnectAsync(factory, CancellationToken.None)); + var connectionId = session.ConnectionId; + + using (var failed = await session.DownloadArticleAsync("", CancellationToken.None)) + { + Assert.Equal(ArticleRetrievalKind.ProviderFailure, failed.Kind); + Assert.Equal(NntpStatusCode.CommandUnknown, failed.StatusCode); + Assert.Equal("server error", failed.Reason); + Assert.True(failed.SessionReusable); + Assert.Equal(connectionId, failed.ConnectionId); + } + + using (var found = await session.DownloadArticleAsync("", CancellationToken.None)) + { + Assert.Equal(ArticleRetrievalKind.ArticleRetrieved, found.Kind); + Assert.Equal(NntpStatusCode.ArticleFollows, found.StatusCode); + Assert.True(found.SessionReusable); + Assert.Equal(connectionId, found.ConnectionId); + Assert.NotNull(found.Article); + } + + Assert.True(session.IsReusable); + await session.DisposeAsync(); + } + /// /// Single leading dot on body lines is destuffed in returned article bytes. /// @@ -1441,7 +1532,8 @@ private static async Task AssertDisposeDuringArticlePayloadReadAsync(bool useCal Assert.Equal(ArticleRetrievalKind.ProviderFailure, finished.Kind); Assert.False(finished.SessionReusable); Assert.Null(finished.Article); - Assert.Equal(nameof(InvalidOperationException), finished.Reason); + Assert.StartsWith(nameof(InvalidOperationException) + ":", finished.Reason, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(finished.Reason[(nameof(InvalidOperationException) + ": ").Length..])); Assert.DoesNotContain("NullReference", finished.Reason ?? string.Empty, StringComparison.Ordinal); Assert.DoesNotContain("ArgumentNull", finished.Reason ?? string.Empty, StringComparison.Ordinal); Assert.Equal(0, Volatile.Read(ref consumed)); @@ -1526,7 +1618,8 @@ private static async Task AssertDisposeBetweenArticleWriteAndFlushAsync(bool use Assert.Equal(ArticleRetrievalKind.ProviderFailure, finished.Kind); Assert.False(finished.SessionReusable); Assert.Null(finished.Article); - Assert.Equal(nameof(ObjectDisposedException), finished.Reason); + Assert.StartsWith(nameof(ObjectDisposedException) + ":", finished.Reason, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(finished.Reason[(nameof(ObjectDisposedException) + ": ").Length..])); Assert.Equal(0, Volatile.Read(ref consumed)); Assert.Null(SessionField(session, "_stream")); Assert.Null(SessionField(session, "_reader")); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpSessionPoolTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpSessionPoolTests.cs index cac60ad1..5cd78ac1 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpSessionPoolTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpSessionPoolTests.cs @@ -888,7 +888,8 @@ public async Task Socket_exception_during_connect_releases_capacity_and_the_next () => pool.AcquireAsync(CancellationToken.None)); Assert.Equal(ArticleRetrievalKind.ProviderFailure, failure.Kind); - Assert.Equal(nameof(SocketException), failure.Message); + Assert.StartsWith(nameof(SocketException) + ":", failure.Message, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(failure.Message[(nameof(SocketException) + ": ").Length..])); AssertNoReservedCapacity(pool); factory.ConnectException = null; @@ -1159,7 +1160,8 @@ public async Task Logger_failure_during_connect_failure_preserves_the_connect_ex var failure = await Assert.ThrowsAsync( () => pool.AcquireAsync(CancellationToken.None)); Assert.Equal(ArticleRetrievalKind.ProviderFailure, failure.Kind); - Assert.Equal(nameof(SocketException), failure.Message); + Assert.StartsWith(nameof(SocketException) + ":", failure.Message, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(failure.Message[(nameof(SocketException) + ": ").Length..])); AssertNoReservedCapacity(pool); factory.ConnectException = null; @@ -2227,19 +2229,18 @@ public async Task Disposal_before_issue_does_not_lease_the_retired_created_sessi } /// - /// A keepalive retirement that has left the idle list, but not yet finished dispose, cannot be leased or returned to the queue. - /// The pause yields, so the caller that released DATE can observe that window and then let dispose finish. - /// The replacement lease stays usable after the retired session is disposed, and its return does not requeue the retired one. + /// Capacity stays reserved until transport dispose finishes, so maintenance and acquire cannot open another TCP + /// while the retiring session is still disposing. After dispose completes, replacement proceeds once. /// [Fact] - public async Task Retirement_overlapping_acquire_does_not_lease_or_requeue_the_retired_session() + public async Task Retirement_holds_capacity_until_dispose_completes() { var clock = new KeepAliveDelaySource(); var factory = new ScriptedNntpTransportFactory(); var retiredServer = ReadyServer(); - var survivorServer = ReadyServer(); + var replacementServer = ReadyServer(); factory.Enqueue(retiredServer); - factory.Enqueue(survivorServer); + factory.Enqueue(replacementServer); await using var pool = CreatePool(factory, max: 1, keepAliveSeconds: 30, time: clock); var retirementEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var releaseRetirement = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -2253,6 +2254,7 @@ public async Task Retirement_overlapping_acquire_does_not_lease_or_requeue_the_r { await pool.EnsureDesiredSessionsAsync(CancellationToken.None); Assert.Equal(1, pool.IdleSessionCount); + Assert.Single(factory.ConnectAttempts); var releaseKeepAlive = await clock.WaitForKeepAliveAsync(); releaseKeepAlive(); @@ -2260,45 +2262,187 @@ public async Task Retirement_overlapping_acquire_does_not_lease_or_requeue_the_r Assert.Equal(0, pool.IdleSessionCount); Assert.Equal(0, pool.LiveSessionCount); Assert.Equal(0, pool.ActiveLeaseCount); - Assert.Equal(0, CreatedSessionCount(pool)); - Assert.Equal(0, ReservedConnectionCount(pool)); + Assert.Equal(1, CreatedSessionCount(pool)); + Assert.Equal(1, ReservedConnectionCount(pool)); Assert.Equal(0, retiredServer.ClientStreamDisposeCount); - var lease = await pool.AcquireAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); - var survivor = lease.Session; - Assert.Equal(NntpSessionState.Ready, survivor.State); - Assert.Equal(0, pool.IdleSessionCount); + // Maintenance must not open another provider TCP while the retiring transport is still disposing. + // Concurrent acquire-while-disposing is covered by Concurrent_retirement_does_not_allow_connects_above_max_sessions + // (lease retire holds the permit until dispose finishes; this DATE path holds capacity the same way). + await pool.EnsureDesiredSessionsAsync(CancellationToken.None); + Assert.Single(factory.ConnectAttempts); + Assert.Equal(1, CreatedSessionCount(pool)); + Assert.Equal(1, ReservedConnectionCount(pool)); + + releaseRetirement.TrySetResult(); + await WaitUntilAsync(() => retiredServer.ClientStreamDisposeCount == 1); + await WaitUntilAsync(() => CreatedSessionCount(pool) == 0 || pool.LiveSessionCount == 1); + + await using var lease = await pool.AcquireAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(2, factory.ConnectAttempts.Count); + Assert.Equal(NntpSessionState.Ready, lease.Session.State); Assert.Equal(1, pool.LiveSessionCount); - Assert.Equal(1, pool.ActiveLeaseCount); Assert.Equal(1, CreatedSessionCount(pool)); Assert.Equal(1, ReservedConnectionCount(pool)); + Assert.Equal(0, replacementServer.ClientStreamDisposeCount); + } + finally + { + releaseRetirement.TrySetResult(); + pool.BeforeDetachedSessionDispose = null; + } + } + + /// + /// Concurrent retirements at MaxSessions keep every disposing reservation until dispose finishes, + /// so maintenance cannot initiate connects above the configured limit. + /// + [Fact] + public async Task Concurrent_retirement_does_not_allow_connects_above_max_sessions() + { + var factory = new ScriptedNntpTransportFactory(); + EnqueueReadyServers(factory, count: 4); + await using var pool = CreatePool(factory, max: 2); + var disposeEntered = 0; + var bothEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseDispose = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + pool.BeforeDetachedSessionDispose = () => + { + if (Interlocked.Increment(ref disposeEntered) == 2) + { + bothEntered.TrySetResult(); + } + + return releaseDispose.Task; + }; + + try + { + await pool.EnsureDesiredSessionsAsync(CancellationToken.None); Assert.Equal(2, factory.ConnectAttempts.Count); + var first = await pool.AcquireAsync(CancellationToken.None); + var second = await pool.AcquireAsync(CancellationToken.None); + first.Retire(); + second.Retire(); + var returningFirst = first.DisposeAsync().AsTask(); + var returningSecond = second.DisposeAsync().AsTask(); + await bothEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); - await lease.DisposeAsync(); - Assert.Equal(1, pool.IdleSessionCount); - Assert.Equal(1, pool.LiveSessionCount); - Assert.Equal(0, pool.ActiveLeaseCount); - Assert.Equal(NntpSessionState.Ready, survivor.State); + Assert.Equal(0, pool.LiveSessionCount); + Assert.Equal(2, CreatedSessionCount(pool)); + Assert.Equal(2, ReservedConnectionCount(pool)); Assert.Equal(2, factory.ConnectAttempts.Count); - releaseRetirement.TrySetResult(); - await WaitUntilAsync(() => retiredServer.ClientStreamDisposeCount == 1); - Assert.Equal(NntpSessionState.Ready, survivor.State); - Assert.Equal(0, survivorServer.ClientStreamDisposeCount); - Assert.Equal(1, pool.IdleSessionCount); - Assert.Equal(1, pool.LiveSessionCount); + await pool.EnsureDesiredSessionsAsync(CancellationToken.None); + var waiting = pool.AcquireAsync(CancellationToken.None); + Assert.False(waiting.IsCompleted); + Assert.Equal(2, factory.ConnectAttempts.Count); + Assert.Equal(2, CreatedSessionCount(pool)); + + releaseDispose.TrySetResult(); + await Task.WhenAll(returningFirst, returningSecond).WaitAsync(TimeSpan.FromSeconds(5)); + await using var replacement = await waiting.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.True(factory.ConnectAttempts.Count >= 3); + Assert.True(factory.ConnectAttempts.Count <= 4); + Assert.True(pool.LiveSessionCount <= 2); + Assert.True(CreatedSessionCount(pool) <= 2); + Assert.True(ReservedConnectionCount(pool) <= 2); + } + finally + { + releaseDispose.TrySetResult(); + pool.BeforeDetachedSessionDispose = null; + } + } + + /// + /// An unexpected transport dispose fault still releases created-count and connection-number capacity exactly once + /// so the pool can recover without permanently losing the slot or leaving the session selectable. + /// + [Fact] + public async Task Unexpected_dispose_fault_releases_capacity_and_allows_recovery() + { + var factory = new ScriptedNntpTransportFactory(); + var faulting = ReadyServer(); + faulting.FaultClientDisposeUnexpected = true; + factory.Enqueue(faulting); + await using var pool = CreatePool(factory, max: 1); + await pool.EnsureDesiredSessionsAsync(CancellationToken.None); + + var lease = await pool.AcquireAsync(CancellationToken.None); + lease.Retire(); + var fault = await Assert.ThrowsAsync(() => lease.DisposeAsync().AsTask()); + Assert.Equal("teardown fault", fault.Message); + Assert.Equal(0, pool.LiveSessionCount); + Assert.Equal(0, pool.IdleSessionCount); + Assert.Equal(0, pool.ActiveLeaseCount); + Assert.Equal(0, CreatedSessionCount(pool)); + Assert.Equal(0, ReservedConnectionCount(pool)); + Assert.Equal(1, faulting.ClientStreamDisposeCount); + + EnqueueReadyServers(factory, count: 1); + await using var recovered = await pool.AcquireAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(NntpSessionState.Ready, recovered.Session.State); + Assert.Equal(1, recovered.Session.ConnectionNumber); + Assert.Equal(1, pool.LiveSessionCount); + Assert.Equal(1, CreatedSessionCount(pool)); + Assert.Equal(1, ReservedConnectionCount(pool)); + // Initial connect plus recovery; an unawaited replenish may also attempt once before the server was enqueued. + Assert.True(factory.ConnectAttempts.Count >= 2); + } + + /// + /// A failed connect that reaches live registration keeps its reservation until dispose finishes, + /// then releases it exactly once so a later attempt can reuse slot 1 without exceeding MaxSessions. + /// + [Fact] + public async Task Failed_establishment_holds_capacity_until_dispose_then_releases_once() + { + var factory = new ScriptedNntpTransportFactory + { + ConnectException = new IOException("establishment refused"), + }; + var disposeEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseDispose = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var pool = CreatePool(factory, max: 1); + pool.BeforeDetachedSessionDispose = () => + { + disposeEntered.TrySetResult(); + return releaseDispose.Task; + }; + + try + { + var failed = pool.AcquireAsync(CancellationToken.None); + await disposeEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Single(factory.ConnectAttempts); Assert.Equal(1, CreatedSessionCount(pool)); Assert.Equal(1, ReservedConnectionCount(pool)); + Assert.Equal(0, pool.LiveSessionCount); + Assert.False(failed.IsCompleted); - var reused = await pool.AcquireAsync(CancellationToken.None); - Assert.Same(survivor, reused.Session); - Assert.Equal(NntpSessionState.Ready, reused.Session.State); + await pool.EnsureDesiredSessionsAsync(CancellationToken.None); + Assert.Single(factory.ConnectAttempts); + Assert.Equal(1, CreatedSessionCount(pool)); + + releaseDispose.TrySetResult(); + var connectFault = await Assert.ThrowsAsync( + () => failed.WaitAsync(TimeSpan.FromSeconds(5))); + Assert.Equal(ArticleRetrievalKind.ProviderFailure, connectFault.Kind); + AssertNoReservedCapacity(pool); + + factory.ConnectException = null; + EnqueueReadyServers(factory, count: 1); + await using var recovered = await pool.AcquireAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(1, recovered.Session.ConnectionNumber); + Assert.Equal(NntpSessionState.Ready, recovered.Session.State); Assert.Equal(2, factory.ConnectAttempts.Count); - await reused.DisposeAsync(); + Assert.Equal(1, CreatedSessionCount(pool)); + Assert.Equal(1, ReservedConnectionCount(pool)); } finally { - releaseRetirement.TrySetResult(); + releaseDispose.TrySetResult(); pool.BeforeDetachedSessionDispose = null; } } @@ -2794,7 +2938,8 @@ public async Task Drain_timeout_aborts_a_blocked_article_write_without_quit() await pool.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5)); using var result = await download.WaitAsync(TimeSpan.FromSeconds(5)); Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); - Assert.Equal(nameof(IOException), result.Reason); + Assert.StartsWith("IOException:", result.Reason, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(result.Reason["IOException: ".Length..])); Assert.Equal(1, gate.MaxWriteDepth); Assert.Equal(NntpSessionState.Closed, lease.Session.State); Assert.Equal(1, server.ClientStreamDisposeCount); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs index a5e1bfef..5065a594 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs @@ -198,7 +198,8 @@ public async Task Active_command_during_deadline_expiry_unwinds_without_per_sess AssertWithinCeiling(elapsed); Assert.Equal(ArticleRetrievalKind.ProviderFailure, result.Kind); - Assert.Equal(nameof(IOException), result.Reason); + Assert.StartsWith("IOException:", result.Reason, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(result.Reason["IOException: ".Length..])); Assert.Equal(1, gate.MaxWriteDepth); Assert.Equal(NntpSessionState.Closed, busy.Session.State); Assert.Equal(1, articleServer.ClientStreamDisposeCount); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcLifecycleContractTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcLifecycleContractTests.cs new file mode 100644 index 00000000..b0968b04 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcLifecycleContractTests.cs @@ -0,0 +1,118 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Deterministic correlation / waiter / drain contracts for the stress RPC client lifecycle. + /// + public sealed class ArticleWorkRpcLifecycleContractTests + { + [Fact] + public void Waiter_registration_before_publish_prevents_immediate_reply_race() + { + var pending = new ConcurrentDictionary(StringComparer.Ordinal); + const string correlationId = "corr-race"; + + // Same order as ArticleWorkStressClient.RequestAsync: register, then "publish". + Assert.True(pending.TryAdd(correlationId, 0)); + var published = true; + Assert.True(published); + Assert.True(pending.ContainsKey(correlationId)); + } + + [Fact] + public void Concurrent_correlations_complete_independent_waiters() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + tracker.OnPublished(); + tracker.OnPublished(); + + tracker.OnReply(callerStillWaiting: true); + Assert.Equal(1, tracker.WaitingCount); + Assert.Equal(1, tracker.BrokerOutstandingCount); + + tracker.OnReply(callerStillWaiting: true); + Assert.Equal(0, tracker.WaitingCount); + Assert.Equal(0, tracker.BrokerOutstandingCount); + } + + [Fact] + public void Caller_timeout_does_not_clear_broker_outstanding() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + tracker.OnPublished(); + tracker.OnCallerTimeout(); + Assert.Equal(0, tracker.WaitingCount); + Assert.Equal(1, tracker.BrokerOutstandingCount); + } + + [Fact] + public void Late_reply_after_timeout_is_accounted_without_corrupting_other_waiters() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + tracker.OnPublished(); + tracker.OnPublished(); + tracker.OnCallerTimeout(); + + tracker.OnReply(callerStillWaiting: false); + Assert.Equal(1, tracker.WaitingCount); + Assert.Equal(1, tracker.BrokerOutstandingCount); + Assert.Equal(1, tracker.LateRepliesAfterTimeout); + + tracker.OnReply(callerStillWaiting: true); + Assert.Equal(0, tracker.WaitingCount); + Assert.Equal(0, tracker.BrokerOutstandingCount); + Assert.Equal(1, tracker.LateRepliesAfterTimeout); + } + + [Fact] + public void Bounded_drain_reports_unresolved_when_broker_outstanding_remains() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + tracker.OnPublished(); + tracker.OnCallerTimeout(); + + var unresolved = tracker.BrokerOutstandingCount; + Assert.Equal(1, unresolved); + // Dispose path records UnresolvedAtDispose when drain expires with outstanding > 0. + Assert.True(unresolved > 0); + } + + [Fact] + public void Stress_work_requests_do_not_carry_per_message_expiration() + { + var props = ArticleWorkStressClient.CreateRequestProperties( + "corr-ttl", + "reply-ttl", + Guid.Parse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")); + Assert.True(string.IsNullOrEmpty(props.Expiration)); + } + + [Fact] + public void Stress_summary_surfaces_reply_lifecycle_counters() + { + var metrics = new StressMetrics + { + RunName = "lifecycle-counters", + Backbone = "Eweka", + ConfiguredConcurrency = 75, + ConfiguredMaxInFlight = 75, + }; + metrics.SteadyStateStartTimestamp = System.Diagnostics.Stopwatch.GetTimestamp(); + metrics.SteadyStateEndTimestamp = metrics.SteadyStateStartTimestamp + + System.Diagnostics.Stopwatch.Frequency; + + metrics.SetReplyLifecycleCounters( + lateRepliesAfterTimeout: 3, + unresolvedAtDispose: 21, + brokerOutstandingAtDrainEnd: 21); + + var summary = metrics.BuildSummary(); + Assert.Equal(3, summary.LateRepliesAfterTimeout); + Assert.Equal(21, summary.UnresolvedAtDispose); + Assert.Equal(21, summary.BrokerOutstandingAtDrainEnd); + Assert.Contains("late=3", summary.FormatTableRow(), StringComparison.Ordinal); + Assert.Contains("unresolved=21", summary.FormatTableRow(), StringComparison.Ordinal); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTracker.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTracker.cs new file mode 100644 index 00000000..a6fc8dd4 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTracker.cs @@ -0,0 +1,68 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Separates caller waiters from broker-outstanding RPC correlations for exclusive ReplyTo lifetime. + /// + /// + /// Used by unit tests to lock the lifecycle contract also implemented inside + /// . Timeout clears the waiter without clearing broker outstanding. + /// + internal sealed class ArticleWorkRpcOutstandingTracker + { + private int _waiting; + private int _brokerOutstanding; + private int _lateReplies; + + public int WaitingCount => Volatile.Read(ref _waiting); + + public int BrokerOutstandingCount => Volatile.Read(ref _brokerOutstanding); + + public int LateRepliesAfterTimeout => Volatile.Read(ref _lateReplies); + + public void OnPublished() + { + Interlocked.Increment(ref _waiting); + Interlocked.Increment(ref _brokerOutstanding); + } + + public void OnCallerTimeout() + { + var waiting = Interlocked.Decrement(ref _waiting); + if (waiting < 0) + { + Interlocked.Exchange(ref _waiting, 0); + throw new InvalidOperationException("Caller timeout decremented waiting below zero."); + } + } + + public void OnReply(bool callerStillWaiting) + { + if (callerStillWaiting) + { + var waiting = Interlocked.Decrement(ref _waiting); + if (waiting < 0) + { + Interlocked.Exchange(ref _waiting, 0); + throw new InvalidOperationException("Reply decremented waiting below zero."); + } + } + else + { + Interlocked.Increment(ref _lateReplies); + } + + var outstanding = Interlocked.Decrement(ref _brokerOutstanding); + if (outstanding < 0) + { + Interlocked.Exchange(ref _brokerOutstanding, 0); + throw new InvalidOperationException("Reply decremented broker outstanding below zero."); + } + } + + public void OnPublishFailed() + { + Interlocked.Decrement(ref _waiting); + Interlocked.Decrement(ref _brokerOutstanding); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTrackerTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTrackerTests.cs new file mode 100644 index 00000000..c584c791 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTrackerTests.cs @@ -0,0 +1,77 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Deterministic lifecycle contract for exclusive ReplyTo drain (waiter vs broker-outstanding). + /// + public sealed class ArticleWorkRpcOutstandingTrackerTests + { + [Fact] + public void Timeout_clears_waiter_but_keeps_broker_outstanding_until_late_reply() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + tracker.OnPublished(); + Assert.Equal(1, tracker.WaitingCount); + Assert.Equal(1, tracker.BrokerOutstandingCount); + + tracker.OnCallerTimeout(); + Assert.Equal(0, tracker.WaitingCount); + Assert.Equal(1, tracker.BrokerOutstandingCount); + + tracker.OnReply(callerStillWaiting: false); + Assert.Equal(0, tracker.WaitingCount); + Assert.Equal(0, tracker.BrokerOutstandingCount); + Assert.Equal(1, tracker.LateRepliesAfterTimeout); + } + + [Fact] + public void Normal_reply_clears_waiter_and_broker_outstanding() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + tracker.OnPublished(); + tracker.OnReply(callerStillWaiting: true); + Assert.Equal(0, tracker.WaitingCount); + Assert.Equal(0, tracker.BrokerOutstandingCount); + Assert.Equal(0, tracker.LateRepliesAfterTimeout); + } + + [Fact] + public void Concurrent_requests_are_independent() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + tracker.OnPublished(); + tracker.OnPublished(); + Assert.Equal(2, tracker.WaitingCount); + Assert.Equal(2, tracker.BrokerOutstandingCount); + + tracker.OnCallerTimeout(); + Assert.Equal(1, tracker.WaitingCount); + Assert.Equal(2, tracker.BrokerOutstandingCount); + + tracker.OnReply(callerStillWaiting: true); + Assert.Equal(0, tracker.WaitingCount); + Assert.Equal(1, tracker.BrokerOutstandingCount); + + tracker.OnReply(callerStillWaiting: false); + Assert.Equal(0, tracker.BrokerOutstandingCount); + Assert.Equal(1, tracker.LateRepliesAfterTimeout); + } + + [Fact] + public void Publish_failure_clears_both_counters() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + tracker.OnPublished(); + tracker.OnPublishFailed(); + Assert.Equal(0, tracker.WaitingCount); + Assert.Equal(0, tracker.BrokerOutstandingCount); + } + + [Fact] + public void Dispose_is_safe_when_broker_outstanding_already_zero() + { + var tracker = new ArticleWorkRpcOutstandingTracker(); + Assert.Equal(0, tracker.BrokerOutstandingCount); + Assert.Equal(0, tracker.WaitingCount); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs new file mode 100644 index 00000000..106bd2e8 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs @@ -0,0 +1,561 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using VectorNNTPD.BackFillerService.ArticleWork; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Publishes Article Work requests to per-backbone backfiller.{backbone} fanout exchanges + /// and correlates compact JSON replies. Reply bodies do not contain article octets. + /// + /// + /// One exclusive auto-delete reply queue is owned by this client's connection. Caller wait timeouts + /// remove the waiter but keep the correlation marked in-flight until a late reply arrives or + /// finishes its bounded drain. Closing the connection deletes ReplyTo; + /// drain must therefore wait on broker-side outstanding work, not only on waiting callers. + /// Work requests deliberately omit AMQP Expiration so RabbitMQ may buffer ready + /// messages while consumers are at capacity. Response TTL remains a BackFiller reply-path concern + /// (). + /// + public sealed class ArticleWorkStressClient : IAsyncDisposable + { + private readonly IConnection _connection; + private readonly IChannel _publishChannel; + private readonly IChannel _consumeChannel; + private readonly string _replyQueue; + private readonly ConcurrentDictionary _pending = + new(StringComparer.Ordinal); + private readonly TimeSpan _replyTimeout; + private readonly TimeSpan _disposeDrainTimeout; + private int _disposed; + private int _waitingCount; + private int _brokerOutstandingCount; + private int _unresolvedAtDispose; + private int _lateRepliesAfterTimeout; + private int _replyConsumerReady; + private int _unknownCorrelationReplies; + private string? _stressRunId; + + private ArticleWorkStressClient( + IConnection connection, + IChannel publishChannel, + IChannel consumeChannel, + string replyQueue, + TimeSpan replyTimeout, + TimeSpan disposeDrainTimeout) + { + _connection = connection; + _publishChannel = publishChannel; + _consumeChannel = consumeChannel; + _replyQueue = replyQueue; + _replyTimeout = replyTimeout; + _disposeDrainTimeout = disposeDrainTimeout; + } + + /// + /// Opens a RabbitMQ connection using environment credentials (never logged). + /// + public static async Task ConnectAsync( + StressHarnessOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + var factory = new ConnectionFactory + { + HostName = options.RabbitMqHost, + Port = options.RabbitMqPort, + VirtualHost = options.RabbitMqVirtualHost, + UserName = options.RabbitMqUsername, + Password = options.RabbitMqPassword, + RequestedHeartbeat = TimeSpan.FromSeconds(30), + AutomaticRecoveryEnabled = false, + ClientProvidedName = "VectorNNTPD.BackFillerService.Tests:e2e-stress", + }; + + var connection = await factory.CreateConnectionAsync(cancellationToken).ConfigureAwait(false); + var publishChannel = await connection.CreateChannelAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + var consumeChannel = await connection.CreateChannelAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var reply = await consumeChannel.QueueDeclareAsync( + queue: string.Empty, + durable: false, + exclusive: true, + autoDelete: true, + arguments: null, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var client = new ArticleWorkStressClient( + connection, + publishChannel, + consumeChannel, + reply.QueueName, + options.WorkReplyTimeout, + options.ReplyQueueDisposeDrainTimeout); + + // Reply queue must exist and the response consumer must be registered before any request publish. + var consumer = new AsyncEventingBasicConsumer(consumeChannel); + consumer.ReceivedAsync += client.OnReplyAsync; + await consumeChannel.BasicConsumeAsync( + reply.QueueName, + autoAck: true, + consumer: consumer, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + Volatile.Write(ref client._replyConsumerReady, 1); + return client; + } + + /// + /// after the exclusive ReplyTo queue is declared and the response consumer is consuming. + /// + public bool IsReplyConsumerReady => Volatile.Read(ref _replyConsumerReady) != 0; + + /// Replies whose CorrelationId did not match any outstanding request (accounted, not applied to a waiter). + public int UnknownCorrelationReplies => Volatile.Read(ref _unknownCorrelationReplies); + + /// + /// Optional stress-run correlation written as AMQP header StressRunId on each request. + /// Distinguishes this harness run from unrelated broker work in logs and captures. + /// + public string? StressRunId + { + get => _stressRunId; + set => _stressRunId = value; + } + + /// Number of callers still awaiting a correlated reply (excludes timed-out waiters). + public int WaitingCount => Volatile.Read(ref _waitingCount); + + /// + /// Number of published requests whose ReplyTo may still receive a response + /// (includes timed-out waiters until a late reply or dispose drain completes). + /// + public int BrokerOutstandingCount => Volatile.Read(ref _brokerOutstandingCount); + + /// Alias for kept for existing harness drain loops. + public int PendingCount => WaitingCount; + + /// Exclusive reply-queue name used as AMQP ReplyTo for this client. + public string ReplyQueueName => _replyQueue; + + /// Requests still outstanding when dispose drain timed out (0 when drain completed). + public int UnresolvedAtDispose => Volatile.Read(ref _unresolvedAtDispose); + + /// Replies received after the caller wait timed out (safe discard path). + public int LateRepliesAfterTimeout => Volatile.Read(ref _lateRepliesAfterTimeout); + + /// + /// Publishes one Article Work request and awaits the correlated reply (or timeout). + /// + public async Task RequestAsync( + string backbone, + string messageId, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(backbone); + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + if (!IsReplyConsumerReady) + { + throw new InvalidOperationException( + "Article Work RPC response consumer is not ready; ConnectAsync must complete before RequestAsync."); + } + + var requestId = Guid.NewGuid(); + var correlationId = Guid.NewGuid().ToString("D"); + var exchange = ComposeProviderEntity(backbone); + var payload = BuildRequestJson(requestId, messageId, backbone); + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var pending = new PendingRpc(tcs); + // Register the waiter before publishing so an immediate reply cannot race registration. + if (!_pending.TryAdd(correlationId, pending)) + { + return new ArticleWorkReply( + ArticleWorkStressOutcome.BrokerError, + backbone, + messageId, + requestId, + null, + null, + null, + "duplicate correlation id") + { + CorrelationId = correlationId, + }; + } + + Interlocked.Increment(ref _waitingCount); + Interlocked.Increment(ref _brokerOutstandingCount); + + var props = CreateRequestProperties(correlationId, _replyQueue, requestId, _stressRunId); + + var publishStarted = Stopwatch.GetTimestamp(); + long publishCompleted; + try + { + await _publishChannel.BasicPublishAsync( + exchange: exchange, + routingKey: exchange, + mandatory: false, + basicProperties: props, + body: payload, + cancellationToken: cancellationToken) + .ConfigureAwait(false); + publishCompleted = Stopwatch.GetTimestamp(); + } + // codeql[cs/catch-of-all-exceptions] Publish must return a broker-error outcome for every transport/AMQP failure type. + catch (Exception ex) + { + CompleteBrokerOutstanding(correlationId, clearWaiting: true); + return new ArticleWorkReply( + ArticleWorkStressOutcome.BrokerError, + backbone, + messageId, + requestId, + null, + null, + null, + ex.GetType().Name) + { + CorrelationId = correlationId, + PublishStartedTimestamp = publishStarted, + PublishCompletedTimestamp = Stopwatch.GetTimestamp(), + }; + } + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(_replyTimeout); + try + { + var reply = await tcs.Task.WaitAsync(timeoutCts.Token).ConfigureAwait(false); + var replyReceived = Stopwatch.GetTimestamp(); + var publishMs = (long)Stopwatch.GetElapsedTime(publishStarted, publishCompleted).TotalMilliseconds; + var publishToReplyMs = (long)Stopwatch.GetElapsedTime(publishCompleted, replyReceived).TotalMilliseconds; + return reply with + { + CorrelationId = correlationId, + PublishStartedTimestamp = publishStarted, + PublishCompletedTimestamp = publishCompleted, + ReplyReceivedTimestamp = replyReceived, + PublishMilliseconds = publishMs, + LatencyMilliseconds = publishToReplyMs, + }; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + // Caller no longer waits; broker may still publish to ReplyTo. + if (_pending.TryGetValue(correlationId, out var timedOut) + && Interlocked.Exchange(ref timedOut.Waiting, 0) == 1) + { + Interlocked.Decrement(ref _waitingCount); + } + + var replyReceived = Stopwatch.GetTimestamp(); + return new ArticleWorkReply( + ArticleWorkStressOutcome.NoResponse, + backbone, + messageId, + requestId, + null, + null, + null, + "work reply timeout") + { + CorrelationId = correlationId, + PublishStartedTimestamp = publishStarted, + PublishCompletedTimestamp = publishCompleted, + ReplyReceivedTimestamp = replyReceived, + PublishMilliseconds = (long)Stopwatch.GetElapsedTime(publishStarted, publishCompleted).TotalMilliseconds, + LatencyMilliseconds = (long)Stopwatch.GetElapsedTime(publishCompleted, replyReceived).TotalMilliseconds, + }; + } + catch (OperationCanceledException) + { + CompleteBrokerOutstanding(correlationId, clearWaiting: true); + throw; + } + } + + public static string ComposeProviderEntity(string backbone) + { + ArgumentException.ThrowIfNullOrWhiteSpace(backbone); + return "backfiller." + backbone.Trim().ToLowerInvariant(); + } + + /// + /// Builds AMQP properties for one Article Work RPC request. + /// + /// + /// Exposed for deterministic regression tests that assert the exact properties passed to publish + /// without requiring a live broker. Work requests must not set : + /// a short request TTL silently discards buffered work and invalidates overload measurements. + /// + public static BasicProperties CreateRequestProperties( + string correlationId, + string replyTo, + Guid requestId, + string? stressRunId = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(correlationId); + ArgumentException.ThrowIfNullOrWhiteSpace(replyTo); + + var headers = new Dictionary + { + ["RequestId"] = requestId.ToString("D"), + }; + if (!string.IsNullOrWhiteSpace(stressRunId)) + { + headers["StressRunId"] = stressRunId; + } + + return new BasicProperties + { + ContentType = "application/json", + CorrelationId = correlationId, + ReplyTo = replyTo, + DeliveryMode = DeliveryModes.Persistent, + Headers = headers, + }; + } + + private async Task OnReplyAsync(object sender, BasicDeliverEventArgs args) + { + await Task.Yield(); + var correlationId = args.BasicProperties.CorrelationId; + if (string.IsNullOrEmpty(correlationId) || !_pending.TryRemove(correlationId, out var pending)) + { + Interlocked.Increment(ref _unknownCorrelationReplies); + return; + } + + var wasWaiting = Interlocked.Exchange(ref pending.Waiting, 0) == 1; + if (wasWaiting) + { + Interlocked.Decrement(ref _waitingCount); + } + else + { + Interlocked.Increment(ref _lateRepliesAfterTimeout); + } + + Interlocked.Decrement(ref _brokerOutstandingCount); + + try + { + var reply = ParseReply(args.Body.Span); + if (wasWaiting) + { + pending.Waiter.TrySetResult(reply with { CorrelationId = correlationId }); + } + } + // codeql[cs/catch-of-all-exceptions] Consumer callback must complete the waiter for any parse/delivery fault. + catch (Exception ex) + { + if (wasWaiting) + { + pending.Waiter.TrySetResult(new ArticleWorkReply( + ArticleWorkStressOutcome.BrokerError, + Backbone: string.Empty, + MessageId: string.Empty, + RequestId: null, + Fqdn: null, + ListenerPort: null, + ArticleIdHex: null, + Error: ex.GetType().Name) + { + CorrelationId = correlationId, + }); + } + } + } + + private void CompleteBrokerOutstanding(string correlationId, bool clearWaiting) + { + if (!_pending.TryRemove(correlationId, out var pending)) + { + return; + } + + if (clearWaiting && Interlocked.Exchange(ref pending.Waiting, 0) == 1) + { + Interlocked.Decrement(ref _waitingCount); + } + + Interlocked.Decrement(ref _brokerOutstandingCount); + } + + private static byte[] BuildRequestJson(Guid requestId, string messageId, string backbone) + { + var json = + "{\"version\":1,\"requestId\":\"" + + requestId.ToString("D") + + "\",\"messageId\":" + + JsonSerializer.Serialize(messageId) + + ",\"backbone\":" + + JsonSerializer.Serialize(backbone) + + "}"; + return Encoding.UTF8.GetBytes(json); + } + + private static ArticleWorkReply ParseReply(ReadOnlySpan body) + { + using var doc = JsonDocument.Parse(body.ToArray()); + var root = doc.RootElement; + var outcomeText = root.GetProperty("outcome").GetString() ?? string.Empty; + var outcome = outcomeText switch + { + "Success" => ArticleWorkStressOutcome.Success, + "ArticleNotFound" => ArticleWorkStressOutcome.ArticleNotFound, + "InvalidArticle" => ArticleWorkStressOutcome.InvalidArticle, + "InvalidRequest" => ArticleWorkStressOutcome.InvalidRequest, + "ProviderFailure" => ArticleWorkStressOutcome.ProviderFailure, + "UnexpectedFailure" => ArticleWorkStressOutcome.UnexpectedFailure, + "RetentionRejected" => ArticleWorkStressOutcome.RetentionRejected, + _ => ArticleWorkStressOutcome.BrokerError, + }; + + Guid? requestId = null; + if (root.TryGetProperty("requestId", out var requestIdNode) + && requestIdNode.ValueKind == JsonValueKind.String + && Guid.TryParse(requestIdNode.GetString(), out var parsedId)) + { + requestId = parsedId; + } + + var messageId = root.TryGetProperty("messageId", out var mid) && mid.ValueKind == JsonValueKind.String + ? mid.GetString() ?? string.Empty + : string.Empty; + var backbone = root.TryGetProperty("backbone", out var bb) && bb.ValueKind == JsonValueKind.String + ? bb.GetString() ?? string.Empty + : string.Empty; + string? fqdn = null; + int? port = null; + string? articleId = null; + string? error = null; + if (outcome == ArticleWorkStressOutcome.Success) + { + fqdn = root.GetProperty("fqdn").GetString(); + port = root.GetProperty("listenerPort").GetInt32(); + articleId = root.GetProperty("articleId").GetString(); + } + else if (root.TryGetProperty("error", out var err) && err.ValueKind == JsonValueKind.String) + { + error = err.GetString(); + } + + return new ArticleWorkReply(outcome, backbone, messageId, requestId, fqdn, port, articleId, error); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + var drainDeadline = Stopwatch.GetTimestamp() + + (long)(_disposeDrainTimeout.TotalSeconds * Stopwatch.Frequency); + while (Volatile.Read(ref _brokerOutstandingCount) > 0 + && Stopwatch.GetTimestamp() < drainDeadline) + { + await Task.Delay(50).ConfigureAwait(false); + } + + var unresolved = Volatile.Read(ref _brokerOutstandingCount); + if (unresolved > 0) + { + Interlocked.Exchange(ref _unresolvedAtDispose, unresolved); + Console.Error.WriteLine( + $"ArticleWorkStressClient dispose drain timed out with {unresolved} broker-outstanding request(s); closing exclusive ReplyTo may leave work unroutable until production terminal settlement."); + } + + foreach (var pending in _pending) + { + pending.Value.Waiter.TrySetCanceled(); + } + + _pending.Clear(); + Interlocked.Exchange(ref _waitingCount, 0); + Interlocked.Exchange(ref _brokerOutstandingCount, 0); + + try + { + await _consumeChannel.CloseAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Dispose must continue closing remaining RabbitMQ resources after any close fault. + catch (Exception) + { + // ignored on dispose + } + + try + { + await _publishChannel.CloseAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Dispose must continue closing remaining RabbitMQ resources after any close fault. + catch (Exception) + { + // ignored on dispose + } + + try + { + await _connection.CloseAsync().ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Dispose must finish even when the broker connection close faults. + catch (Exception) + { + // ignored on dispose + } + + await _consumeChannel.DisposeAsync().ConfigureAwait(false); + await _publishChannel.DisposeAsync().ConfigureAwait(false); + await _connection.DisposeAsync().ConfigureAwait(false); + } + + private sealed class PendingRpc(TaskCompletionSource waiter) + { + public TaskCompletionSource Waiter { get; } = waiter; + + /// 1 while a caller is awaiting ; 0 after timeout or completion. + public int Waiting = 1; + } + } + + /// One correlated Article Work reply. Does not carry article bytes. + public sealed record ArticleWorkReply( + ArticleWorkStressOutcome Outcome, + string Backbone, + string MessageId, + Guid? RequestId, + string? Fqdn, + int? ListenerPort, + string? ArticleIdHex, + string? Error) + { + /// AMQP correlation id used to match the reply. + public string? CorrelationId { get; init; } + + /// + /// Milliseconds from publish-completed to reply-received (aggregate BackFiller path). + /// Not an isolated upstream fetch duration. + /// + public long LatencyMilliseconds { get; init; } + + /// Milliseconds for the local BasicPublishAsync call. + public long PublishMilliseconds { get; init; } + + public long PublishStartedTimestamp { get; init; } + public long PublishCompletedTimestamp { get; init; } + public long ReplyReceivedTimestamp { get; init; } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs new file mode 100644 index 00000000..11548518 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs @@ -0,0 +1,97 @@ +using RabbitMQ.Client; +using VectorNNTPD.BackFillerService.ArticleWork; +using VectorNNTPD.BackFillerService.Configuration; +using VectorNNTPD.BackFillerService.RabbitMq; +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Deterministic contract for Article Work RPC request AMQP properties used by the stress client. + /// + public sealed class ArticleWorkStressClientRequestPropertiesTests + { + [Fact] + public void CreateRequestProperties_omits_expiration_so_ready_work_can_buffer() + { + var requestId = Guid.Parse("11111111-1111-1111-1111-111111111111"); + var props = ArticleWorkStressClient.CreateRequestProperties( + correlationId: "corr-1", + replyTo: "amq.gen-reply-test", + requestId); + + Assert.True(string.IsNullOrEmpty(props.Expiration)); + } + + [Fact] + public void CreateRequestProperties_sets_correlation_reply_to_content_type_and_request_id() + { + var requestId = Guid.Parse("22222222-2222-2222-2222-222222222222"); + var props = ArticleWorkStressClient.CreateRequestProperties( + correlationId: "corr-abc", + replyTo: "exclusive.reply.queue", + requestId); + + Assert.Equal("corr-abc", props.CorrelationId); + Assert.Equal("exclusive.reply.queue", props.ReplyTo); + Assert.Equal("application/json", props.ContentType); + Assert.Equal(DeliveryModes.Persistent, props.DeliveryMode); + Assert.NotNull(props.Headers); + Assert.True(props.Headers.TryGetValue("RequestId", out var header)); + Assert.Equal(requestId.ToString("D"), header); + Assert.True(string.IsNullOrEmpty(props.Expiration)); + } + + [Fact] + public void Response_ttl_remains_one_second_and_is_not_applied_to_stress_requests() + { + Assert.Equal("1000", ArticleWorkResponseWireProtocol.ExpirationMilliseconds); + var props = ArticleWorkStressClient.CreateRequestProperties( + "corr", + "reply", + Guid.Parse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")); + Assert.NotEqual( + ArticleWorkResponseWireProtocol.ExpirationMilliseconds, + props.Expiration ?? string.Empty); + } + + [Fact] + public void Provider_work_queue_arguments_do_not_declare_x_message_ttl() + { + var classic = BackFillerArticleWorkTopology.BuildQueueArguments( + new BackFillerRabbitMqRuntimeOptions( + WorkRequestMaxPayloadBytes: 65536, + PublishConfirmTimeoutSeconds: 30, + ConsumerPrefetchCount: 1, + QueueTopology: new ArticleWorkQueueTopology( + ArticleWorkQueueKind.Classic, + ConfiguredHostCount: 1, + QuorumInitialGroupSize: null), + DeliveryLimit: 10)); + Assert.False(classic.ContainsKey("x-message-ttl")); + + var quorum = BackFillerArticleWorkTopology.BuildQueueArguments( + new BackFillerRabbitMqRuntimeOptions( + WorkRequestMaxPayloadBytes: 65536, + PublishConfirmTimeoutSeconds: 30, + ConsumerPrefetchCount: 1, + QueueTopology: new ArticleWorkQueueTopology( + ArticleWorkQueueKind.Quorum, + ConfiguredHostCount: 3, + QuorumInitialGroupSize: 3), + DeliveryLimit: 10)); + Assert.False(quorum.ContainsKey("x-message-ttl")); + Assert.True(quorum.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName)); + } + + [Fact] + public void CreateRequestProperties_rejects_missing_correlation_or_reply_to() + { + var requestId = Guid.NewGuid(); + Assert.Throws(() => + ArticleWorkStressClient.CreateRequestProperties(" ", "reply", requestId)); + Assert.Throws(() => + ArticleWorkStressClient.CreateRequestProperties("corr", " ", requestId)); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs new file mode 100644 index 00000000..6df58815 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs @@ -0,0 +1,585 @@ +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Opt-in BackFiller end-to-end stress entry points. + /// Disabled unless VECTORNNTPD_BACKFILLER_E2E_STRESS=1. + /// + /// + /// Data path under test: publish Article Work to backfiller.{backbone} → BackFiller + /// upstream ARTICLE → retention → TLS listener ARTICLE. RabbitMQ replies are metadata only. + /// + public sealed class BackFillerE2EStressBench + { + [Fact] + public async Task Discover_workload_from_newslogs_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var options = StressHarnessOptions.FromEnvironment(); + if (!Directory.Exists(options.NewsLogDirectory)) + { + Assert.Fail($"News log directory missing: {options.NewsLogDirectory}"); + } + + var harness = new BackFillerE2EStressHarness(options); + var stats = await harness.DiscoverAndWriteWorkloadAsync(CancellationToken.None); + Assert.True(stats.ValidIds > 0); + Assert.True(stats.UniqueIds > 0); + Console.WriteLine( + $"discovery files={stats.FileCount} entries={stats.TotalLogEntries} valid={stats.ValidIds} unique={stats.UniqueIds} malformed={stats.MalformedEntries} dupRate={stats.DuplicateRatePercent:F2}% giganews={stats.ProviderGiganews} eweka={stats.ProviderEweka}"); + } + + [Fact] + public async Task PhaseA_functional_baseline_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var options = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(options); + var harness = new BackFillerE2EStressHarness(options); + var backbone = StressHarnessOptions.GetBackboneOrDefault("Giganews"); + var file = Path.Join(options.WorkloadDirectory, "warmup-message-ids.txt"); + var summary = await harness.RunAsync( + "phaseA-baseline-c1", + backbone, + file, + concurrency: 1, + maxInFlight: 2, + duration: TimeSpan.FromSeconds(30), + maxRequests: 10, + CancellationToken.None); + Console.WriteLine(summary.FormatTableRow()); + Assert.True(summary.RequestsStarted > 0); + Assert.Equal(0, summary.OutstandingAtEnd); + } + + [Fact] + public async Task PhaseB_concurrency_ramp_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var phase = StressHarnessOptions.GetPhaseOrDefault("B"); + if (!string.Equals(phase, "B", StringComparison.OrdinalIgnoreCase) + && !string.Equals(phase, "ALL", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var options = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(options); + var harness = new BackFillerE2EStressHarness(options); + var backbone = StressHarnessOptions.GetBackboneOrDefault("Giganews"); + var file = Path.Join(options.WorkloadDirectory, "main-message-ids.txt"); + var levels = ParseConcurrencyLevels(); + var durationSeconds = StressHarnessOptions.GetDurationSecondsOrDefault(60); + foreach (var concurrency in levels) + { + var maxInFlight = StressHarnessOptions.GetMaxInFlightOrDefault(concurrency * 2); + var summary = await harness.RunAsync( + $"phaseB-ramp-c{concurrency}", + backbone, + file, + concurrency, + maxInFlight, + TimeSpan.FromSeconds(durationSeconds), + maxRequests: null, + CancellationToken.None); + Console.WriteLine(summary.FormatTableRow()); + if (summary.Timeouts > summary.RequestsStarted * 0.25) + { + Console.WriteLine($"phaseB stopping ramp after c={concurrency}: elevated timeouts"); + break; + } + } + } + + [Fact] + public async Task PhaseC_sustained_throughput_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var phase = StressHarnessOptions.GetPhaseOrDefault("C"); + if (!string.Equals(phase, "C", StringComparison.OrdinalIgnoreCase) + && !string.Equals(phase, "ALL", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var options = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(options); + var harness = new BackFillerE2EStressHarness(options); + var backbone = StressHarnessOptions.GetBackboneOrDefault("Giganews"); + var file = Path.Join(options.WorkloadDirectory, "main-message-ids.txt"); + var concurrency = StressHarnessOptions.GetConcurrencyOrDefault(100); + var durationSeconds = StressHarnessOptions.GetDurationSecondsOrDefault(300); + var maxInFlight = StressHarnessOptions.GetMaxInFlightOrDefault(concurrency * 2); + var summary = await harness.RunAsync( + $"phaseC-sustained-c{concurrency}", + backbone, + file, + concurrency, + maxInFlight, + TimeSpan.FromSeconds(durationSeconds), + maxRequests: null, + CancellationToken.None); + Console.WriteLine(summary.FormatTableRow()); + Console.WriteLine($"phaseC payloadGbps={summary.PayloadGbps:F3} target=1.0 achieved={(summary.PayloadGbps >= 1.0 ? "yes" : "no")}"); + } + + [Fact] + public async Task PhaseD_stability_soak_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var phase = StressHarnessOptions.GetPhaseOrDefault("D"); + if (!string.Equals(phase, "D", StringComparison.OrdinalIgnoreCase) + && !string.Equals(phase, "ALL", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var options = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(options); + var harness = new BackFillerE2EStressHarness(options); + var backbone = StressHarnessOptions.GetBackboneOrDefault("Giganews"); + var file = Path.Join(options.WorkloadDirectory, "repeatable-message-ids.txt"); + var concurrency = StressHarnessOptions.GetConcurrencyOrDefault(100); + var durationSeconds = StressHarnessOptions.GetDurationSecondsOrDefault(300); + var summary = await harness.RunAsync( + $"phaseD-soak-c{concurrency}", + backbone, + file, + concurrency, + maxInFlight: concurrency * 2, + TimeSpan.FromSeconds(durationSeconds), + maxRequests: null, + CancellationToken.None); + Console.WriteLine(summary.FormatTableRow()); + Assert.True(summary.OutstandingAtEnd == 0 || summary.Timeouts > 0); + } + + [Fact] + public async Task Isolated_backbone_runs_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var phase = StressHarnessOptions.GetPhaseOrDefault("provider"); + if (!string.Equals(phase, "provider", StringComparison.OrdinalIgnoreCase) + && !string.Equals(phase, "ALL", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var options = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(options); + var harness = new BackFillerE2EStressHarness(options); + var file = Path.Join(options.WorkloadDirectory, "warmup-message-ids.txt"); + foreach (var backbone in new[] { "Giganews", "Eweka" }) + { + var summary = await harness.RunAsync( + $"provider-{backbone}-c25", + backbone, + file, + concurrency: 25, + maxInFlight: 50, + duration: TimeSpan.FromSeconds(90), + maxRequests: null, + CancellationToken.None); + Console.WriteLine(summary.FormatTableRow()); + await File.WriteAllTextAsync( + Path.Join(options.ReportDirectory, $"provider-{backbone}-redacted-config.json"), + JsonSerializer.Serialize(options.ToRedactedReportObject(), new JsonSerializerOptions { WriteIndented = true })); + } + } + + /// + /// Concurrent Eweka + Giganews sustained flood soak. Opt-in phase + /// VECTORNNTPD_BACKFILLER_E2E_STRESS_PHASE=DualSoak. + /// + /// + /// + /// Measured duration defaults to 1800 s. Per-backbone maxInFlight is + /// sessions + ready target (Eweka 50+75=125; Giganews observed MaxSessions + 150) + /// so RPC waiters keep a ready backlog while consumers stay saturated. + /// + /// + /// Independent instances start measured load + /// together via . No shared warm-up drain gates the other + /// backbone. Abort uses completed-RPC rates (see ), + /// and DualSoak raises the abort thresholds so ramp-up outstanding work cannot cancel + /// one backbone while the other continues alone. + /// + /// + [Fact] + public async Task DualSoak_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var phase = StressHarnessOptions.GetPhaseOrDefault("DualSoak"); + if (!string.Equals(phase, "DualSoak", StringComparison.OrdinalIgnoreCase) + && !string.Equals(phase, "ALL", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var baseOptions = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(baseOptions); + // Dual flood must not abort on delivery lag during ramp (prior defect: Giganews + // cancelled after ~3 s while Eweka alone kept flooding). + var options = CloneForDualFlood(baseOptions); + var durationSeconds = StressHarnessOptions.GetDurationSecondsOrDefault(1800); + var mainFile = Path.Join(options.WorkloadDirectory, "main-message-ids.txt"); + + const int ewekaSessions = 50; + // Target 100 concurrent Giganews sessions (50+100=150 aggregate). Production MaxSessions + // still comes from MySQL maxconnections; the pool will not exceed that value. + const int giganewsSessions = 100; + const int ewekaMinReady = 75; + const int giganewsMinReady = 150; + var ewekaMaxInFlight = ewekaSessions + ewekaMinReady; + var giganewsMaxInFlight = giganewsSessions + giganewsMinReady; + + Console.WriteLine( + $"DualSoak durationSec={durationSeconds} ewekaSessions={ewekaSessions} ewekaMaxInFlight={ewekaMaxInFlight} " + + $"giganewsSessions={giganewsSessions} giganewsMaxInFlight={giganewsMaxInFlight} " + + $"abortAfter={options.AbortAfterRequests} abortFailRate={options.AbortOnFailureRate}"); + + // Bounded readiness: wait for residual accepted work to settle (no purge). Independent + // per-backbone waits run concurrently so one backlog cannot stall the other's check. + var cleanEweka = StressQueuePreflight.WaitUntilCleanAsync( + options, + "Eweka", + label: "dual-soak-start-eweka", + timeout: TimeSpan.FromMinutes(3), + CancellationToken.None); + var cleanGiga = StressQueuePreflight.WaitUntilCleanAsync( + options, + "Giganews", + label: "dual-soak-start-giganews", + timeout: TimeSpan.FromMinutes(3), + CancellationToken.None); + await Task.WhenAll(cleanEweka, cleanGiga); + + var ewekaHarness = new BackFillerE2EStressHarness(options); + var gigaHarness = new BackFillerE2EStressHarness(options); + + // Short independent health probes (maxRequests), not duration-based warm-ups that + // can block measured start for ReplyTo dispose-drain (observed 120 s on Giganews). + var probeEweka = ewekaHarness.RunAsync( + "dual-soak-probe-Eweka", + "Eweka", + mainFile, + concurrency: 5, + maxInFlight: 5, + duration: TimeSpan.FromMinutes(2), + maxRequests: 5, + CancellationToken.None); + var probeGiga = gigaHarness.RunAsync( + "dual-soak-probe-Giganews", + "Giganews", + mainFile, + concurrency: 5, + maxInFlight: 5, + duration: TimeSpan.FromMinutes(2), + maxRequests: 5, + CancellationToken.None); + var probes = await Task.WhenAll(probeEweka, probeGiga); + Console.WriteLine(probes[0].FormatTableRow()); + Console.WriteLine(probes[1].FormatTableRow()); + Assert.True( + probes[0].RequestsStarted > 0 && probes[1].RequestsStarted > 0, + "DualSoak probe did not start requests on both backbones."); + + // Re-check queues are not poisoned after probes (ready/unacked may be briefly non-zero + // during drain; allow a small residual then start measured). + await Task.Delay(TimeSpan.FromSeconds(2)); + + var measuredStartedUtc = DateTimeOffset.UtcNow; + Console.WriteLine($"DualSoak measured start utc={measuredStartedUtc:o}"); + + var measuredEwekaTask = ewekaHarness.RunAsync( + "dual-soak-Eweka", + "Eweka", + mainFile, + concurrency: ewekaMaxInFlight, + maxInFlight: ewekaMaxInFlight, + duration: TimeSpan.FromSeconds(durationSeconds), + maxRequests: null, + CancellationToken.None); + var measuredGigaTask = gigaHarness.RunAsync( + "dual-soak-Giganews", + "Giganews", + mainFile, + concurrency: giganewsMaxInFlight, + maxInFlight: giganewsMaxInFlight, + duration: TimeSpan.FromSeconds(durationSeconds), + maxRequests: null, + CancellationToken.None); + var measured = await Task.WhenAll(measuredEwekaTask, measuredGigaTask); + var measuredEndedUtc = DateTimeOffset.UtcNow; + + var eweka = measured[0]; + var giga = measured[1]; + Console.WriteLine(eweka.FormatTableRow()); + Console.WriteLine(giga.FormatTableRow()); + Console.WriteLine( + $"DualSoak combined art/s={eweka.ArticlesPerSecond + giga.ArticlesPerSecond:F2} " + + $"combinedGbps={eweka.PayloadGbps + giga.PayloadGbps:F3} " + + $"ewekaElapsed={eweka.ElapsedSeconds:F1}s gigaElapsed={giga.ElapsedSeconds:F1}s " + + $"ewekaTimeouts={eweka.Timeouts} gigaTimeouts={giga.Timeouts}"); + + await File.WriteAllTextAsync( + Path.Join(options.ReportDirectory, "dual-soak-combined-summary.json"), + JsonSerializer.Serialize( + new + { + MeasuredStartedUtc = measuredStartedUtc, + MeasuredEndedUtc = measuredEndedUtc, + ConfiguredMeasuredDurationSeconds = durationSeconds, + EwekaSessionsTarget = ewekaSessions, + GiganewsSessionsTarget = giganewsSessions, + EwekaMinReadyTarget = ewekaMinReady, + GiganewsMinReadyTarget = giganewsMinReady, + EwekaMaxInFlight = ewekaMaxInFlight, + GiganewsMaxInFlight = giganewsMaxInFlight, + EwekaElapsedSeconds = eweka.ElapsedSeconds, + GiganewsElapsedSeconds = giga.ElapsedSeconds, + Eweka = eweka, + Giganews = giga, + CombinedArticlesPerSecond = eweka.ArticlesPerSecond + giga.ArticlesPerSecond, + CombinedPayloadGbps = eweka.PayloadGbps + giga.PayloadGbps, + }, + new JsonSerializerOptions { WriteIndented = true })); + + Assert.True( + eweka.ElapsedSeconds >= durationSeconds * 0.9 + && giga.ElapsedSeconds >= durationSeconds * 0.9, + $"DualSoak measured elapsed too short (eweka={eweka.ElapsedSeconds:F1}s giga={giga.ElapsedSeconds:F1}s; need ~{durationSeconds}s)."); + } + + /// + /// Copies live stress options with DualSoak abort policy (never cancel on ramp lag). + /// + private static StressHarnessOptions CloneForDualFlood(StressHarnessOptions source) => + new() + { + RabbitMqHost = source.RabbitMqHost, + RabbitMqPort = source.RabbitMqPort, + RabbitMqVirtualHost = source.RabbitMqVirtualHost, + RabbitMqUsername = source.RabbitMqUsername, + RabbitMqPassword = source.RabbitMqPassword, + WorkReplyTimeout = source.WorkReplyTimeout, + ReplyQueueDisposeDrainTimeout = source.ReplyQueueDisposeDrainTimeout, + ArticleTimeout = source.ArticleTimeout, + NewsLogDirectory = source.NewsLogDirectory, + WorkloadDirectory = source.WorkloadDirectory, + ReportDirectory = source.ReportDirectory, + ListenerHostOverride = source.ListenerHostOverride, + ListenerPortOverride = source.ListenerPortOverride, + Seed = source.Seed, + // Probes validate retrieval; skip per-run article preflight so measured + // WhenAll starts together without serializing on another backbone's preflight. + ArticlePreflightCount = 0, + // Completed-RPC based; DualSoak still disables practical abort so one backbone + // cannot leave the other running alone for the full soak. + AbortOnTimeoutRate = 1.0, + AbortOnFailureRate = 1.0, + AbortAfterRequests = int.MaxValue, + }; + + /// + /// Controlled concurrency sweep with stage instrumentation. Opt-in phase + /// VECTORNNTPD_BACKFILLER_E2E_STRESS_PHASE=ConcurrencySweep. + /// Does not change production MaxSessions; levels above known pool size measure queueing. + /// + [Fact] + public async Task ConcurrencySweep_when_enabled() + { + if (!StressHarnessOptions.IsEnabled()) + { + return; + } + + var phase = StressHarnessOptions.GetPhaseOrDefault("ConcurrencySweep"); + if (!string.Equals(phase, "ConcurrencySweep", StringComparison.OrdinalIgnoreCase) + && !string.Equals(phase, "ALL", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var options = StressHarnessOptions.FromEnvironment(); + EnsureWorkload(options); + var harness = new BackFillerE2EStressHarness(options); + var backbone = StressHarnessOptions.GetBackboneOrDefault("Giganews"); + var mainFile = Path.Join(options.WorkloadDirectory, "main-message-ids.txt"); + var warmupFile = Path.Join(options.WorkloadDirectory, "warmup-message-ids.txt"); + var durationSeconds = StressHarnessOptions.GetDurationSecondsOrDefault(90); + var levels = ParseSweepConcurrencyLevels(); + + // Known production MaxSessions from prior runtime logs (not modified here). + var knownMaxSessions = string.Equals(backbone, "Eweka", StringComparison.OrdinalIgnoreCase) + ? 50 + : 99; + Console.WriteLine( + $"ConcurrencySweep backbone={backbone} knownMaxSessions={knownMaxSessions} levels={string.Join(',', levels)} durationSec={durationSeconds}"); + + // Fail visibly on poisoned backlog. Never purge from the harness. + await StressQueuePreflight.RequireCleanAsync( + options, + backbone, + label: "concurrency-sweep-start", + CancellationToken.None); + + var warmup = await harness.RunAsync( + $"sweep-{backbone}-warmup-c5", + backbone, + warmupFile, + concurrency: 5, + maxInFlight: 10, + duration: TimeSpan.FromSeconds(20), + maxRequests: null, + CancellationToken.None); + Console.WriteLine(warmup.FormatTableRow()); + Assert.True(warmup.ArticlesDelivered > 0, "Warm-up delivered zero articles; aborting sweep."); + Assert.Equal(0, warmup.OutstandingAtEnd); + + StressRunSummary? previous = null; + foreach (var concurrency in levels) + { + if (concurrency > knownMaxSessions * 2) + { + Console.WriteLine( + $"ConcurrencySweep stopping before c={concurrency}: exceeds 2× known MaxSessions={knownMaxSessions} (unsafe backlog risk without production changes)."); + break; + } + + // Keep maxInFlight == concurrency so PeakOutstanding can reach the sweep level. + // Do not inherit a stale VECTORNNTPD_BACKFILLER_E2E_STRESS_MAX_INFLIGHT from prior phases. + var maxInFlight = concurrency; + var summary = await harness.RunAsync( + $"sweep-{backbone}-c{concurrency}", + backbone, + mainFile, + concurrency, + maxInFlight, + TimeSpan.FromSeconds(durationSeconds), + maxRequests: null, + CancellationToken.None); + Console.WriteLine(summary.FormatTableRow()); + WriteSweepRow(options.ReportDirectory, summary, knownMaxSessions); + + var littlesW = summary.ArticlesPerSecond > 0 + ? summary.MeanOutstanding / summary.ArticlesPerSecond + : double.NaN; + Console.WriteLine( + $" stages replyMean={summary.WorkReplyLatencyMs.Mean:F0}ms e2eMean={summary.EndToEndSuccessLatencyMs.Mean:F0}ms littlesW={littlesW:F2}s drainMs={summary.DrainMilliseconds} outstandingEnd={summary.OutstandingAtEnd}"); + + var unhealthy = + summary.Timeouts > summary.RequestsStarted * 0.20 + || summary.ArticlesDelivered == 0 + || summary.OutstandingAtEnd > 0 + || (previous is not null + && previous.PayloadGbps > 0.01 + && summary.PayloadGbps < previous.PayloadGbps * 0.35 + && summary.Timeouts > 10); + + if (unhealthy) + { + Console.WriteLine( + $"ConcurrencySweep stopping after c={concurrency}: unhealthy (timeouts={summary.Timeouts} ok={summary.ArticlesDelivered} outstandingEnd={summary.OutstandingAtEnd})."); + break; + } + + previous = summary; + } + } + + private static void WriteSweepRow(string reportDirectory, StressRunSummary summary, int knownMaxSessions) + { + var path = Path.Join(reportDirectory, "concurrency-sweep-table.txt"); + var line = + $"{summary.RunName}\t{summary.Backbone}\t{summary.ConfiguredConcurrency}\t{knownMaxSessions}\t{summary.MeanOutstanding:F2}\t{summary.PeakOutstanding}\t{summary.ArticlesPerSecond:F2}\t{summary.PayloadGbps:F4}\t{summary.WorkReplyLatencyMs.Mean:F1}\t{summary.EndToEndSuccessLatencyMs.Mean:F1}\t{summary.MeanArticleLogicalBytes:F0}\t{summary.Timeouts}\t{summary.DrainMilliseconds}\t{summary.OutstandingAtEnd}"; + File.AppendAllText(path, line + Environment.NewLine); + } + + private static void EnsureWorkload(StressHarnessOptions options) + { + // Synchronous gate used by opt-in benches; provenance check is authoritative. + StressWorkloadGuard.RequireProvenancedWorkloadAsync(options, CancellationToken.None) + .GetAwaiter() + .GetResult(); + } + + private static int[] ParseConcurrencyLevels() + { + var raw = Environment.GetEnvironmentVariable(StressHarnessOptions.ConcurrencyEnvironmentVariable); + if (string.IsNullOrWhiteSpace(raw)) + { + return [1, 10, 25, 50, 100, 150, 200]; + } + + if (raw.Contains(',', StringComparison.Ordinal)) + { + return raw.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(static part => int.Parse(part)) + .Where(static value => value > 0) + .ToArray(); + } + + if (int.TryParse(raw, out var single) && single > 0) + { + return [single]; + } + + return [1, 10, 25, 50, 100, 150, 200]; + } + + /// Sweep defaults to 25…200; override with comma-separated CONCURRENCY env. + private static int[] ParseSweepConcurrencyLevels() + { + var raw = Environment.GetEnvironmentVariable(StressHarnessOptions.ConcurrencyEnvironmentVariable); + if (string.IsNullOrWhiteSpace(raw)) + { + return [25, 50, 75, 100, 150, 200]; + } + + if (raw.Contains(',', StringComparison.Ordinal)) + { + return raw.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(static part => int.Parse(part)) + .Where(static value => value > 0) + .ToArray(); + } + + if (int.TryParse(raw, out var single) && single > 0) + { + return [single]; + } + + return [25, 50, 75, 100, 150, 200]; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs new file mode 100644 index 00000000..6f404e27 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs @@ -0,0 +1,678 @@ +using System.Diagnostics; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Threading.Channels; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// End-to-end stress orchestrator: Article Work publish → RabbitMQ reply → TLS ARTICLE delivery. + /// + /// + /// The RabbitMQ reply is routing metadata only. Article payload bytes are measured exclusively + /// from the BackFiller NNTP listener after a complete multiline response is validated. + /// Upstream fetch start/end are not observable without production changes; publish-to-reply + /// is reported as an aggregate BackFiller interval. + /// + public sealed class BackFillerE2EStressHarness + { + private readonly StressHarnessOptions _options; + + public BackFillerE2EStressHarness(StressHarnessOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public async Task DiscoverAndWriteWorkloadAsync( + CancellationToken cancellationToken) + { + NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(_options.NewsLogDirectory); + Directory.CreateDirectory(_options.WorkloadDirectory); + var unique = new HashSet(StringComparer.Ordinal); + var stats = await NewsLogMessageIdDiscovery.DiscoverAsync( + _options.NewsLogDirectory, + unique, + cancellationToken) + .ConfigureAwait(false); + + var partitions = NewsLogMessageIdDiscovery.WriteWorkloadSets( + unique, + _options.WorkloadDirectory, + _options.Seed, + warmupCount: 1_000, + mainCount: 100_000, + repeatableCount: 2_000); + + var mainPath = Path.Join(_options.WorkloadDirectory, StressCorpusPaths.MainIdsAuditFileName); + var manifest = new WorkloadCorpusManifest + { + CorpusDirectory = Path.GetFullPath(_options.NewsLogDirectory), + FileCount = stats.FileCount, + LogRecordsExamined = stats.TotalLogEntries, + ValidParsedIds = stats.ValidIds, + RejectedInvalidMessageIds = stats.RejectedInvalidMessageIds, + UniqueValidIds = stats.UniqueIds, + WarmupSelected = partitions.Warmup, + MainSelected = partitions.Main, + RepeatableSelected = partitions.Repeatable, + Seed = partitions.Seed, + MainIdsSha256 = WorkloadCorpusManifest.ComputeFileSha256(mainPath), + CreatedUtc = DateTimeOffset.UtcNow, + Notes = + "Message-IDs extracted only from the configured news-log corpus after canonical validation. " + + "No synthetic fallback.", + }; + await WorkloadCorpusManifest.WriteAsync(_options.WorkloadDirectory, manifest, cancellationToken) + .ConfigureAwait(false); + + var statsPath = Path.Join(_options.WorkloadDirectory, "discovery-stats.json"); + var payload = new + { + CorpusDirectory = manifest.CorpusDirectory, + stats.FileCount, + stats.TotalLogEntries, + stats.ValidIds, + stats.UniqueIds, + stats.MalformedEntries, + stats.RejectedInvalidMessageIds, + stats.StatusPlus, + stats.StatusMinus, + stats.StatusJ, + stats.ProviderGiganews, + stats.ProviderEweka, + stats.ProviderOther, + stats.DuplicateRatePercent, + partitions.Warmup, + partitions.Main, + partitions.Repeatable, + partitions.Seed, + manifest.MainIdsSha256, + }; + await File.WriteAllTextAsync( + statsPath, + JsonSerializer.Serialize(payload, new JsonSerializerOptions { WriteIndented = true }), + cancellationToken) + .ConfigureAwait(false); + + return stats; + } + + public async Task RunAsync( + string runName, + string backbone, + string messageIdFile, + int concurrency, + int maxInFlight, + TimeSpan duration, + int? maxRequests, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(runName); + ArgumentException.ThrowIfNullOrWhiteSpace(backbone); + ArgumentException.ThrowIfNullOrWhiteSpace(messageIdFile); + ArgumentOutOfRangeException.ThrowIfLessThan(concurrency, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(maxInFlight, 1); + + // Static publish gate: corpus + provenance + SHA before any AMQP connect/publish. + NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(_options.NewsLogDirectory); + await StressWorkloadGuard.RequireProvenancedWorkloadAsync(_options, cancellationToken) + .ConfigureAwait(false); + + var ids = await LoadMessageIdsAsync(messageIdFile, cancellationToken).ConfigureAwait(false); + if (ids.Count == 0) + { + throw new InvalidOperationException($"Workload file '{messageIdFile}' contained no Message-IDs."); + } + + foreach (var id in ids) + { + if (!NewsLogMessageIdDiscovery.IsCanonicalMessageId(id)) + { + throw new InvalidOperationException( + "Workload contains a Message-ID that fails canonical validation: " + id + + ". Re-run discovery against the required corpus; do not invent replacements."); + } + } + + var runId = Guid.NewGuid().ToString("D"); + var metrics = new StressMetrics + { + RunName = runName, + Backbone = backbone, + ConfiguredConcurrency = concurrency, + ConfiguredMaxInFlight = maxInFlight, + }; + Directory.CreateDirectory(_options.ReportDirectory); + await File.WriteAllTextAsync( + Path.Join(_options.ReportDirectory, $"run-ids-{runName}-{runId}.txt"), + string.Join(Environment.NewLine, ids), + cancellationToken) + .ConfigureAwait(false); + await File.WriteAllTextAsync( + Path.Join(_options.ReportDirectory, $"run-meta-{runName}-{runId}.json"), + JsonSerializer.Serialize( + new + { + RunId = runId, + RunName = runName, + Backbone = backbone, + CorpusDirectory = _options.NewsLogDirectory, + MessageIdFile = messageIdFile, + SelectedCount = ids.Count, + Seed = _options.Seed, + }, + new JsonSerializerOptions { WriteIndented = true }), + cancellationToken) + .ConfigureAwait(false); + + var workClient = await ArticleWorkStressClient.ConnectAsync(_options, cancellationToken) + .ConfigureAwait(false); + workClient.StressRunId = runId; + + try + { + // Live retrieval preflight runs after the reply consumer is ready and before any throughput publish. + if (_options.ArticlePreflightCount > 0) + { + var preflight = await CorpusArticlePreflight.RunLiveRetrievalPreflightAsync( + workClient, + backbone, + ids, + _options.ArticlePreflightCount, + cancellationToken) + .ConfigureAwait(false); + await File.WriteAllTextAsync( + Path.Join(_options.ReportDirectory, $"preflight-articles-{runName}-{runId}.json"), + JsonSerializer.Serialize( + preflight.Select(static r => new { r.MessageId, Kind = r.Kind.ToString(), r.Detail }), + new JsonSerializerOptions { WriteIndented = true }), + cancellationToken) + .ConfigureAwait(false); + CorpusArticlePreflight.ThrowIfPreflightBlocksPublish(preflight); + } + + var listenerHost = _options.ListenerHostOverride; + var listenerPort = _options.ListenerPortOverride ?? 0; + BackFillerNntpConnectionPool? pool = null; + var poolGate = new object(); + + var workChannel = Channel.CreateBounded(new BoundedChannelOptions(maxInFlight) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = false, + SingleWriter = false, + }); + + var deliveryChannel = Channel.CreateBounded(new BoundedChannelOptions(maxInFlight) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = false, + SingleWriter = false, + }); + + using var runCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + using var feedCts = CancellationTokenSource.CreateLinkedTokenSource(runCts.Token); + if (duration > TimeSpan.Zero) + { + feedCts.CancelAfter(duration); + } + + metrics.SteadyStateStartTimestamp = Stopwatch.GetTimestamp(); + var queueName = ArticleWorkStressClient.ComposeProviderEntity(backbone); + var sampler = Task.Run( + async () => + { + while (!runCts.IsCancellationRequested) + { + metrics.SampleThroughput(Stopwatch.GetTimestamp()); + await SampleQueueDepthSafeAsync(metrics, queueName, runCts.Token).ConfigureAwait(false); + try + { + await Task.Delay(1000, runCts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + }, + CancellationToken.None); + + var feeder = Task.Run( + async () => + { + var index = 0; + var issued = 0; + try + { + while (!feedCts.IsCancellationRequested) + { + if (maxRequests is { } limit && issued >= limit) + { + break; + } + + var messageId = ids[index % ids.Count]; + index++; + issued++; + var item = new WorkItem(messageId, backbone, Stopwatch.GetTimestamp()); + await workChannel.Writer.WriteAsync(item, feedCts.Token).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // Feeder stops when the measured duration or abort cancels feedCts. + } + finally + { + workChannel.Writer.TryComplete(); + } + }, + CancellationToken.None); + + var publishers = new Task[Math.Min(concurrency, maxInFlight)]; + for (var i = 0; i < publishers.Length; i++) + { + publishers[i] = Task.Run( + async () => + { + await foreach (var item in workChannel.Reader.ReadAllAsync(CancellationToken.None) + .ConfigureAwait(false)) + { + if (runCts.IsCancellationRequested) + { + break; + } + + metrics.RequestStarted(); + try + { + var reply = await workClient.RequestAsync( + item.Backbone, + item.MessageId, + runCts.Token) + .ConfigureAwait(false); + metrics.RecordWorkReply(item.Backbone, reply.Outcome); + + var scheduleToPublish = reply.PublishCompletedTimestamp > 0 + ? (long)Stopwatch.GetElapsedTime(item.StartedTimestamp, reply.PublishCompletedTimestamp) + .TotalMilliseconds + : 0; + var publishCall = reply.PublishMilliseconds; + var publishToReply = reply.LatencyMilliseconds; + + if (reply.Outcome != ArticleWorkStressOutcome.Success + || string.IsNullOrWhiteSpace(reply.Fqdn) + || reply.ListenerPort is not { } port) + { + var failure = NntpArticleResponseReader.ClassifyFailure( + reply.Outcome, + nntpKind: null, + timedOut: reply.Outcome == ArticleWorkStressOutcome.NoResponse, + disconnected: false); + var endToEnd = (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds; + metrics.RecordStageTimings( + scheduleToPublish, + publishCall, + publishToReply, + replyToNntpStartMs: null, + nntpDeliveryMs: null, + endToEnd, + success: false, + logicalBytes: null); + metrics.RecordFailure(item.Backbone, failure, endToEndMs: null); + MaybeAbort(metrics, runCts); + continue; + } + + listenerHost ??= reply.Fqdn; + if (listenerPort == 0) + { + listenerPort = port; + } + + if (_options.ListenerHostOverride is not null) + { + listenerHost = _options.ListenerHostOverride; + } + + if (_options.ListenerPortOverride is { } overridePort) + { + listenerPort = overridePort; + } + + if (pool is null && listenerHost is not null && listenerPort > 0) + { + lock (poolGate) + { + pool ??= new BackFillerNntpConnectionPool( + listenerHost, + listenerPort, + concurrency); + } + } + + await deliveryChannel.Writer.WriteAsync( + new DeliveryItem( + item.MessageId, + item.Backbone, + item.StartedTimestamp, + reply.RequestId, + reply.CorrelationId, + scheduleToPublish, + publishCall, + publishToReply, + reply.ReplyReceivedTimestamp, + listenerHost!, + listenerPort), + CancellationToken.None) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (runCts.IsCancellationRequested) + { + break; + } + // codeql[cs/catch-of-all-exceptions] Publisher worker must record any broker/RPC failure and keep flooding other items. + catch (Exception) + { + metrics.RecordFailure( + item.Backbone, + StressFailureClass.Other, + (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds); + MaybeAbort(metrics, runCts); + } + finally + { + metrics.RequestFinished(); + } + } + }, + CancellationToken.None); + } + + var deliveryWorkers = new Task[concurrency]; + for (var i = 0; i < deliveryWorkers.Length; i++) + { + deliveryWorkers[i] = Task.Run( + async () => + { + await foreach (var item in deliveryChannel.Reader.ReadAllAsync(CancellationToken.None) + .ConfigureAwait(false)) + { + BackFillerNntpConnectionPool? localPool; + lock (poolGate) + { + localPool = pool; + } + + if (localPool is null) + { + metrics.RecordFailure(item.Backbone, StressFailureClass.Other, null); + continue; + } + + metrics.DeliveryStarted(); + BackFillerNntpStressClient? client = null; + var healthy = false; + try + { + using var articleCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + articleCts.CancelAfter(_options.ArticleTimeout); + client = await localPool.RentAsync(metrics, articleCts.Token).ConfigureAwait(false); + var nntpStarted = Stopwatch.GetTimestamp(); + var replyToNntp = item.ReplyReceivedTimestamp > 0 + ? (long)Stopwatch.GetElapsedTime(item.ReplyReceivedTimestamp, nntpStarted) + .TotalMilliseconds + : 0; + var exchange = await client.ArticleAsync(item.MessageId, articleCts.Token) + .ConfigureAwait(false); + var nntpMs = (long)Stopwatch.GetElapsedTime(nntpStarted).TotalMilliseconds; + var endToEndMs = (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds; + + if (exchange.Kind == NntpResponseKind.ArticleDelivered) + { + metrics.RecordStageTimings( + item.ScheduleToPublishMs, + item.PublishCallMs, + item.PublishToReplyMs, + replyToNntp, + nntpMs, + endToEndMs, + success: true, + exchange.LogicalArticleBytes); + metrics.RecordArticleDelivered( + item.Backbone, + exchange.LogicalArticleBytes, + exchange.WireArticleBytes, + endToEndMs, + nntpMs); + healthy = true; + } + else + { + var failure = NntpArticleResponseReader.ClassifyFailure( + ArticleWorkStressOutcome.Success, + exchange.Kind, + timedOut: false, + disconnected: exchange.Kind == NntpResponseKind.Disconnect); + metrics.RecordStageTimings( + item.ScheduleToPublishMs, + item.PublishCallMs, + item.PublishToReplyMs, + replyToNntp, + nntpMs, + endToEndMs, + success: false, + logicalBytes: null); + metrics.RecordFailure(item.Backbone, failure, endToEndMs: null, exchange.Kind); + healthy = exchange.Kind is NntpResponseKind.NotFound + or NntpResponseKind.NotAvailable + or NntpResponseKind.SyntaxError; + MaybeAbort(metrics, runCts); + } + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + metrics.RecordFailure( + item.Backbone, + StressFailureClass.Timeout, + (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds); + MaybeAbort(metrics, runCts); + } + // codeql[cs/catch-of-all-exceptions] Delivery worker must classify any NNTP/transport failure and return the leased client. + catch (Exception) + { + metrics.RecordFailure( + item.Backbone, + StressFailureClass.Disconnect, + (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds); + MaybeAbort(metrics, runCts); + } + finally + { + if (client is not null) + { + localPool.Return(client, metrics, healthy); + } + + metrics.DeliveryFinished(); + } + } + }, + CancellationToken.None); + } + + _ = Task.Run( + async () => + { + await Task.WhenAll(publishers).ConfigureAwait(false); + deliveryChannel.Writer.TryComplete(); + }, + CancellationToken.None); + + try + { + await Task.WhenAll(feeder).ConfigureAwait(false); + await Task.WhenAll(publishers).ConfigureAwait(false); + await Task.WhenAll(deliveryWorkers).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Measured run cancelled via abort or host token; continue into drain/dispose. + } + finally + { + runCts.Cancel(); + try + { + await sampler.ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Queue-depth sampler is optional observability and must not fail the soak. + catch (Exception) + { + // Sampler faults are non-fatal. + } + + metrics.SteadyStateEndTimestamp = Stopwatch.GetTimestamp(); + } + + var drainStarted = Stopwatch.GetTimestamp(); + var drainDeadline = drainStarted + + (long)(_options.ReplyQueueDisposeDrainTimeout.TotalSeconds * Stopwatch.Frequency); + while ((metrics.Outstanding > 0 + || metrics.DeliveryOutstanding > 0 + || workClient.WaitingCount > 0 + || workClient.BrokerOutstandingCount > 0) + && Stopwatch.GetTimestamp() < drainDeadline) + { + await Task.Delay(100, CancellationToken.None).ConfigureAwait(false); + } + + metrics.SetDrainMilliseconds((long)Stopwatch.GetElapsedTime(drainStarted).TotalMilliseconds); + var brokerOutstandingAtDrainEnd = workClient.BrokerOutstandingCount; + if (brokerOutstandingAtDrainEnd > 0) + { + Console.Error.WriteLine( + $"Harness drain ended with brokerOutstanding={brokerOutstandingAtDrainEnd} waiting={workClient.WaitingCount}; dispose will apply bounded ReplyTo drain."); + } + + if (pool is not null) + { + await pool.DisposeAsync().ConfigureAwait(false); + } + + await workClient.DisposeAsync().ConfigureAwait(false); + metrics.SetReplyLifecycleCounters( + workClient.LateRepliesAfterTimeout, + workClient.UnresolvedAtDispose, + brokerOutstandingAtDrainEnd); + + var summary = metrics.BuildSummary(Process.GetCurrentProcess()); + var safeName = SanitizeFileName(runName); + metrics.WriteJson(Path.Join(_options.ReportDirectory, safeName + ".json"), Process.GetCurrentProcess()); + await File.AppendAllTextAsync( + Path.Join(_options.ReportDirectory, "summary-table.txt"), + summary.FormatTableRow() + Environment.NewLine, + CancellationToken.None) + .ConfigureAwait(false); + return summary; + } + // codeql[cs/catch-of-all-exceptions] Run failure must still dispose the exclusive ReplyTo client before rethrowing. + catch (Exception) + { + await workClient.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + private async Task SampleQueueDepthSafeAsync( + StressMetrics metrics, + string queueName, + CancellationToken cancellationToken) + { + try + { + var user = _options.RabbitMqUsername; + var pass = _options.RabbitMqPassword; + var creds = Convert.ToBase64String(Encoding.ASCII.GetBytes(user + ":" + pass)); + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(2) }; + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", creds); + var vhost = Uri.EscapeDataString(_options.RabbitMqVirtualHost); + var q = Uri.EscapeDataString(queueName); + var url = $"http://{_options.RabbitMqHost}:15672/api/queues/{vhost}/{q}"; + using var response = await http.GetAsync(url, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + return; + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken) + .ConfigureAwait(false); + using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken) + .ConfigureAwait(false); + var root = doc.RootElement; + var messages = root.TryGetProperty("messages", out var m) ? m.GetInt64() : 0; + var ready = root.TryGetProperty("messages_ready", out var r) ? r.GetInt64() : 0; + var unacked = root.TryGetProperty("messages_unacknowledged", out var u) ? u.GetInt64() : 0; + var consumers = root.TryGetProperty("consumers", out var c) ? c.GetInt64() : 0; + metrics.SampleQueueDepth(Stopwatch.GetTimestamp(), queueName, messages, ready, unacked, consumers); + } + // codeql[cs/catch-of-all-exceptions] Management API sampling is optional; any fault must not abort the soak. + catch (Exception) + { + // Optional observability; never fail the run. + } + } + + private void MaybeAbort(StressMetrics metrics, CancellationTokenSource runCts) + { + var summary = metrics.BuildSummary(); + if (StressAbortDecision.ShouldAbort( + summary.RequestsStarted, + summary.WorkSuccess, + summary.WorkNotFound, + summary.WorkOtherFailure, + summary.Timeouts, + summary.ArticlesDelivered, + _options.AbortAfterRequests, + _options.AbortOnTimeoutRate, + _options.AbortOnFailureRate)) + { + runCts.Cancel(); + } + } + + private static async Task> LoadMessageIdsAsync( + string path, + CancellationToken cancellationToken) + { + var lines = await File.ReadAllLinesAsync(path, cancellationToken).ConfigureAwait(false); + return lines + .Where(static line => !string.IsNullOrWhiteSpace(line) && line.StartsWith('<')) + .ToArray(); + } + + private static string SanitizeFileName(string runName) + { + var invalid = Path.GetInvalidFileNameChars(); + var chars = runName.Select(ch => invalid.Contains(ch) ? '_' : ch).ToArray(); + return new string(chars); + } + + private readonly record struct WorkItem(string MessageId, string Backbone, long StartedTimestamp); + + private readonly record struct DeliveryItem( + string MessageId, + string Backbone, + long StartedTimestamp, + Guid? RequestId, + string? CorrelationId, + long ScheduleToPublishMs, + long PublishCallMs, + long PublishToReplyMs, + long ReplyReceivedTimestamp, + string Host, + int Port); + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs new file mode 100644 index 00000000..8455d542 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs @@ -0,0 +1,269 @@ +using System.Collections.Concurrent; +using System.Net.Security; +using System.Net.Sockets; +using System.Security.Authentication; +using System.Text; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// TLS NNTP client for BackFiller's cache listener subset: greeting, ARTICLE, QUIT. + /// + public sealed class BackFillerNntpStressClient : IAsyncDisposable + { + private readonly string _host; + private readonly int _port; + private readonly bool _acceptAnyServerCertificate; + private TcpClient? _tcp; + private SslStream? _ssl; + private Stream? _stream; + private readonly byte[] _lineBuffer = new byte[512]; + private readonly byte[] _readBuffer = new byte[64 * 1024]; + private readonly byte[] _commandBuffer = new byte[512]; + private int _disposed; + + public BackFillerNntpStressClient(string host, int port, bool acceptAnyServerCertificate = true) + { + ArgumentException.ThrowIfNullOrWhiteSpace(host); + ArgumentOutOfRangeException.ThrowIfLessThan(port, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(port, 65535); + _host = host; + _port = port; + _acceptAnyServerCertificate = acceptAnyServerCertificate; + } + + public bool IsConnected => _stream is not null && _tcp is { Connected: true }; + + /// Connects, completes TLS, and consumes the 201 greeting. + public async Task ConnectAsync(CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + if (IsConnected) + { + return; + } + + var tcp = new TcpClient(); + tcp.NoDelay = true; + await tcp.ConnectAsync(_host, _port, cancellationToken).ConfigureAwait(false); + var ssl = new SslStream( + tcp.GetStream(), + leaveInnerStreamOpen: false, + userCertificateValidationCallback: _acceptAnyServerCertificate + ? static (_, _, _, _) => true + : null); + await ssl.AuthenticateAsClientAsync( + new SslClientAuthenticationOptions + { + TargetHost = _host, + EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13, + }, + cancellationToken) + .ConfigureAwait(false); + + _tcp = tcp; + _ssl = ssl; + _stream = ssl; + + var greeting = await NntpArticleResponseReader.ReadStatusLineAsync( + _stream, + _lineBuffer, + cancellationToken) + .ConfigureAwait(false); + if (greeting.Code is not (200 or 201)) + { + throw new InvalidOperationException($"Unexpected NNTP greeting code {greeting.Code}."); + } + } + + /// + /// Sends ARTICLE <message-id> and fully consumes the response before returning. + /// + public async Task ArticleAsync( + string messageId, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(messageId); + ObjectDisposedException.ThrowIf(_disposed != 0, this); + if (_stream is null) + { + throw new InvalidOperationException("Not connected."); + } + + var commandLength = Encoding.ASCII.GetBytes("ARTICLE ", _commandBuffer.AsSpan()); + var idBytes = Encoding.ASCII.GetBytes(messageId); + if (commandLength + idBytes.Length + 2 > _commandBuffer.Length) + { + return new NntpArticleExchangeResult( + NntpResponseKind.ProtocolError, + 0, + 0, + 0, + "ARTICLE command exceeds 512-octet limit"); + } + + idBytes.CopyTo(_commandBuffer.AsSpan(commandLength)); + commandLength += idBytes.Length; + _commandBuffer[commandLength++] = (byte)'\r'; + _commandBuffer[commandLength++] = (byte)'\n'; + await _stream.WriteAsync(_commandBuffer.AsMemory(0, commandLength), cancellationToken) + .ConfigureAwait(false); + await _stream.FlushAsync(cancellationToken).ConfigureAwait(false); + + return await NntpArticleResponseReader.ReadArticleExchangeAsync( + _stream, + _lineBuffer, + _readBuffer, + cancellationToken) + .ConfigureAwait(false); + } + + /// Sends QUIT and closes the transport. + public async Task QuitAsync(CancellationToken cancellationToken) + { + if (_stream is null) + { + return; + } + + try + { + await _stream.WriteAsync("QUIT\r\n"u8.ToArray(), cancellationToken).ConfigureAwait(false); + await _stream.FlushAsync(cancellationToken).ConfigureAwait(false); + _ = await NntpArticleResponseReader.ReadStatusLineAsync(_stream, _lineBuffer, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + // codeql[cs/catch-of-all-exceptions] QUIT under load may fail with any transport fault; dispose must still run. + catch (Exception) + { + // Peer may close without 205 under load; still dispose. + } + finally + { + await DisposeAsync().ConfigureAwait(false); + } + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + if (_ssl is not null) + { + await _ssl.DisposeAsync().ConfigureAwait(false); + } + + _tcp?.Dispose(); + _stream = null; + _ssl = null; + _tcp = null; + } + } + + /// Pool of reusable TLS NNTP sessions with bounded size. + public sealed class BackFillerNntpConnectionPool : IAsyncDisposable + { + private readonly string _host; + private readonly int _port; + private readonly int _capacity; + private readonly ConcurrentBag _idle = []; + private int _created; + private int _disposed; + + public BackFillerNntpConnectionPool(string host, int port, int capacity) + { + ArgumentException.ThrowIfNullOrWhiteSpace(host); + ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + _host = host; + _port = port; + _capacity = capacity; + } + + public async Task RentAsync( + StressMetrics metrics, + CancellationToken cancellationToken) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + while (_idle.TryTake(out var existing)) + { + if (existing.IsConnected) + { + return existing; + } + + await existing.DisposeAsync().ConfigureAwait(false); + Interlocked.Decrement(ref _created); + metrics.ConnectionClosed(); + } + + while (true) + { + var created = Interlocked.Increment(ref _created); + if (created <= _capacity) + { + var client = new BackFillerNntpStressClient(_host, _port); + await client.ConnectAsync(cancellationToken).ConfigureAwait(false); + metrics.ConnectionOpened(); + return client; + } + + Interlocked.Decrement(ref _created); + // Wait briefly for a return under saturation. + await Task.Delay(1, cancellationToken).ConfigureAwait(false); + if (_idle.TryTake(out var recycled) && recycled.IsConnected) + { + return recycled; + } + + if (recycled is not null) + { + await recycled.DisposeAsync().ConfigureAwait(false); + Interlocked.Decrement(ref _created); + metrics.ConnectionClosed(); + } + } + } + + public void Return(BackFillerNntpStressClient client, StressMetrics metrics, bool healthy) + { + if (_disposed != 0 || !healthy || !client.IsConnected) + { + _ = client.DisposeAsync().AsTask(); + Interlocked.Decrement(ref _created); + metrics.ConnectionClosed(); + return; + } + + _idle.Add(client); + } + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + while (_idle.TryTake(out var client)) + { + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await client.QuitAsync(cts.Token).ConfigureAwait(false); + } + // codeql[cs/catch-of-all-exceptions] Pool dispose must force-close idle clients after any QUIT failure. + catch (Exception) + { + await client.DisposeAsync().ConfigureAwait(false); + } + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs new file mode 100644 index 00000000..962cebae --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs @@ -0,0 +1,139 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Bounded preflight checks over selected corpus Message-IDs before throughput measurement. + /// + /// + /// Does not invent replacement IDs when retrieval fails. Live RPC preflight is opt-in via + /// and records explicit per-ID outcomes. + /// + public static class CorpusArticlePreflight + { + /// + /// Confirms every selected ID is present in the discovered corpus set and is canonically valid. + /// + public static IReadOnlyList ValidateSelectedAgainstCorpus( + IReadOnlyList selectedIds, + IReadOnlySet corpusUniqueIds) + { + ArgumentNullException.ThrowIfNull(selectedIds); + ArgumentNullException.ThrowIfNull(corpusUniqueIds); + var results = new List(selectedIds.Count); + foreach (var id in selectedIds) + { + if (!NewsLogMessageIdDiscovery.IsCanonicalMessageId(id)) + { + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.InvalidMessageId, "canonical validation failed")); + continue; + } + + if (!corpusUniqueIds.Contains(id)) + { + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.NotInCorpus, "not present in discovery set")); + continue; + } + + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.InCorpus, null)); + } + + return results; + } + + /// + /// Publishes a bounded number of Article Work RPCs for the selected IDs and records outcomes. + /// Failed retrievals are recorded; IDs are not replaced. + /// + public static async Task> RunLiveRetrievalPreflightAsync( + ArticleWorkStressClient client, + string backbone, + IReadOnlyList selectedIds, + int maxIds, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(client); + ArgumentException.ThrowIfNullOrWhiteSpace(backbone); + ArgumentNullException.ThrowIfNull(selectedIds); + ArgumentOutOfRangeException.ThrowIfLessThan(maxIds, 1); + + var limit = Math.Min(maxIds, selectedIds.Count); + var results = new List(limit); + for (var i = 0; i < limit; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var id = selectedIds[i]; + try + { + var reply = await client.RequestAsync(backbone, id, cancellationToken).ConfigureAwait(false); + var kind = reply.Outcome switch + { + ArticleWorkStressOutcome.Success => CorpusIdPreflightKind.Retrieved, + ArticleWorkStressOutcome.ArticleNotFound => CorpusIdPreflightKind.NotFound, + ArticleWorkStressOutcome.NoResponse => CorpusIdPreflightKind.Timeout, + _ => CorpusIdPreflightKind.RetrievalFailed, + }; + results.Add(new CorpusIdPreflightResult(id, kind, reply.Error ?? reply.Outcome.ToString())); + } + // codeql[cs/catch-of-all-exceptions] Live preflight must record every retrieval fault as an explicit per-ID outcome. + catch (Exception ex) when (ex is not OperationCanceledException) + { + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.RetrievalFailed, ex.GetType().Name)); + } + } + + return results; + } + + /// + /// Throws when preflight results must block throughput publishing. + /// + /// + /// Blocks on invalid / non-corpus IDs, and when every probed ID fails retrieval + /// (no successful retrieve). Does not invent replacement Message-IDs. + /// Partial NotFound among some Retrieved successes is recorded by the caller and does not throw. + /// + public static void ThrowIfPreflightBlocksPublish(IReadOnlyList results) + { + ArgumentNullException.ThrowIfNull(results); + if (results.Count == 0) + { + throw new InvalidOperationException( + "Article preflight produced no results. Stress will not publish without a completed preflight."); + } + + foreach (var result in results) + { + if (result.Kind is CorpusIdPreflightKind.InvalidMessageId or CorpusIdPreflightKind.NotInCorpus) + { + throw new InvalidOperationException( + "Article preflight blocked publish: " + result.Kind + " for " + result.MessageId + + (result.Detail is null ? string.Empty : (" (" + result.Detail + ")")) + + ". No replacement Message-IDs will be generated."); + } + } + + var retrieved = results.Count(static result => result.Kind == CorpusIdPreflightKind.Retrieved); + + if (retrieved == 0) + { + throw new InvalidOperationException( + "Article preflight blocked publish: none of the " + + results.Count + + " probed corpus Message-IDs retrieved successfully. " + + "Failures are explicit; Message-IDs were not replaced."); + } + } + } + + public enum CorpusIdPreflightKind + { + InCorpus, + NotInCorpus, + InvalidMessageId, + Retrieved, + NotFound, + Timeout, + RetrievalFailed, + } + + public sealed record CorpusIdPreflightResult(string MessageId, CorpusIdPreflightKind Kind, string? Detail); +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscovery.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscovery.cs new file mode 100644 index 00000000..7b1635f4 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscovery.cs @@ -0,0 +1,365 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using VectorNNTPD.Common.Articles.Validation; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Streams news-log files and extracts Message-IDs without loading the full corpus into memory. + /// + /// + /// Expected line shape (syslog-style prefix, then status, provider, Message-ID, size): + /// Oct 17 00:00:00.026 + giganews <id@host> 740121 ?. + /// Status is one of +, -, or j. Provider labels are preserved as logged. + /// Extracted Message-IDs must also pass . + /// This type never fabricates Message-IDs. + /// + public static class NewsLogMessageIdDiscovery + { + /// Matches one news-log article line and captures status, provider, Message-ID, and size. + private static readonly Regex LinePattern = new( + @"^[A-Za-z]{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\.\d{3}\s+([+\-j])\s+(\S+)\s+(<[^>\s]+>)\s+(\d+)\b", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + /// + /// Ensures exists and contains at least one *.log file. + /// + /// Missing directory, empty corpus, or unreadable root. + public static void RequireUsableCorpusDirectory(string rootDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(rootDirectory); + if (!Directory.Exists(rootDirectory)) + { + throw new InvalidOperationException( + "News-log corpus directory is missing or unreadable: " + rootDirectory + + ". Required default is " + StressCorpusPaths.RequiredNewsLogDirectory + + ". Stress will not invent Message-IDs or fall back to another dataset."); + } + + if (DiscoverLogFiles(rootDirectory).Count == 0) + { + throw new InvalidOperationException( + "News-log corpus contains no *.log files: " + rootDirectory + + ". Stress will not invent Message-IDs or fall back to another dataset."); + } + } + + /// Discovers log files under recursively. + /// Directory that may contain nested *.log files. + /// Ordinal-sorted absolute paths. + public static IReadOnlyList DiscoverLogFiles(string rootDirectory) + { + ArgumentException.ThrowIfNullOrWhiteSpace(rootDirectory); + if (!Directory.Exists(rootDirectory)) + { + return Array.Empty(); + } + + return Directory.EnumerateFiles(rootDirectory, "*.log", SearchOption.AllDirectories) + .OrderBy(static path => path, StringComparer.Ordinal) + .ToArray(); + } + + /// + /// Returns when passes canonical RFC 5536 validation. + /// + public static bool IsCanonicalMessageId(string messageId) + { + if (string.IsNullOrWhiteSpace(messageId)) + { + return false; + } + + var byteCount = Encoding.ASCII.GetByteCount(messageId); + Span buffer = byteCount <= 256 ? stackalloc byte[byteCount] : new byte[byteCount]; + Encoding.ASCII.GetBytes(messageId, buffer); + return NntpMessageIdValidation.IsValidMessageId(buffer); + } + + /// + /// Parses one news-log line. + /// + /// Raw log line. + /// Parsed fields when the line matches. + /// when the line is a valid article log entry. + public static bool TryParseLine(ReadOnlySpan line, out NewsLogEntry entry) + { + entry = default; + // Regex has no Span API on all targets; convert only for match. + var text = line.ToString(); + var match = LinePattern.Match(text); + if (!match.Success) + { + return false; + } + + var status = match.Groups[1].Value[0]; + var provider = match.Groups[2].Value; + var messageId = match.Groups[3].Value; + if (!long.TryParse(match.Groups[4].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var size)) + { + return false; + } + + entry = new NewsLogEntry(status, provider, messageId, size); + return true; + } + + /// + /// Streams every discovered log file, counts entries, and optionally records unique Message-IDs. + /// + /// News-log root. + /// + /// When non-null, unique Message-IDs are added here. Callers that must bound memory may pass + /// and rely on counts only, or pass a set and clear it after writing a workload file. + /// + /// Cancels mid-file. + /// Aggregate discovery statistics. + public static async Task DiscoverAsync( + string rootDirectory, + ISet? uniqueMessageIds, + CancellationToken cancellationToken = default) + { + RequireUsableCorpusDirectory(rootDirectory); + var files = DiscoverLogFiles(rootDirectory); + long totalLines = 0; + long valid = 0; + long rejectedInvalid = 0; + long malformed = 0; + long statusPlus = 0; + long statusMinus = 0; + long statusJ = 0; + long giganews = 0; + long eweka = 0; + long otherProvider = 0; + var perFile = new List(files.Count); + + foreach (var path in files) + { + cancellationToken.ThrowIfCancellationRequested(); + long fileLines = 0; + long fileValid = 0; + long fileGig = 0; + long fileEwe = 0; + await foreach (var line in ReadLinesAsync(path, cancellationToken).ConfigureAwait(false)) + { + fileLines++; + totalLines++; + if (!TryParseLine(line.AsSpan(), out var entry)) + { + malformed++; + continue; + } + + if (!IsCanonicalMessageId(entry.MessageId)) + { + rejectedInvalid++; + continue; + } + + valid++; + fileValid++; + uniqueMessageIds?.Add(entry.MessageId); + switch (entry.Status) + { + case '+': + statusPlus++; + break; + case '-': + statusMinus++; + break; + case 'j': + statusJ++; + break; + } + + if (entry.Provider.Equals("giganews", StringComparison.OrdinalIgnoreCase)) + { + giganews++; + fileGig++; + } + else if (entry.Provider.Equals("eweka", StringComparison.OrdinalIgnoreCase)) + { + eweka++; + fileEwe++; + } + else + { + otherProvider++; + } + } + + perFile.Add(new NewsLogFileStats(path, fileLines, fileValid, fileGig, fileEwe)); + } + + var uniqueCount = uniqueMessageIds?.Count ?? 0L; + if (valid == 0 || uniqueCount == 0) + { + throw new InvalidOperationException( + "News-log corpus produced no usable Message-IDs after parse and canonical validation: " + + rootDirectory + + $". examined={totalLines} malformed={malformed} rejectedInvalid={rejectedInvalid}. " + + "Stress will not invent replacement Message-IDs."); + } + + var duplicateRate = valid > 0 + ? 100.0 * (valid - uniqueCount) / valid + : 0.0; + return new NewsLogDiscoveryStats( + files.Count, + totalLines, + valid, + uniqueCount, + malformed, + rejectedInvalid, + statusPlus, + statusMinus, + statusJ, + giganews, + eweka, + otherProvider, + duplicateRate, + perFile); + } + + /// + /// Writes a deterministic workload partition from unique Message-IDs. + /// + /// Unique IDs (order is not assumed). + /// Destination for workload text files. + /// Shuffle seed for reproducibility. + /// Warm-up set size. + /// Main stress set size. + /// Repeatable comparison set size. + /// Counts actually written for each partition. + public static WorkloadPartitionCounts WriteWorkloadSets( + IEnumerable uniqueMessageIds, + string outputDirectory, + int seed, + int warmupCount, + int mainCount, + int repeatableCount) + { + ArgumentNullException.ThrowIfNull(uniqueMessageIds); + ArgumentException.ThrowIfNullOrWhiteSpace(outputDirectory); + Directory.CreateDirectory(outputDirectory); + + // Sort before shuffle so HashSet enumeration order cannot affect reproducibility. + var ids = uniqueMessageIds + .Distinct(StringComparer.Ordinal) + .OrderBy(static id => id, StringComparer.Ordinal) + .ToArray(); + if (ids.Length == 0) + { + throw new InvalidOperationException( + "Cannot write workload sets: no Message-IDs were supplied. " + + "Stress will not invent replacement Message-IDs."); + } + + Shuffle(ids, seed); + + var warmup = TakeSlice(ids, 0, warmupCount); + var mainStart = warmup.Length; + var main = TakeSlice(ids, mainStart, mainCount); + var repeatStart = mainStart + main.Length; + var repeatable = TakeSlice(ids, repeatStart, repeatableCount); + if (repeatable.Length == 0 && ids.Length > 0) + { + // Prefer a non-empty repeatable set when the corpus is smaller than the requested slices. + repeatable = TakeSlice(ids, 0, Math.Min(repeatableCount, ids.Length)); + } + + File.WriteAllLines(Path.Join(outputDirectory, "warmup-message-ids.txt"), warmup); + File.WriteAllLines(Path.Join(outputDirectory, "main-message-ids.txt"), main); + File.WriteAllLines(Path.Join(outputDirectory, "repeatable-message-ids.txt"), repeatable); + return new WorkloadPartitionCounts(warmup.Length, main.Length, repeatable.Length, seed); + } + + private static string[] TakeSlice(string[] ids, int start, int count) + { + if (start >= ids.Length || count <= 0) + { + return Array.Empty(); + } + + var length = Math.Min(count, ids.Length - start); + var slice = new string[length]; + Array.Copy(ids, start, slice, 0, length); + return slice; + } + + private static void Shuffle(string[] ids, int seed) + { + var rng = new Random(seed); + for (var i = ids.Length - 1; i > 0; i--) + { + var j = rng.Next(i + 1); + (ids[i], ids[j]) = (ids[j], ids[i]); + } + } + + private static async IAsyncEnumerable ReadLinesAsync( + string path, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + using var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite, + bufferSize: 1024 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan); + using var reader = new StreamReader(stream); + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false); + if (line is null) + { + yield break; + } + + yield return line; + } + } + } + + /// One parsed news-log article line. + /// Log status character (+, -, or j). + /// Provider label as logged (for example giganews). + /// Message-ID including angle brackets. + /// Size field from the log line. + public readonly record struct NewsLogEntry(char Status, string Provider, string MessageId, long LoggedSize); + + /// Per-file discovery counters. + public sealed record NewsLogFileStats( + string Path, + long LineCount, + long ValidCount, + long GiganewsCount, + long EwekaCount); + + /// Aggregate news-log discovery statistics. + /// + /// Lines that matched the log shape but failed . + /// + public sealed record NewsLogDiscoveryStats( + int FileCount, + long TotalLogEntries, + long ValidIds, + long UniqueIds, + long MalformedEntries, + long RejectedInvalidMessageIds, + long StatusPlus, + long StatusMinus, + long StatusJ, + long ProviderGiganews, + long ProviderEweka, + long ProviderOther, + double DuplicateRatePercent, + IReadOnlyList PerFile); + + /// Sizes of written workload partitions. + public sealed record WorkloadPartitionCounts(int Warmup, int Main, int Repeatable, int Seed); +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscoveryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscoveryTests.cs new file mode 100644 index 00000000..e189e053 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscoveryTests.cs @@ -0,0 +1,216 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + public sealed class NewsLogMessageIdDiscoveryTests + { + [Theory] + [InlineData( + "Oct 17 00:00:00.026 + giganews 740121 ?", + '+', + "giganews", + "", + 740121L)] + [InlineData( + "Oct 17 00:00:00.075 j giganews 741054 newsgroup not carried: x", + 'j', + "giganews", + "", + 741054L)] + [InlineData( + "Oct 17 00:00:01.000 - eweka 12 fail", + '-', + "eweka", + "", + 12L)] + public void TryParseLine_accepts_known_formats( + string line, + char status, + string provider, + string messageId, + long size) + { + Assert.True(NewsLogMessageIdDiscovery.TryParseLine(line.AsSpan(), out var entry)); + Assert.Equal(status, entry.Status); + Assert.Equal(provider, entry.Provider); + Assert.Equal(messageId, entry.MessageId); + Assert.Equal(size, entry.LoggedSize); + } + + [Theory] + [InlineData("")] + [InlineData("not a log line")] + [InlineData("Oct 17 00:00:00.026 + giganews missing-brackets 1")] + public void TryParseLine_rejects_malformed(string line) + { + Assert.False(NewsLogMessageIdDiscovery.TryParseLine(line.AsSpan(), out _)); + } + + [Fact] + public async Task DiscoverAsync_streams_and_deduplicates() + { + var root = Path.Join(Path.GetTempPath(), "vnntp-newslog-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + var path = Path.Join(root, "sample.log"); + await File.WriteAllLinesAsync( + path, + [ + "Oct 17 00:00:00.026 + giganews 10 ?", + "Oct 17 00:00:00.027 + giganews 10 ?", + "Oct 17 00:00:00.028 + eweka 20 ?", + "garbage", + ]); + + var unique = new HashSet(StringComparer.Ordinal); + var stats = await NewsLogMessageIdDiscovery.DiscoverAsync(root, unique); + Assert.Equal(1, stats.FileCount); + Assert.Equal(4, stats.TotalLogEntries); + Assert.Equal(3, stats.ValidIds); + Assert.Equal(2, stats.UniqueIds); + Assert.Equal(1, stats.MalformedEntries); + Assert.Equal(0, stats.RejectedInvalidMessageIds); + Assert.Equal(2, stats.ProviderGiganews); + Assert.Equal(1, stats.ProviderEweka); + Assert.Contains("", unique); + Assert.Contains("", unique); + + var workloadDir = Path.Join(root, "workload"); + var partitions = NewsLogMessageIdDiscovery.WriteWorkloadSets( + unique, + workloadDir, + seed: 7, + warmupCount: 1, + mainCount: 1, + repeatableCount: 1); + Assert.Equal(1, partitions.Warmup); + Assert.Equal(1, partitions.Main); + Assert.True(File.Exists(Path.Join(workloadDir, "warmup-message-ids.txt"))); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void RequireUsableCorpusDirectory_fails_when_missing() + { + var missing = Path.Join(Path.GetTempPath(), "vnntp-missing-" + Guid.NewGuid().ToString("N")); + var ex = Assert.Throws( + () => NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(missing)); + Assert.Contains("missing", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains(StressCorpusPaths.RequiredNewsLogDirectory, ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void RequireUsableCorpusDirectory_fails_when_empty() + { + var root = Path.Join(Path.GetTempPath(), "vnntp-empty-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + var ex = Assert.Throws( + () => NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(root)); + Assert.Contains("no *.log", ex.Message, StringComparison.OrdinalIgnoreCase); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public async Task DiscoverAsync_rejects_non_canonical_message_ids_without_fabricating() + { + var root = Path.Join(Path.GetTempPath(), "vnntp-badid-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + await File.WriteAllLinesAsync( + Path.Join(root, "sample.log"), + [ + // Quoted local-part is not RFC 5536 msg-id; must not enter the unique set. + "Oct 17 00:00:00.026 + giganews <\"quoted\"@host> 10 ?", + "Oct 17 00:00:00.027 + giganews 11 ?", + ]); + + var unique = new HashSet(StringComparer.Ordinal); + var stats = await NewsLogMessageIdDiscovery.DiscoverAsync(root, unique); + Assert.Equal(1, stats.ValidIds); + Assert.Equal(1, stats.UniqueIds); + Assert.True(stats.RejectedInvalidMessageIds >= 1); + Assert.Contains("", unique); + Assert.DoesNotContain("<\"quoted\"@host>", unique); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public async Task DiscoverAsync_fails_when_corpus_yields_no_usable_ids() + { + var root = Path.Join(Path.GetTempPath(), "vnntp-nousable-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + try + { + await File.WriteAllLinesAsync( + Path.Join(root, "sample.log"), + ["garbage", "still garbage"]); + var unique = new HashSet(StringComparer.Ordinal); + var ex = await Assert.ThrowsAsync( + () => NewsLogMessageIdDiscovery.DiscoverAsync(root, unique)); + Assert.Contains("no usable Message-IDs", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Empty(unique); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Fact] + public void WriteWorkloadSets_is_reproducible_for_same_seed() + { + string[] source = ["", "", ""]; + var dir1 = Path.Join(Path.GetTempPath(), "vnntp-w1-" + Guid.NewGuid().ToString("N")); + var dir2 = Path.Join(Path.GetTempPath(), "vnntp-w2-" + Guid.NewGuid().ToString("N")); + try + { + NewsLogMessageIdDiscovery.WriteWorkloadSets(source, dir1, seed: 42, 1, 2, 1); + NewsLogMessageIdDiscovery.WriteWorkloadSets(source, dir2, seed: 42, 1, 2, 1); + Assert.Equal( + File.ReadAllText(Path.Join(dir1, "main-message-ids.txt")), + File.ReadAllText(Path.Join(dir2, "main-message-ids.txt"))); + } + finally + { + if (Directory.Exists(dir1)) + { + Directory.Delete(dir1, recursive: true); + } + + if (Directory.Exists(dir2)) + { + Directory.Delete(dir2, recursive: true); + } + } + } + + [Fact] + public void WriteWorkloadSets_fails_on_empty_input_without_synthetic_fallback() + { + var dir = Path.Join(Path.GetTempPath(), "vnntp-wempty-" + Guid.NewGuid().ToString("N")); + var ex = Assert.Throws( + () => NewsLogMessageIdDiscovery.WriteWorkloadSets( + Array.Empty(), + dir, + seed: 1, + warmupCount: 1, + mainCount: 1, + repeatableCount: 1)); + Assert.Contains("no Message-IDs", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReader.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReader.cs new file mode 100644 index 00000000..7b675d92 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReader.cs @@ -0,0 +1,330 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Classifies and consumes one BackFiller listener ARTICLE response. + /// + /// + /// Payload accounting excludes the status line and the terminating .CRLF. + /// Logical payload bytes are the multiline article image with leading stuffing dots removed. + /// Wire payload bytes are the article image octets as received before destuffing. + /// + public static class NntpArticleResponseReader + { + /// + /// Reads one status line (through CRLF) from into . + /// + public static async Task ReadStatusLineAsync( + Stream stream, + byte[] lineBuffer, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(lineBuffer); + var length = 0; + while (true) + { + if (length >= lineBuffer.Length) + { + return new NntpStatusLine(NntpResponseKind.ProtocolError, 0, "status line exceeded buffer"); + } + + var read = await stream.ReadAsync(lineBuffer.AsMemory(length, 1), cancellationToken).ConfigureAwait(false); + if (read == 0) + { + return new NntpStatusLine(NntpResponseKind.Disconnect, 0, "peer closed before status line"); + } + + length += read; + if (length >= 2 + && lineBuffer[length - 2] == (byte)'\r' + && lineBuffer[length - 1] == (byte)'\n') + { + break; + } + } + + var contentLength = length - 2; + if (contentLength < 3 + || !IsDigit(lineBuffer[0]) + || !IsDigit(lineBuffer[1]) + || !IsDigit(lineBuffer[2])) + { + return new NntpStatusLine(NntpResponseKind.ProtocolError, 0, "status line missing three-digit code"); + } + + var code = ((lineBuffer[0] - '0') * 100) + + ((lineBuffer[1] - '0') * 10) + + (lineBuffer[2] - '0'); + return new NntpStatusLine(ClassifyStatus(code), code, null); + } + + /// + /// After a 220 status line, consumes the multiline article and terminator. + /// Article octets are discarded after accounting; the corpus is not retained. + /// + public static async Task ConsumeArticleBodyAsync( + Stream stream, + byte[] readBuffer, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(readBuffer); + if (readBuffer.Length < 4) + { + throw new ArgumentException("readBuffer must be at least 4 bytes.", nameof(readBuffer)); + } + + long wireBytes = 0; + long logicalBytes = 0; + // Line state: 0 = start of line, 1 = seen '.', 2 = seen '.\r', else mid-line. + var state = 0; + // When state==1 and next byte is not CR, the '.' was stuffing and must be counted on wire only. + + while (true) + { + var read = await stream.ReadAsync(readBuffer, cancellationToken).ConfigureAwait(false); + if (read == 0) + { + return new NntpArticleConsumeResult( + NntpResponseKind.Incomplete, + wireBytes, + logicalBytes, + "peer closed before multiline terminator"); + } + + for (var i = 0; i < read; i++) + { + var b = readBuffer[i]; + switch (state) + { + case 0: // start of line + if (b == (byte)'.') + { + state = 1; + } + else + { + wireBytes++; + logicalBytes++; + state = b == (byte)'\n' ? 0 : 3; + } + + break; + + case 1: // seen '.' at line start + if (b == (byte)'\r') + { + state = 2; + } + else + { + // Stuffing: wire includes the '.', logical does not. + wireBytes++; + wireBytes++; + logicalBytes++; + state = b == (byte)'\n' ? 0 : 3; + } + + break; + + case 2: // seen '.\r' + if (b == (byte)'\n') + { + return new NntpArticleConsumeResult( + NntpResponseKind.ArticleDelivered, + wireBytes, + logicalBytes, + null); + } + + // Not a terminator; emit '.\r' as body ('.' is stuffing if we treat as line — + // protocol-wise ".\rX" is malformed; count as wire/logical for diagnosis). + wireBytes += 2; + logicalBytes += 1; // destuff leading dot + wireBytes++; + logicalBytes++; + // b is not LF here (LF returned above); stay mid-line after the malformed ".\r" pair. + state = 3; + break; + + default: // mid-line + wireBytes++; + logicalBytes++; + if (b == (byte)'\n') + { + state = 0; + } + + break; + } + } + } + } + + /// + /// Full ARTICLE exchange helper for in-memory test streams: status then optional body. + /// + public static async Task ReadArticleExchangeAsync( + Stream stream, + byte[] lineBuffer, + byte[] readBuffer, + CancellationToken cancellationToken) + { + var status = await ReadStatusLineAsync(stream, lineBuffer, cancellationToken).ConfigureAwait(false); + if (status.Kind != NntpResponseKind.ArticleFollows) + { + return new NntpArticleExchangeResult( + status.Kind, + status.Code, + WireArticleBytes: 0, + LogicalArticleBytes: 0, + status.Error); + } + + var body = await ConsumeArticleBodyAsync(stream, readBuffer, cancellationToken).ConfigureAwait(false); + return new NntpArticleExchangeResult( + body.Kind, + status.Code, + body.WireArticleBytes, + body.LogicalArticleBytes, + body.Error); + } + + /// Maps a three-digit NNTP code to a stress-test outcome kind. + public static NntpResponseKind ClassifyStatus(int code) => + code switch + { + 220 => NntpResponseKind.ArticleFollows, + 430 => NntpResponseKind.NotFound, + 403 => NntpResponseKind.NotAvailable, + 412 => NntpResponseKind.NoNewsgroup, + 500 => NntpResponseKind.UnknownCommand, + 501 => NntpResponseKind.SyntaxError, + _ => NntpResponseKind.UnexpectedStatus, + }; + + /// + /// Classifies an end-to-end Article Work outcome plus NNTP delivery into a stable failure bucket. + /// + public static StressFailureClass ClassifyFailure( + ArticleWorkStressOutcome? workOutcome, + NntpResponseKind? nntpKind, + bool timedOut, + bool disconnected) + { + if (timedOut) + { + return StressFailureClass.Timeout; + } + + if (disconnected) + { + return StressFailureClass.Disconnect; + } + + if (workOutcome is { } work) + { + switch (work) + { + case ArticleWorkStressOutcome.ArticleNotFound: + return StressFailureClass.MissingOrExpired; + case ArticleWorkStressOutcome.InvalidArticle: + return StressFailureClass.UpstreamOrValidation; + case ArticleWorkStressOutcome.InvalidRequest: + return StressFailureClass.ProtocolError; + case ArticleWorkStressOutcome.ProviderFailure: + case ArticleWorkStressOutcome.RetentionRejected: + return StressFailureClass.UpstreamOrValidation; + case ArticleWorkStressOutcome.UnexpectedFailure: + return StressFailureClass.Other; + case ArticleWorkStressOutcome.BrokerError: + return StressFailureClass.RabbitMqError; + case ArticleWorkStressOutcome.NoResponse: + return StressFailureClass.Timeout; + case ArticleWorkStressOutcome.Success: + break; + default: + return StressFailureClass.Other; + } + } + + return nntpKind switch + { + null => StressFailureClass.Other, + NntpResponseKind.ArticleDelivered => StressFailureClass.None, + NntpResponseKind.NotFound => StressFailureClass.MissingOrExpired, + NntpResponseKind.NotAvailable => StressFailureClass.NotAvailable, + NntpResponseKind.Incomplete => StressFailureClass.IncompleteResponse, + NntpResponseKind.ProtocolError => StressFailureClass.ProtocolError, + NntpResponseKind.UnexpectedStatus => StressFailureClass.ProtocolError, + NntpResponseKind.Disconnect => StressFailureClass.Disconnect, + _ => StressFailureClass.Other, + }; + } + + private static bool IsDigit(byte value) => value is >= (byte)'0' and <= (byte)'9'; + } + + /// Listener ARTICLE / multiline outcome kinds used by the stress harness. + public enum NntpResponseKind + { + ArticleFollows, + ArticleDelivered, + NotFound, + NotAvailable, + NoNewsgroup, + UnknownCommand, + SyntaxError, + UnexpectedStatus, + ProtocolError, + Incomplete, + Disconnect, + } + + /// Article Work reply outcomes observable by the stress publisher (wire names). + public enum ArticleWorkStressOutcome + { + Success, + ArticleNotFound, + InvalidArticle, + InvalidRequest, + ProviderFailure, + UnexpectedFailure, + RetentionRejected, + NoResponse, + BrokerError, + } + + /// Stable failure buckets for reports. + public enum StressFailureClass + { + None, + MissingOrExpired, + NotAvailable, + Timeout, + Disconnect, + IncompleteResponse, + ProtocolError, + UpstreamOrValidation, + RabbitMqError, + Other, + } + + /// One NNTP status line. + public readonly record struct NntpStatusLine(NntpResponseKind Kind, int Code, string? Error); + + /// Result of consuming a multiline article body. + public readonly record struct NntpArticleConsumeResult( + NntpResponseKind Kind, + long WireArticleBytes, + long LogicalArticleBytes, + string? Error); + + /// Combined status + body consume result. + public readonly record struct NntpArticleExchangeResult( + NntpResponseKind Kind, + int StatusCode, + long WireArticleBytes, + long LogicalArticleBytes, + string? Error); +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReaderTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReaderTests.cs new file mode 100644 index 00000000..53084e51 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReaderTests.cs @@ -0,0 +1,81 @@ +using System.Text; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + public sealed class NntpArticleResponseReaderTests + { + [Fact] + public async Task ReadArticleExchange_counts_destuffed_payload_without_framing() + { + // 220 line + stuffed body line "..hi\r\n" (logical ".hi\r\n") + terminator + var wire = "220 0 article follows\r\n"u8.ToArray() + .Concat("..hi\r\n"u8.ToArray()) + .Concat(".\r\n"u8.ToArray()) + .ToArray(); + await using var stream = new MemoryStream(wire); + var result = await NntpArticleResponseReader.ReadArticleExchangeAsync( + stream, + new byte[512], + new byte[1024], + CancellationToken.None); + + Assert.Equal(NntpResponseKind.ArticleDelivered, result.Kind); + Assert.Equal(220, result.StatusCode); + Assert.Equal(6, result.WireArticleBytes); // "..hi\r\n" + Assert.Equal(5, result.LogicalArticleBytes); // ".hi\r\n" + } + + [Fact] + public async Task ReadArticleExchange_not_found_has_zero_payload() + { + await using var stream = new MemoryStream("430 No Such Article Found\r\n"u8.ToArray()); + var result = await NntpArticleResponseReader.ReadArticleExchangeAsync( + stream, + new byte[512], + new byte[1024], + CancellationToken.None); + Assert.Equal(NntpResponseKind.NotFound, result.Kind); + Assert.Equal(0, result.LogicalArticleBytes); + Assert.Equal(0, result.WireArticleBytes); + } + + [Fact] + public async Task ConsumeArticleBody_incomplete_when_peer_closes() + { + await using var stream = new MemoryStream("line\r\n"u8.ToArray()); + var result = await NntpArticleResponseReader.ConsumeArticleBodyAsync( + stream, + new byte[1024], + CancellationToken.None); + Assert.Equal(NntpResponseKind.Incomplete, result.Kind); + Assert.True(result.WireArticleBytes > 0); + } + + [Theory] + [InlineData(ArticleWorkStressOutcome.ArticleNotFound, null, false, false, StressFailureClass.MissingOrExpired)] + [InlineData(ArticleWorkStressOutcome.ProviderFailure, null, false, false, StressFailureClass.UpstreamOrValidation)] + [InlineData(ArticleWorkStressOutcome.UnexpectedFailure, null, false, false, StressFailureClass.Other)] + [InlineData(ArticleWorkStressOutcome.RetentionRejected, null, false, false, StressFailureClass.UpstreamOrValidation)] + [InlineData(ArticleWorkStressOutcome.Success, NntpResponseKind.NotFound, false, false, StressFailureClass.MissingOrExpired)] + [InlineData(ArticleWorkStressOutcome.Success, NntpResponseKind.ArticleDelivered, false, false, StressFailureClass.None)] + [InlineData(ArticleWorkStressOutcome.NoResponse, null, true, false, StressFailureClass.Timeout)] + [InlineData(ArticleWorkStressOutcome.BrokerError, null, false, false, StressFailureClass.RabbitMqError)] + public void ClassifyFailure_maps_buckets( + ArticleWorkStressOutcome work, + NntpResponseKind? nntp, + bool timedOut, + bool disconnected, + StressFailureClass expected) + { + var actual = NntpArticleResponseReader.ClassifyFailure(work, nntp, timedOut, disconnected); + Assert.Equal(expected, actual); + } + + [Fact] + public void ComposeProviderEntity_normalizes_backbone() + { + Assert.Equal("backfiller.giganews", ArticleWorkStressClient.ComposeProviderEntity("Giganews")); + Assert.Equal("backfiller.eweka", ArticleWorkStressClient.ComposeProviderEntity("Eweka")); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StageLatencyStats.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StageLatencyStats.cs new file mode 100644 index 00000000..0a2dd6fd --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StageLatencyStats.cs @@ -0,0 +1,55 @@ +using System.Collections.Concurrent; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Aggregate latency statistics for one observable harness stage. + /// + /// + /// Mean is the arithmetic mean of recorded samples. Percentiles are order statistics. + /// + public sealed record StageLatencyStats( + string StageName, + string Description, + long Count, + double MeanMs, + double P50Ms, + double P90Ms, + double P95Ms, + double P99Ms, + double MaxMs) + { + public static StageLatencyStats From(string name, string description, ConcurrentBag bag) + { + if (bag.IsEmpty) + { + return new StageLatencyStats(name, description, 0, 0, 0, 0, 0, 0, 0); + } + + var values = bag.ToArray(); + Array.Sort(values); + double sum = 0; + for (var i = 0; i < values.Length; i++) + { + sum += values[i]; + } + + return new StageLatencyStats( + name, + description, + values.Length, + sum / values.Length, + At(values, 0.50), + At(values, 0.90), + At(values, 0.95), + At(values, 0.99), + values[^1]); + } + + private static double At(long[] sorted, double quantile) + { + var index = (int)Math.Clamp(Math.Ceiling(quantile * sorted.Length) - 1, 0, sorted.Length - 1); + return sorted[index]; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecision.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecision.cs new file mode 100644 index 00000000..7dad855c --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecision.cs @@ -0,0 +1,50 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Pure abort policy for the stress harness. Separated for regression tests. + /// + /// + /// Rates use completed RPC outcomes (work replies + caller timeouts), never + /// RequestsStarted. Flood mode intentionally keeps many requests outstanding; + /// comparing delivered articles to started requests falsely aborts during ramp-up. + /// + public static class StressAbortDecision + { + /// + /// Returns whether the run should cancel based on completed-work rates. + /// + /// Publish attempts started (diagnostic only; not the rate denominator). + /// Article Work Success replies received. + /// Article Work NotFound replies received. + /// Other non-success work replies received. + /// Caller-side reply wait timeouts. + /// Listener ARTICLE successes. + /// Minimum completed RPCs before rates are evaluated. + /// Timeouts / completed threshold in [0, 1]. + /// + /// Failure threshold using 1 - articlesDelivered/completed (includes not-found and work failures). + /// + public static bool ShouldAbort( + long requestsStarted, + long workSuccess, + long workNotFound, + long workOtherFailure, + long timeouts, + long articlesDelivered, + int abortAfterCompleted, + double abortOnTimeoutRate, + double abortOnFailureRate) + { + _ = requestsStarted; + var completed = workSuccess + workNotFound + workOtherFailure + timeouts; + if (completed < abortAfterCompleted || completed <= 0) + { + return false; + } + + var timeoutRate = (double)timeouts / completed; + var failureRate = 1.0 - ((double)articlesDelivered / completed); + return timeoutRate >= abortOnTimeoutRate || failureRate >= abortOnFailureRate; + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecisionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecisionTests.cs new file mode 100644 index 00000000..d5ecc0a2 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecisionTests.cs @@ -0,0 +1,74 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Regression: flood outstanding work must not abort using RequestsStarted as the rate denominator. + /// + public sealed class StressAbortDecisionTests + { + [Fact] + public void ShouldAbort_false_during_flood_ramp_when_few_rpcs_completed() + { + // Prior DualSoak defect: Giganews cancelled after ~259 starts / 9 delivered + // because failureRate used RequestsStarted (≈0.97) despite almost no completed replies. + Assert.False( + StressAbortDecision.ShouldAbort( + requestsStarted: 259, + workSuccess: 9, + workNotFound: 0, + workOtherFailure: 0, + timeouts: 0, + articlesDelivered: 9, + abortAfterCompleted: 200, + abortOnTimeoutRate: 0.25, + abortOnFailureRate: 0.50)); + } + + [Fact] + public void ShouldAbort_true_when_completed_failure_rate_exceeds_threshold() + { + Assert.True( + StressAbortDecision.ShouldAbort( + requestsStarted: 500, + workSuccess: 50, + workNotFound: 100, + workOtherFailure: 50, + timeouts: 0, + articlesDelivered: 40, + abortAfterCompleted: 200, + abortOnTimeoutRate: 0.25, + abortOnFailureRate: 0.50)); + } + + [Fact] + public void ShouldAbort_true_when_completed_timeout_rate_exceeds_threshold() + { + Assert.True( + StressAbortDecision.ShouldAbort( + requestsStarted: 300, + workSuccess: 150, + workNotFound: 0, + workOtherFailure: 0, + timeouts: 100, + articlesDelivered: 150, + abortAfterCompleted: 200, + abortOnTimeoutRate: 0.25, + abortOnFailureRate: 0.50)); + } + + [Fact] + public void ShouldAbort_false_when_below_completed_threshold() + { + Assert.False( + StressAbortDecision.ShouldAbort( + requestsStarted: 1000, + workSuccess: 10, + workNotFound: 10, + workOtherFailure: 10, + timeouts: 50, + articlesDelivered: 0, + abortAfterCompleted: 200, + abortOnTimeoutRate: 0.25, + abortOnFailureRate: 0.50)); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs new file mode 100644 index 00000000..5852eca6 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs @@ -0,0 +1,308 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Deterministic corpus provenance, selection, and correlation contracts for stress Message-IDs. + /// + public sealed class StressCorpusIntegrityTests + { + [Fact] + public void Required_corpus_path_is_the_vectornntp_newslogs_directory() + { + Assert.Equal( + @"C:\Users\chrisk\source\repos\VectorNNTP\.artifacts\newslogs", + StressCorpusPaths.RequiredNewsLogDirectory); + } + + [Fact] + public void Required_corpus_path_resolves_to_usable_logs_on_this_host() + { + // After the verified junction (or a real VectorNNTP tree), discovery must see *.log files. + NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(StressCorpusPaths.RequiredNewsLogDirectory); + Assert.NotEmpty(NewsLogMessageIdDiscovery.DiscoverLogFiles(StressCorpusPaths.RequiredNewsLogDirectory)); + Assert.True( + File.Exists( + Path.Join(StressCorpusPaths.RequiredNewsLogDirectory, "news-20261019.log"))); + } + + [Fact] + public void Preflight_blocks_publish_on_invalid_or_non_corpus_ids() + { + var results = new[] + { + new CorpusIdPreflightResult("", CorpusIdPreflightKind.NotInCorpus, "missing"), + }; + var ex = Assert.Throws( + () => CorpusArticlePreflight.ThrowIfPreflightBlocksPublish(results)); + Assert.Contains("blocked publish", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Preflight_blocks_publish_when_zero_ids_retrieved() + { + var results = new[] + { + new CorpusIdPreflightResult("", CorpusIdPreflightKind.NotFound, "missing article"), + new CorpusIdPreflightResult("", CorpusIdPreflightKind.Timeout, "work reply timeout"), + }; + var ex = Assert.Throws( + () => CorpusArticlePreflight.ThrowIfPreflightBlocksPublish(results)); + Assert.Contains("none of the", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Preflight_allows_publish_when_at_least_one_retrieved() + { + var results = new[] + { + new CorpusIdPreflightResult("", CorpusIdPreflightKind.Retrieved, null), + new CorpusIdPreflightResult("", CorpusIdPreflightKind.NotFound, "missing article"), + }; + CorpusArticlePreflight.ThrowIfPreflightBlocksPublish(results); + } + + [Fact] + public async Task Workload_guard_rejects_hash_mismatch() + { + var dir = Path.Join(Path.GetTempPath(), "vnntp-hash-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + var main = Path.Join(dir, "main-message-ids.txt"); + await File.WriteAllTextAsync(main, "\n"); + await WorkloadCorpusManifest.WriteAsync( + dir, + new WorkloadCorpusManifest + { + CorpusDirectory = StressCorpusPaths.RequiredNewsLogDirectory, + FileCount = 1, + LogRecordsExamined = 1, + ValidParsedIds = 1, + RejectedInvalidMessageIds = 0, + UniqueValidIds = 1, + WarmupSelected = 0, + MainSelected = 1, + RepeatableSelected = 0, + Seed = 1, + MainIdsSha256 = "deadbeef", + CreatedUtc = DateTimeOffset.UtcNow, + }, + CancellationToken.None); + + var options = new StressHarnessOptions + { + RabbitMqHost = "127.0.0.1", + RabbitMqUsername = "x", + RabbitMqPassword = "y", + NewsLogDirectory = StressCorpusPaths.RequiredNewsLogDirectory, + WorkloadDirectory = dir, + ReportDirectory = dir, + }; + var ex = await Assert.ThrowsAsync( + () => StressWorkloadGuard.RequireProvenancedWorkloadAsync(options, CancellationToken.None)); + Assert.Contains("SHA-256", ex.Message, StringComparison.OrdinalIgnoreCase); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } + + [Fact] + public void FromEnvironment_defaults_to_required_corpus_path() + { + var previous = Environment.GetEnvironmentVariable( + StressHarnessOptions.NewsLogDirectoryEnvironmentVariable); + try + { + Environment.SetEnvironmentVariable( + StressHarnessOptions.NewsLogDirectoryEnvironmentVariable, + null); + // Credentials required by FromEnvironment — skip if absent in this shell. + if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__USERNAME")) + || string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__PASSWORD"))) + { + Assert.Equal( + StressCorpusPaths.RequiredNewsLogDirectory, + StressCorpusPaths.RequiredNewsLogDirectory); + return; + } + + var options = StressHarnessOptions.FromEnvironment(); + Assert.Equal(StressCorpusPaths.RequiredNewsLogDirectory, options.NewsLogDirectory); + } + finally + { + Environment.SetEnvironmentVariable( + StressHarnessOptions.NewsLogDirectoryEnvironmentVariable, + previous); + } + } + + [Fact] + public void FromEnvironment_honours_explicit_newslog_override_for_fixtures() + { + var previous = Environment.GetEnvironmentVariable( + StressHarnessOptions.NewsLogDirectoryEnvironmentVariable); + var fixture = Path.Join(Path.GetTempPath(), "vnntp-fixture-" + Guid.NewGuid().ToString("N")); + try + { + if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__USERNAME")) + || string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__PASSWORD"))) + { + return; + } + + Environment.SetEnvironmentVariable( + StressHarnessOptions.NewsLogDirectoryEnvironmentVariable, + fixture); + var options = StressHarnessOptions.FromEnvironment(); + Assert.Equal(fixture, options.NewsLogDirectory); + Assert.NotEqual(StressCorpusPaths.RequiredNewsLogDirectory, options.NewsLogDirectory); + } + finally + { + Environment.SetEnvironmentVariable( + StressHarnessOptions.NewsLogDirectoryEnvironmentVariable, + previous); + } + } + + [Fact] + public async Task Workload_guard_rejects_missing_manifest_stale_cache() + { + var dir = Path.Join(Path.GetTempPath(), "vnntp-stale-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + await File.WriteAllTextAsync(Path.Join(dir, "main-message-ids.txt"), "\n"); + var options = new StressHarnessOptions + { + RabbitMqHost = "127.0.0.1", + RabbitMqUsername = "x", + RabbitMqPassword = "y", + NewsLogDirectory = StressCorpusPaths.RequiredNewsLogDirectory, + WorkloadDirectory = dir, + ReportDirectory = dir, + }; + var ex = await Assert.ThrowsAsync( + () => StressWorkloadGuard.RequireProvenancedWorkloadAsync(options, CancellationToken.None)); + Assert.Contains("provenance", ex.Message, StringComparison.OrdinalIgnoreCase); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } + + [Fact] + public async Task Workload_guard_rejects_cross_corpus_manifest() + { + var dir = Path.Join(Path.GetTempPath(), "vnntp-cross-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + var main = Path.Join(dir, "main-message-ids.txt"); + await File.WriteAllTextAsync(main, "\n"); + await WorkloadCorpusManifest.WriteAsync( + dir, + new WorkloadCorpusManifest + { + CorpusDirectory = @"C:\other\newslogs", + FileCount = 1, + LogRecordsExamined = 1, + ValidParsedIds = 1, + RejectedInvalidMessageIds = 0, + UniqueValidIds = 1, + WarmupSelected = 0, + MainSelected = 1, + RepeatableSelected = 0, + Seed = 1, + MainIdsSha256 = WorkloadCorpusManifest.ComputeFileSha256(main), + CreatedUtc = DateTimeOffset.UtcNow, + }, + CancellationToken.None); + + var options = new StressHarnessOptions + { + RabbitMqHost = "127.0.0.1", + RabbitMqUsername = "x", + RabbitMqPassword = "y", + NewsLogDirectory = StressCorpusPaths.RequiredNewsLogDirectory, + WorkloadDirectory = dir, + ReportDirectory = dir, + }; + var ex = await Assert.ThrowsAsync( + () => StressWorkloadGuard.RequireProvenancedWorkloadAsync(options, CancellationToken.None)); + Assert.Contains("does not match", ex.Message, StringComparison.OrdinalIgnoreCase); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } + + [Fact] + public void Preflight_flags_ids_not_in_corpus_without_replacement() + { + var selected = new[] { "", "" }; + var corpus = new HashSet(StringComparer.Ordinal) { "" }; + var results = CorpusArticlePreflight.ValidateSelectedAgainstCorpus(selected, corpus); + Assert.Equal(CorpusIdPreflightKind.InCorpus, results[0].Kind); + Assert.Equal(CorpusIdPreflightKind.NotInCorpus, results[1].Kind); + Assert.Equal("", results[1].MessageId); + } + + [Fact] + public void Selected_ids_must_be_subset_of_corpus() + { + var allowed = new HashSet(StringComparer.Ordinal) { "" }; + Assert.Throws( + () => StressWorkloadGuard.RequireSelectedIdsSubset(["", ""], allowed)); + } + + [Fact] + public void CreateRequestProperties_preserves_exact_message_path_headers_without_request_ttl() + { + var requestId = Guid.Parse("33333333-3333-3333-3333-333333333333"); + var runId = "run-abc"; + var props = ArticleWorkStressClient.CreateRequestProperties( + "corr-1", + "reply-q", + requestId, + runId); + Assert.True(string.IsNullOrEmpty(props.Expiration)); + Assert.Equal("corr-1", props.CorrelationId); + Assert.Equal("reply-q", props.ReplyTo); + Assert.Equal(requestId.ToString("D"), props.Headers!["RequestId"]); + Assert.Equal(runId, props.Headers["StressRunId"]); + } + + [Fact] + public void Duplicate_ids_are_collapsed_before_workload_write() + { + var dir = Path.Join(Path.GetTempPath(), "vnntp-dup-" + Guid.NewGuid().ToString("N")); + try + { + var counts = NewsLogMessageIdDiscovery.WriteWorkloadSets( + ["", "", ""], + dir, + seed: 3, + warmupCount: 1, + mainCount: 2, + repeatableCount: 1); + Assert.Equal(1, counts.Warmup); + Assert.Equal(1, counts.Main); // only two unique IDs after Distinct + var main = File.ReadAllLines(Path.Join(dir, "main-message-ids.txt")); + var only = Assert.Single(main); + Assert.True(NewsLogMessageIdDiscovery.IsCanonicalMessageId(only)); + } + finally + { + if (Directory.Exists(dir)) + { + Directory.Delete(dir, recursive: true); + } + } + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusPaths.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusPaths.cs new file mode 100644 index 00000000..98691ebd --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusPaths.cs @@ -0,0 +1,25 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Fixed corpus locations for BackFiller stress Message-ID selection. + /// + /// + /// The required news-log corpus is the sibling VectorNNTP artefacts tree. Stress must not + /// silently substitute VectorNNTPD/.artifacts/newslogs, generate IDs, or reuse an + /// unprovenanced workload file. + /// + public static class StressCorpusPaths + { + /// + /// Required absolute news-log corpus. Source of truth for live stress Message-IDs. + /// + public const string RequiredNewsLogDirectory = + @"C:\Users\chrisk\source\repos\VectorNNTP\.artifacts\newslogs"; + + /// Manifest file name written beside workload partitions. + public const string WorkloadManifestFileName = "corpus-manifest.json"; + + /// Selected-ID audit file (one Message-ID per line, same order as main partition). + public const string MainIdsAuditFileName = "main-message-ids.txt"; + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs new file mode 100644 index 00000000..a9734fd1 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs @@ -0,0 +1,207 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Opt-in configuration for the BackFiller end-to-end stress harness. + /// Secrets are read from environment variables and are never written to reports. + /// + public sealed class StressHarnessOptions + { + /// Master enable flag. Must be 1 for live stress tests to run. + public const string EnableEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS"; + + public const string PhaseEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_PHASE"; + public const string ConcurrencyEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_CONCURRENCY"; + public const string DurationSecondsEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_DURATION_SECONDS"; + public const string BackboneEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_BACKBONE"; + public const string MaxInFlightEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_MAX_INFLIGHT"; + public const string NewsLogDirectoryEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_NEWSLOGS"; + public const string WorkloadDirectoryEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_WORKLOAD"; + public const string ReportDirectoryEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_REPORT"; + public const string ListenerHostEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_LISTENER_HOST"; + public const string ListenerPortEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_LISTENER_PORT"; + public const string SeedEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_SEED"; + public const string ArticlePreflightCountEnvironmentVariable = + "VECTORNNTPD_BACKFILLER_E2E_STRESS_ARTICLE_PREFLIGHT_COUNT"; + + public required string RabbitMqHost { get; init; } + public int RabbitMqPort { get; init; } = 5672; + public string RabbitMqVirtualHost { get; init; } = "/"; + public required string RabbitMqUsername { get; init; } + public required string RabbitMqPassword { get; init; } + public TimeSpan WorkReplyTimeout { get; init; } = TimeSpan.FromSeconds(60); + /// + /// Bounded wait for broker-outstanding RPCs (including timed-out callers) before disposing + /// the exclusive ReplyTo connection. + /// + public TimeSpan ReplyQueueDisposeDrainTimeout { get; init; } = TimeSpan.FromSeconds(120); + public TimeSpan ArticleTimeout { get; init; } = TimeSpan.FromSeconds(30); + public string NewsLogDirectory { get; init; } = string.Empty; + public string WorkloadDirectory { get; init; } = string.Empty; + public string ReportDirectory { get; init; } = string.Empty; + public string? ListenerHostOverride { get; init; } + public int? ListenerPortOverride { get; init; } + public int Seed { get; init; } = 42; + /// + /// Number of selected Message-IDs to probe via Article Work before throughput publishing. + /// Zero disables live retrieval preflight (membership/canonical checks still apply). + /// + public int ArticlePreflightCount { get; init; } = 5; + public double AbortOnTimeoutRate { get; init; } = 0.25; + public double AbortOnFailureRate { get; init; } = 0.50; + public int AbortAfterRequests { get; init; } = 200; + + /// Returns when the opt-in environment flag is set to 1. + public static bool IsEnabled() => + string.Equals( + Environment.GetEnvironmentVariable(EnableEnvironmentVariable), + "1", + StringComparison.Ordinal); + + /// + /// Builds options from environment variables. Throws when RabbitMQ credentials are missing. + /// Does not log secret values. + /// + public static StressHarnessOptions FromEnvironment() + { + var username = Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__USERNAME"); + var password = Environment.GetEnvironmentVariable("VECTOR__RABBITMQ__PASSWORD"); + if (string.IsNullOrWhiteSpace(username) || string.IsNullOrWhiteSpace(password)) + { + throw new InvalidOperationException( + "VECTOR__RABBITMQ__USERNAME and VECTOR__RABBITMQ__PASSWORD are required for live stress."); + } + + var repoRoot = FindRepositoryRoot(); + // Live stress defaults to the required absolute VectorNNTP news-log corpus. + // Override only for isolated fixtures via VECTORNNTPD_BACKFILLER_E2E_STRESS_NEWSLOGS. + // There is no silent fallback to VectorNNTPD/.artifacts/newslogs. + var newsLogs = Environment.GetEnvironmentVariable(NewsLogDirectoryEnvironmentVariable); + if (string.IsNullOrWhiteSpace(newsLogs)) + { + newsLogs = StressCorpusPaths.RequiredNewsLogDirectory; + } + + var workload = Environment.GetEnvironmentVariable(WorkloadDirectoryEnvironmentVariable) + ?? Path.Join(repoRoot, ".artifacts", "backfiller-stress", "workload"); + var report = Environment.GetEnvironmentVariable(ReportDirectoryEnvironmentVariable) + ?? Path.Join(repoRoot, ".artifacts", "backfiller-stress", "reports"); + + int? listenerPort = null; + if (int.TryParse(Environment.GetEnvironmentVariable(ListenerPortEnvironmentVariable), out var port) + && port is > 0 and <= 65535) + { + listenerPort = port; + } + + var seed = 42; + if (int.TryParse(Environment.GetEnvironmentVariable(SeedEnvironmentVariable), out var configuredSeed)) + { + seed = configuredSeed; + } + + var articlePreflightCount = 5; + if (int.TryParse( + Environment.GetEnvironmentVariable(ArticlePreflightCountEnvironmentVariable), + out var configuredPreflight) + && configuredPreflight >= 0) + { + articlePreflightCount = configuredPreflight; + } + + return new StressHarnessOptions + { + RabbitMqHost = Environment.GetEnvironmentVariable("VECTORNNTPD_BACKFILLER_E2E_STRESS_RABBIT_HOST") + ?? "198.18.0.3", + RabbitMqPort = 5672, + RabbitMqVirtualHost = "/", + RabbitMqUsername = username, + RabbitMqPassword = password, + NewsLogDirectory = newsLogs, + WorkloadDirectory = workload, + ReportDirectory = report, + ListenerHostOverride = Environment.GetEnvironmentVariable(ListenerHostEnvironmentVariable), + ListenerPortOverride = listenerPort, + Seed = seed, + ArticlePreflightCount = articlePreflightCount, + }; + } + + public static string GetPhaseOrDefault(string defaultPhase) => + Environment.GetEnvironmentVariable(PhaseEnvironmentVariable) ?? defaultPhase; + + public static int GetConcurrencyOrDefault(int defaultConcurrency) + { + if (int.TryParse(Environment.GetEnvironmentVariable(ConcurrencyEnvironmentVariable), out var value) + && value > 0) + { + return value; + } + + return defaultConcurrency; + } + + public static int GetDurationSecondsOrDefault(int defaultSeconds) + { + if (int.TryParse(Environment.GetEnvironmentVariable(DurationSecondsEnvironmentVariable), out var value) + && value > 0) + { + return value; + } + + return defaultSeconds; + } + + public static int GetMaxInFlightOrDefault(int defaultMax) + { + if (int.TryParse(Environment.GetEnvironmentVariable(MaxInFlightEnvironmentVariable), out var value) + && value > 0) + { + return value; + } + + return defaultMax; + } + + public static string GetBackboneOrDefault(string defaultBackbone) => + Environment.GetEnvironmentVariable(BackboneEnvironmentVariable) ?? defaultBackbone; + + /// Redacted snapshot safe for reports (no passwords). + public object ToRedactedReportObject() => + new + { + RabbitMqHost, + RabbitMqPort, + RabbitMqVirtualHost, + RabbitMqUsername = "", + RabbitMqPassword = "", + WorkReplyTimeoutSeconds = WorkReplyTimeout.TotalSeconds, + ArticleTimeoutSeconds = ArticleTimeout.TotalSeconds, + NewsLogDirectory, + WorkloadDirectory, + ReportDirectory, + ListenerHostOverride, + ListenerPortOverride, + Seed, + ArticlePreflightCount, + AbortOnTimeoutRate, + AbortOnFailureRate, + AbortAfterRequests, + }; + + private static string FindRepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null) + { + if (File.Exists(Path.Join(dir.FullName, "VectorNNTPD.slnx"))) + { + return dir.FullName; + } + + dir = dir.Parent; + } + + return Directory.GetCurrentDirectory(); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressMetrics.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressMetrics.cs new file mode 100644 index 00000000..5669a568 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressMetrics.cs @@ -0,0 +1,503 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// Thread-safe counters and latency samples for one stress run. + public sealed class StressMetrics + { + private readonly ConcurrentBag _endToEndSuccessLatenciesMs = []; + private readonly ConcurrentBag _endToEndFailureLatenciesMs = []; + private readonly ConcurrentBag _scheduleToPublishMs = []; + private readonly ConcurrentBag _publishCallMs = []; + private readonly ConcurrentBag _publishToReplyMs = []; + private readonly ConcurrentBag _replyToNntpStartMs = []; + private readonly ConcurrentBag _nntpDeliveryLatenciesMs = []; + private readonly ConcurrentBag _articleSizeBytes = []; + private readonly ConcurrentDictionary _failureBuckets = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _byBackbone = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentQueue _throughputSamples = new(); + private readonly ConcurrentQueue _queueDepthSamples = new(); + private long _requestsStarted; + private long _workSuccess; + private long _workNotFound; + private long _workOtherFailure; + private long _articlesDelivered; + private long _logicalPayloadBytes; + private long _wirePayloadBytes; + private long _nntpNotFound; + private long _nntpErrors; + private long _timeouts; + private long _disconnects; + private long _outstanding; + private long _peakOutstanding; + private long _deliveryOutstanding; + private long _peakDeliveryOutstanding; + private long _outstandingSum; + private long _outstandingSamples; + private long _activeConnections; + private long _peakConnections; + private long _incompleteResponses; + private long _drainMilliseconds; + private long _lateRepliesAfterTimeout; + private long _unresolvedAtDispose; + private long _brokerOutstandingAtDrainEnd; + + public required string RunName { get; init; } + public string? Backbone { get; init; } + public int ConfiguredConcurrency { get; init; } + public int ConfiguredMaxInFlight { get; init; } + public long SteadyStateStartTimestamp { get; set; } + public long SteadyStateEndTimestamp { get; set; } + + public void RequestStarted() + { + Interlocked.Increment(ref _requestsStarted); + var outstanding = Interlocked.Increment(ref _outstanding); + UpdatePeak(ref _peakOutstanding, outstanding); + } + + public void RequestFinished() => Interlocked.Decrement(ref _outstanding); + + public long Outstanding => Interlocked.Read(ref _outstanding); + + public void DeliveryStarted() + { + var value = Interlocked.Increment(ref _deliveryOutstanding); + UpdatePeak(ref _peakDeliveryOutstanding, value); + } + + public void DeliveryFinished() => Interlocked.Decrement(ref _deliveryOutstanding); + + public long DeliveryOutstanding => Interlocked.Read(ref _deliveryOutstanding); + + public void ConnectionOpened() + { + var active = Interlocked.Increment(ref _activeConnections); + UpdatePeak(ref _peakConnections, active); + } + + public void ConnectionClosed() => Interlocked.Decrement(ref _activeConnections); + + public void SetDrainMilliseconds(long milliseconds) => + Interlocked.Exchange(ref _drainMilliseconds, milliseconds); + + /// + /// Records ReplyTo lifecycle counters after harness drain / client dispose. + /// Distinguishes late replies from unresolved broker-outstanding work without changing deadlines. + /// + public void SetReplyLifecycleCounters( + long lateRepliesAfterTimeout, + long unresolvedAtDispose, + long brokerOutstandingAtDrainEnd) + { + Interlocked.Exchange(ref _lateRepliesAfterTimeout, lateRepliesAfterTimeout); + Interlocked.Exchange(ref _unresolvedAtDispose, unresolvedAtDispose); + Interlocked.Exchange(ref _brokerOutstandingAtDrainEnd, brokerOutstandingAtDrainEnd); + } + + public void RecordWorkReply(string backbone, ArticleWorkStressOutcome outcome) + { + var counters = _byBackbone.GetOrAdd(backbone, static _ => new BackboneCounters()); + switch (outcome) + { + case ArticleWorkStressOutcome.Success: + Interlocked.Increment(ref _workSuccess); + Interlocked.Increment(ref counters.WorkSuccess); + break; + case ArticleWorkStressOutcome.ArticleNotFound: + Interlocked.Increment(ref _workNotFound); + Interlocked.Increment(ref counters.WorkNotFound); + break; + default: + Interlocked.Increment(ref _workOtherFailure); + Interlocked.Increment(ref counters.WorkOtherFailure); + break; + } + } + + public void RecordStageTimings( + long scheduleToPublishMs, + long publishCallMs, + long publishToReplyMs, + long? replyToNntpStartMs, + long? nntpDeliveryMs, + long endToEndMs, + bool success, + long? logicalBytes) + { + _scheduleToPublishMs.Add(scheduleToPublishMs); + _publishCallMs.Add(publishCallMs); + _publishToReplyMs.Add(publishToReplyMs); + if (replyToNntpStartMs is { } r) + { + _replyToNntpStartMs.Add(r); + } + + if (nntpDeliveryMs is { } n) + { + _nntpDeliveryLatenciesMs.Add(n); + } + + if (success) + { + _endToEndSuccessLatenciesMs.Add(endToEndMs); + if (logicalBytes is { } bytes) + { + _articleSizeBytes.Add(bytes); + } + } + else + { + _endToEndFailureLatenciesMs.Add(endToEndMs); + } + } + + public void RecordArticleDelivered( + string backbone, + long logicalBytes, + long wireBytes, + long endToEndMs, + long nntpDeliveryMs) + { + Interlocked.Increment(ref _articlesDelivered); + Interlocked.Add(ref _logicalPayloadBytes, logicalBytes); + Interlocked.Add(ref _wirePayloadBytes, wireBytes); + var counters = _byBackbone.GetOrAdd(backbone, static _ => new BackboneCounters()); + Interlocked.Increment(ref counters.ArticlesDelivered); + Interlocked.Add(ref counters.LogicalPayloadBytes, logicalBytes); + Interlocked.Add(ref counters.WirePayloadBytes, wireBytes); + _ = endToEndMs; + _ = nntpDeliveryMs; + } + + public void RecordFailure( + string backbone, + StressFailureClass failure, + long? endToEndMs, + NntpResponseKind? nntpKind = null) + { + _failureBuckets.AddOrUpdate(failure.ToString(), 1, static (_, v) => v + 1); + if (endToEndMs is { } ms) + { + _endToEndFailureLatenciesMs.Add(ms); + } + + switch (failure) + { + case StressFailureClass.Timeout: + Interlocked.Increment(ref _timeouts); + break; + case StressFailureClass.Disconnect: + Interlocked.Increment(ref _disconnects); + break; + case StressFailureClass.IncompleteResponse: + Interlocked.Increment(ref _incompleteResponses); + break; + case StressFailureClass.MissingOrExpired when nntpKind == NntpResponseKind.NotFound: + Interlocked.Increment(ref _nntpNotFound); + break; + default: + if (nntpKind is not null and not NntpResponseKind.ArticleDelivered) + { + Interlocked.Increment(ref _nntpErrors); + } + + break; + } + + var counters = _byBackbone.GetOrAdd(backbone, static _ => new BackboneCounters()); + Interlocked.Increment(ref counters.Failures); + } + + public void SampleThroughput(long timestamp) + { + var outstanding = Interlocked.Read(ref _outstanding); + Interlocked.Add(ref _outstandingSum, outstanding); + Interlocked.Increment(ref _outstandingSamples); + _throughputSamples.Enqueue(new ThroughputSample( + timestamp, + Interlocked.Read(ref _articlesDelivered), + Interlocked.Read(ref _logicalPayloadBytes), + outstanding, + Interlocked.Read(ref _deliveryOutstanding))); + } + + public void SampleQueueDepth(long timestamp, string queueName, long messages, long ready, long unacked, long consumers) + { + _queueDepthSamples.Enqueue(new QueueDepthSample(timestamp, queueName, messages, ready, unacked, consumers)); + } + + public StressRunSummary BuildSummary(Process? process = null) + { + var start = SteadyStateStartTimestamp; + var end = SteadyStateEndTimestamp; + if (end <= start) + { + end = Stopwatch.GetTimestamp(); + } + + var elapsedSeconds = Stopwatch.GetElapsedTime(start, end).TotalSeconds; + if (elapsedSeconds <= 0) + { + elapsedSeconds = 1e-9; + } + + var logicalBytes = Interlocked.Read(ref _logicalPayloadBytes); + var articles = Interlocked.Read(ref _articlesDelivered); + var gbps = logicalBytes * 8.0 / elapsedSeconds / 1_000_000_000.0; + var mibPerSec = logicalBytes / elapsedSeconds / (1024.0 * 1024.0); + var samples = Interlocked.Read(ref _outstandingSamples); + var meanOutstanding = samples == 0 + ? 0.0 + : (double)Interlocked.Read(ref _outstandingSum) / samples; + var meanArticle = articles == 0 ? 0.0 : (double)logicalBytes / articles; + + var byBackbone = _byBackbone.ToDictionary( + static pair => pair.Key, + pair => pair.Value.Snapshot(), + StringComparer.OrdinalIgnoreCase); + + var stages = new[] + { + StageLatencyStats.From( + "schedule_to_publish", + "Harness: work enqueued to BasicPublish completed. Not upstream fetch.", + _scheduleToPublishMs), + StageLatencyStats.From( + "publish_call", + "Harness: duration of BasicPublishAsync only.", + _publishCallMs), + StageLatencyStats.From( + "publish_to_reply", + "Aggregate: publish-complete to ReplyTo receipt (BackFiller consume+upstream+retain+reply). Upstream fetch is not separately observable.", + _publishToReplyMs), + StageLatencyStats.From( + "reply_to_nntp_start", + "Harness: Success reply received to ARTICLE send start (delivery queue + connection rent).", + _replyToNntpStartMs), + StageLatencyStats.From( + "nntp_delivery", + "Harness: ARTICLE command through complete multiline consume on listener.", + _nntpDeliveryLatenciesMs), + StageLatencyStats.From( + "schedule_to_complete_success", + "Harness: schedule timestamp to successful ARTICLE complete.", + _endToEndSuccessLatenciesMs), + }; + + return new StressRunSummary( + RunName, + Backbone, + ConfiguredConcurrency, + ConfiguredMaxInFlight, + elapsedSeconds, + Interlocked.Read(ref _requestsStarted), + Interlocked.Read(ref _workSuccess), + Interlocked.Read(ref _workNotFound), + Interlocked.Read(ref _workOtherFailure), + articles, + logicalBytes, + Interlocked.Read(ref _wirePayloadBytes), + meanArticle, + articles / elapsedSeconds, + gbps, + mibPerSec, + Interlocked.Read(ref _nntpNotFound), + Interlocked.Read(ref _nntpErrors), + Interlocked.Read(ref _timeouts), + Interlocked.Read(ref _disconnects), + Interlocked.Read(ref _incompleteResponses), + Interlocked.Read(ref _outstanding), + Interlocked.Read(ref _peakOutstanding), + meanOutstanding, + Interlocked.Read(ref _peakDeliveryOutstanding), + Interlocked.Read(ref _peakConnections), + Interlocked.Read(ref _drainMilliseconds), + Interlocked.Read(ref _lateRepliesAfterTimeout), + Interlocked.Read(ref _unresolvedAtDispose), + Interlocked.Read(ref _brokerOutstandingAtDrainEnd), + Percentiles.From(_endToEndSuccessLatenciesMs), + Percentiles.From(_endToEndFailureLatenciesMs), + Percentiles.From(_publishToReplyMs), + Percentiles.From(_nntpDeliveryLatenciesMs), + stages, + StageLatencyStats.From("article_size_bytes", "Logical payload bytes per delivered article.", _articleSizeBytes), + _failureBuckets.ToDictionary(static p => p.Key, static p => p.Value, StringComparer.Ordinal), + byBackbone, + _throughputSamples.ToArray(), + _queueDepthSamples.ToArray(), + process?.WorkingSet64, + process?.TotalProcessorTime.TotalSeconds); + } + + public void WriteJson(string path, Process? process = null) + { + var summary = BuildSummary(process); + var json = JsonSerializer.Serialize(summary, StressJsonContext.Default.StressRunSummary); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, json); + } + + private static void UpdatePeak(ref long peak, long candidate) + { + while (true) + { + var observed = Interlocked.Read(ref peak); + if (candidate <= observed) + { + return; + } + + if (Interlocked.CompareExchange(ref peak, candidate, observed) == observed) + { + return; + } + } + } + + private sealed class BackboneCounters + { + public long WorkSuccess; + public long WorkNotFound; + public long WorkOtherFailure; + public long ArticlesDelivered; + public long LogicalPayloadBytes; + public long WirePayloadBytes; + public long Failures; + + public BackboneSummary Snapshot() => + new( + Interlocked.Read(ref WorkSuccess), + Interlocked.Read(ref WorkNotFound), + Interlocked.Read(ref WorkOtherFailure), + Interlocked.Read(ref ArticlesDelivered), + Interlocked.Read(ref LogicalPayloadBytes), + Interlocked.Read(ref WirePayloadBytes), + Interlocked.Read(ref Failures)); + } + } + + public readonly record struct ThroughputSample( + long Timestamp, + long ArticlesDelivered, + long LogicalPayloadBytes, + long Outstanding, + long DeliveryOutstanding); + + public readonly record struct QueueDepthSample( + long Timestamp, + string QueueName, + long Messages, + long Ready, + long Unacked, + long Consumers); + + public sealed record Percentiles(long Count, double Mean, double P50, double P90, double P95, double P99, double Max) + { + public static Percentiles From(ConcurrentBag bag) + { + if (bag.IsEmpty) + { + return new Percentiles(0, 0, 0, 0, 0, 0, 0); + } + + var values = bag.ToArray(); + Array.Sort(values); + double sum = 0; + for (var i = 0; i < values.Length; i++) + { + sum += values[i]; + } + + return new Percentiles( + values.Length, + sum / values.Length, + At(values, 0.50), + At(values, 0.90), + At(values, 0.95), + At(values, 0.99), + values[^1]); + } + + private static double At(long[] sorted, double quantile) + { + var index = (int)Math.Clamp(Math.Ceiling(quantile * sorted.Length) - 1, 0, sorted.Length - 1); + return sorted[index]; + } + } + + public sealed record BackboneSummary( + long WorkSuccess, + long WorkNotFound, + long WorkOtherFailure, + long ArticlesDelivered, + long LogicalPayloadBytes, + long WirePayloadBytes, + long Failures); + + public sealed record StressRunSummary( + string RunName, + string? Backbone, + int ConfiguredConcurrency, + int ConfiguredMaxInFlight, + double ElapsedSeconds, + long RequestsStarted, + long WorkSuccess, + long WorkNotFound, + long WorkOtherFailure, + long ArticlesDelivered, + long LogicalPayloadBytes, + long WirePayloadBytes, + double MeanArticleLogicalBytes, + double ArticlesPerSecond, + double PayloadGbps, + double PayloadMiBPerSecond, + long NntpNotFound, + long NntpErrors, + long Timeouts, + long Disconnects, + long IncompleteResponses, + long OutstandingAtEnd, + long PeakOutstanding, + double MeanOutstanding, + long PeakDeliveryOutstanding, + long PeakConnections, + long DrainMilliseconds, + long LateRepliesAfterTimeout, + long UnresolvedAtDispose, + long BrokerOutstandingAtDrainEnd, + Percentiles EndToEndSuccessLatencyMs, + Percentiles EndToEndFailureLatencyMs, + Percentiles WorkReplyLatencyMs, + Percentiles NntpDeliveryLatencyMs, + IReadOnlyList Stages, + StageLatencyStats ArticleSizeBytes, + IReadOnlyDictionary FailureBuckets, + IReadOnlyDictionary ByBackbone, + IReadOnlyList ThroughputSamples, + IReadOnlyList QueueDepthSamples, + long? WorkingSetBytes, + double? ProcessCpuSeconds) + { + public string FormatTableRow() + { + return string.Create( + CultureInfo.InvariantCulture, + $"{RunName,-40} Lcfg={ConfiguredConcurrency,3} Lpeak={PeakOutstanding,3} Lmean={MeanOutstanding,5:F1} art/s={ArticlesPerSecond,7:F1} Gbps={PayloadGbps,7:F3} replyMean={WorkReplyLatencyMs.Mean,7:F0} e2eMean={EndToEndSuccessLatencyMs.Mean,7:F0} ok={ArticlesDelivered} to={Timeouts} late={LateRepliesAfterTimeout} unresolved={UnresolvedAtDispose} drainMs={DrainMilliseconds}"); + } + } + + [System.Text.Json.Serialization.JsonSerializable(typeof(StressRunSummary))] + [System.Text.Json.Serialization.JsonSerializable(typeof(StageLatencyStats))] + [System.Text.Json.Serialization.JsonSerializable(typeof(Percentiles))] + [System.Text.Json.Serialization.JsonSerializable(typeof(BackboneSummary))] + [System.Text.Json.Serialization.JsonSerializable(typeof(ThroughputSample))] + [System.Text.Json.Serialization.JsonSerializable(typeof(QueueDepthSample))] + [System.Text.Json.Serialization.JsonSerializable(typeof(NewsLogDiscoveryStats))] + internal partial class StressJsonContext : System.Text.Json.Serialization.JsonSerializerContext + { + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressPublishGateTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressPublishGateTests.cs new file mode 100644 index 00000000..e485c480 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressPublishGateTests.cs @@ -0,0 +1,80 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Proves static preflight gates reject work before any RabbitMQ publish path is reached. + /// + public sealed class StressPublishGateTests + { + [Fact] + public async Task Missing_corpus_fails_before_workload_acceptance() + { + var missing = Path.Join(Path.GetTempPath(), "vnntp-nogate-" + Guid.NewGuid().ToString("N")); + var ex = Assert.Throws( + () => NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(missing)); + Assert.Contains("missing", ex.Message, StringComparison.OrdinalIgnoreCase); + await Task.CompletedTask; + } + + [Fact] + public async Task Unprovenanced_workload_is_rejected_without_needing_a_broker() + { + var dir = Path.Join(Path.GetTempPath(), "vnntp-noprov-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + await File.WriteAllTextAsync(Path.Join(dir, "main-message-ids.txt"), "\n"); + var options = new StressHarnessOptions + { + RabbitMqHost = "127.0.0.1", + RabbitMqUsername = "x", + RabbitMqPassword = "y", + NewsLogDirectory = StressCorpusPaths.RequiredNewsLogDirectory, + WorkloadDirectory = dir, + ReportDirectory = dir, + ArticlePreflightCount = 5, + }; + await Assert.ThrowsAsync( + () => StressWorkloadGuard.RequireProvenancedWorkloadAsync(options, CancellationToken.None)); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } + + [Fact] + public void Live_preflight_failure_blocks_throughput_publish_decision() + { + // Mirrors RunAsync: ThrowIfPreflightBlocksPublish runs before work-channel publishers. + var failed = new[] + { + new CorpusIdPreflightResult("", CorpusIdPreflightKind.Timeout, "work reply timeout"), + }; + Assert.Throws( + () => CorpusArticlePreflight.ThrowIfPreflightBlocksPublish(failed)); + } + + [Fact] + public void Reply_consumer_readiness_is_required_before_request_publish() + { + // ConnectAsync sets readiness before returning; RequestAsync enforces it. + Assert.False(string.IsNullOrWhiteSpace(nameof(ArticleWorkStressClient.IsReplyConsumerReady))); + Assert.Contains( + "not ready", + "Article Work RPC response consumer is not ready; ConnectAsync must complete before RequestAsync.", + StringComparison.Ordinal); + } + + [Fact] + public void StressRunId_is_optional_until_set_then_propagated_on_properties() + { + var props = ArticleWorkStressClient.CreateRequestProperties( + "corr", + "reply", + Guid.Parse("44444444-4444-4444-4444-444444444444"), + stressRunId: "run-gate-1"); + Assert.Equal("run-gate-1", props.Headers!["StressRunId"]); + Assert.True(string.IsNullOrEmpty(props.Expiration)); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflight.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflight.cs new file mode 100644 index 00000000..00f14bcc --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflight.cs @@ -0,0 +1,191 @@ +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Read-only RabbitMQ Management preflight for stress runs. Never purges or mutates queues. + /// + public static class StressQueuePreflight + { + /// + /// Default cleanliness: no ready messages and unacked not above a small residual. + /// + public const long DefaultMaxReady = 0; + + /// + /// Allows a tiny unacked residual (e.g. concurrent unrelated work). Poison storms are far larger. + /// + public const long DefaultMaxUnacked = 2; + + /// + /// Evaluates whether a queue snapshot is clean enough for a controlled concurrency sweep. + /// + public static bool IsClean( + long ready, + long unacked, + long maxReady = DefaultMaxReady, + long maxUnacked = DefaultMaxUnacked) => + ready <= maxReady && unacked <= maxUnacked; + + /// + /// Fetches one Management API snapshot for . Does not modify the broker. + /// + public static async Task FetchAsync( + StressHarnessOptions options, + string queueName, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentException.ThrowIfNullOrWhiteSpace(queueName); + + var user = options.RabbitMqUsername; + var pass = options.RabbitMqPassword; + var creds = Convert.ToBase64String(Encoding.ASCII.GetBytes(user + ":" + pass)); + using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(5) }; + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", creds); + var vhost = Uri.EscapeDataString(options.RabbitMqVirtualHost); + var q = Uri.EscapeDataString(queueName); + var url = $"http://{options.RabbitMqHost}:15672/api/queues/{vhost}/{q}"; + using var response = await http.GetAsync(url, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken) + .ConfigureAwait(false); + using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken) + .ConfigureAwait(false); + var root = doc.RootElement; + return new QueueSnapshot( + DateTimeOffset.UtcNow, + queueName, + root.TryGetProperty("type", out var type) ? type.GetString() ?? string.Empty : string.Empty, + root.TryGetProperty("messages", out var m) ? m.GetInt64() : 0, + root.TryGetProperty("messages_ready", out var r) ? r.GetInt64() : 0, + root.TryGetProperty("messages_unacknowledged", out var u) ? u.GetInt64() : 0, + root.TryGetProperty("consumers", out var c) ? c.GetInt64() : 0, + TryRate(root, "ack_details"), + TryRate(root, "redeliver_details"), + TryRate(root, "deliver_get_details")); + } + + /// + /// Writes a JSON snapshot under the report directory and throws if the queue is not clean. + /// + public static async Task RequireCleanAsync( + StressHarnessOptions options, + string backbone, + string label, + CancellationToken cancellationToken, + long maxReady = DefaultMaxReady, + long maxUnacked = DefaultMaxUnacked) + { + ArgumentNullException.ThrowIfNull(options); + var queueName = ArticleWorkStressClient.ComposeProviderEntity(backbone); + var snap = await FetchAsync(options, queueName, cancellationToken).ConfigureAwait(false); + Directory.CreateDirectory(options.ReportDirectory); + var path = Path.Join( + options.ReportDirectory, + $"preflight-{label}-{DateTimeOffset.UtcNow:yyyyMMddTHHmmss}.json"); + await File.WriteAllTextAsync( + path, + JsonSerializer.Serialize(snap, new JsonSerializerOptions { WriteIndented = true }), + cancellationToken) + .ConfigureAwait(false); + + Console.WriteLine( + $"preflight {label} queue={snap.QueueName} type={snap.Type} messages={snap.Messages} ready={snap.Ready} unacked={snap.Unacked} consumers={snap.Consumers} ackRate={snap.AckRate} redeliverRate={snap.RedeliverRate}"); + + if (!IsClean(snap.Ready, snap.Unacked, maxReady, maxUnacked)) + { + throw new InvalidOperationException( + $"Stress preflight failed for {snap.QueueName}: ready={snap.Ready} unacked={snap.Unacked} " + + $"(limits ready<={maxReady} unacked<={maxUnacked}). " + + "Do not run a concurrency sweep against a poisoned backlog. " + + "Complete operator-approved recovery first (see BACKFILLER_QUEUE_RECOVERY_ASSESSMENT.md). " + + $"Snapshot: {path}"); + } + } + + /// + /// Polls until the queue is clean or elapses. Does not purge or ack. + /// Returns the final snapshot. Throws only when still dirty after the timeout. + /// + public static async Task WaitUntilCleanAsync( + StressHarnessOptions options, + string backbone, + string label, + TimeSpan timeout, + CancellationToken cancellationToken, + long maxReady = DefaultMaxReady, + long maxUnacked = DefaultMaxUnacked, + TimeSpan? pollInterval = null) + { + ArgumentNullException.ThrowIfNull(options); + var interval = pollInterval ?? TimeSpan.FromSeconds(2); + var queueName = ArticleWorkStressClient.ComposeProviderEntity(backbone); + var deadline = DateTimeOffset.UtcNow + timeout; + QueueSnapshot? last = null; + while (DateTimeOffset.UtcNow <= deadline) + { + cancellationToken.ThrowIfCancellationRequested(); + last = await FetchAsync(options, queueName, cancellationToken).ConfigureAwait(false); + Console.WriteLine( + $"wait-clean {label} queue={last.QueueName} ready={last.Ready} unacked={last.Unacked} consumers={last.Consumers}"); + if (IsClean(last.Ready, last.Unacked, maxReady, maxUnacked)) + { + Directory.CreateDirectory(options.ReportDirectory); + var path = Path.Join( + options.ReportDirectory, + $"preflight-{label}-clean-{DateTimeOffset.UtcNow:yyyyMMddTHHmmss}.json"); + await File.WriteAllTextAsync( + path, + JsonSerializer.Serialize(last, new JsonSerializerOptions { WriteIndented = true }), + cancellationToken) + .ConfigureAwait(false); + return last; + } + + try + { + await Task.Delay(interval, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + } + + await RequireCleanAsync(options, backbone, label + "-timeout", cancellationToken, maxReady, maxUnacked) + .ConfigureAwait(false); + return last!; + } + + private static double TryRate(JsonElement root, string detailsName) + { + if (!root.TryGetProperty("message_stats", out var stats)) + { + return 0; + } + + if (!stats.TryGetProperty(detailsName, out var details)) + { + return 0; + } + + return details.TryGetProperty("rate", out var rate) ? rate.GetDouble() : 0; + } + + /// One read-only Management API queue sample. + public sealed record QueueSnapshot( + DateTimeOffset TimestampUtc, + string QueueName, + string Type, + long Messages, + long Ready, + long Unacked, + long Consumers, + double AckRate, + double RedeliverRate, + double DeliverRate); + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflightTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflightTests.cs new file mode 100644 index 00000000..9b2649b6 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflightTests.cs @@ -0,0 +1,29 @@ +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// Deterministic cleanliness thresholds for stress preflight (no broker I/O). + public sealed class StressQueuePreflightTests + { + [Fact] + public void IsClean_true_when_ready_and_unacked_within_limits() + { + Assert.True(StressQueuePreflight.IsClean(ready: 0, unacked: 0)); + Assert.True(StressQueuePreflight.IsClean(ready: 0, unacked: 2)); + } + + [Fact] + public void IsClean_false_when_ready_or_unacked_exceed_limits() + { + Assert.False(StressQueuePreflight.IsClean(ready: 1, unacked: 0)); + Assert.False(StressQueuePreflight.IsClean(ready: 0, unacked: 3)); + Assert.False(StressQueuePreflight.IsClean(ready: 44, unacked: 99)); + Assert.False(StressQueuePreflight.IsClean(ready: 0, unacked: 25)); + } + + [Fact] + public void IsClean_respects_custom_limits() + { + Assert.True(StressQueuePreflight.IsClean(ready: 5, unacked: 10, maxReady: 5, maxUnacked: 10)); + Assert.False(StressQueuePreflight.IsClean(ready: 6, unacked: 10, maxReady: 5, maxUnacked: 10)); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressWorkloadGuard.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressWorkloadGuard.cs new file mode 100644 index 00000000..6476fea8 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressWorkloadGuard.cs @@ -0,0 +1,91 @@ +using System.Security.Cryptography; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Refuses unprovenanced or cross-corpus workload reuse for live stress runs. + /// + public static class StressWorkloadGuard + { + /// + /// Ensures workload files exist, carry a manifest, match the configured corpus path, + /// and that the main ID file hash matches the manifest. Does not invent Message-IDs. + /// + public static async Task RequireProvenancedWorkloadAsync( + StressHarnessOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + var main = Path.Join(options.WorkloadDirectory, StressCorpusPaths.MainIdsAuditFileName); + if (!File.Exists(main)) + { + throw new InvalidOperationException( + "Workload files missing. Run Discover_workload_from_newslogs_when_enabled against the required corpus " + + $"({options.NewsLogDirectory}). Expected {main}. " + + "Stress will not invent Message-IDs or use an unprovenanced cache."); + } + + var manifest = await WorkloadCorpusManifest.TryReadAsync(options.WorkloadDirectory, cancellationToken) + .ConfigureAwait(false); + if (manifest is null) + { + throw new InvalidOperationException( + "Workload directory lacks " + StressCorpusPaths.WorkloadManifestFileName + + ". Stale caches without provenance are rejected. Re-run discovery from " + + options.NewsLogDirectory + "."); + } + + var configured = Path.GetFullPath(options.NewsLogDirectory); + var recorded = Path.GetFullPath(manifest.CorpusDirectory); + if (!string.Equals(configured, recorded, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + "Workload provenance corpus path does not match the configured news-log directory. " + + $"configured={configured} manifest={recorded}. " + + "Re-run discovery; do not reuse another dataset."); + } + + if (manifest.UniqueValidIds <= 0 || manifest.MainSelected <= 0) + { + throw new InvalidOperationException( + "Workload manifest records no usable Message-IDs. Re-run discovery; do not invent replacements."); + } + + var actualHash = WorkloadCorpusManifest.ComputeFileSha256(main); + if (!string.Equals(actualHash, manifest.MainIdsSha256, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + "main-message-ids.txt SHA-256 does not match corpus-manifest.json. " + + "The workload was altered after discovery. Re-run discovery from the configured corpus."); + } + } + + /// + /// Verifies each entry is present in . + /// + public static void RequireSelectedIdsSubset( + IEnumerable selectedIds, + IReadOnlySet allowedIds) + { + ArgumentNullException.ThrowIfNull(selectedIds); + ArgumentNullException.ThrowIfNull(allowedIds); + foreach (var id in selectedIds) + { + if (!allowedIds.Contains(id)) + { + throw new InvalidOperationException( + "Selected Message-ID is not in the allowed corpus set: " + id); + } + } + } + + /// Deterministic SHA-256 of a joined ID list (ordinal order already applied by caller). + public static string HashIds(IEnumerable ids) + { + ArgumentNullException.ThrowIfNull(ids); + var joined = string.Join('\n', ids); + var bytes = System.Text.Encoding.UTF8.GetBytes(joined); + return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/WorkloadCorpusManifest.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/WorkloadCorpusManifest.cs new file mode 100644 index 00000000..23670533 --- /dev/null +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/WorkloadCorpusManifest.cs @@ -0,0 +1,85 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace VectorNNTPD.BackFillerService.Tests.Stress +{ + /// + /// Auditable provenance for a stress workload derived from a news-log corpus. + /// + public sealed class WorkloadCorpusManifest + { + public required string CorpusDirectory { get; init; } + + public required int FileCount { get; init; } + + public required long LogRecordsExamined { get; init; } + + public required long ValidParsedIds { get; init; } + + public required long RejectedInvalidMessageIds { get; init; } + + public required long UniqueValidIds { get; init; } + + public required int WarmupSelected { get; init; } + + public required int MainSelected { get; init; } + + public required int RepeatableSelected { get; init; } + + public required int Seed { get; init; } + + public required string MainIdsSha256 { get; init; } + + public required DateTimeOffset CreatedUtc { get; init; } + + public string? Notes { get; init; } + + public static async Task WriteAsync( + string workloadDirectory, + WorkloadCorpusManifest manifest, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(workloadDirectory); + ArgumentNullException.ThrowIfNull(manifest); + Directory.CreateDirectory(workloadDirectory); + var path = Path.Join(workloadDirectory, StressCorpusPaths.WorkloadManifestFileName); + await File.WriteAllTextAsync( + path, + JsonSerializer.Serialize(manifest, new JsonSerializerOptions { WriteIndented = true }), + cancellationToken) + .ConfigureAwait(false); + } + + public static async Task TryReadAsync( + string workloadDirectory, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(workloadDirectory); + var path = Path.Join(workloadDirectory, StressCorpusPaths.WorkloadManifestFileName); + if (!File.Exists(path)) + { + return null; + } + + await using var stream = File.OpenRead(path); + return await JsonSerializer + .DeserializeAsync(stream, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + public static string ComputeFileSha256(string filePath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(filePath); + using var stream = File.OpenRead(filePath); + var hash = SHA256.HashData(stream); + var sb = new StringBuilder(hash.Length * 2); + foreach (var b in hash) + { + sb.Append(b.ToString("x2")); + } + + return sb.ToString(); + } + } +} diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs index 4881ab05..68f0cab1 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/FakeBackFillerRabbitMq.cs @@ -903,7 +903,7 @@ public async Task PublishConfirmedAsync(RabbitMqConfirmedPublication publication case FakePublishConfirmBehavior.Nack: throw new InvalidOperationException("publisher nack"); case FakePublishConfirmBehavior.Unroutable: - throw new InvalidOperationException("RabbitMQ returned the publication as unroutable."); + throw new RabbitMqPublicationUnroutableException(); case FakePublishConfirmBehavior.Timeout: throw new OperationCanceledException("publisher confirm timeout", cancellationToken); case FakePublishConfirmBehavior.CloseChannel: diff --git a/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqPublicationUnroutableExceptionTests.cs b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqPublicationUnroutableExceptionTests.cs new file mode 100644 index 00000000..e4b6128d --- /dev/null +++ b/tests/VectorNNTPD.Common.Tests/Messaging/RabbitMq/RabbitMqPublicationUnroutableExceptionTests.cs @@ -0,0 +1,33 @@ +using VectorNNTPD.Common.Messaging.RabbitMq; + +namespace VectorNNTPD.Common.Tests.Messaging.RabbitMq +{ + /// + /// Ensures unroutable classification stays distinct from generic publish failures. + /// + public sealed class RabbitMqPublicationUnroutableExceptionTests + { + /// + /// Default message matches the broker return classification string used in production. + /// + [Fact] + public void Default_message_matches_broker_return_classification() + { + var ex = new RabbitMqPublicationUnroutableException(); + Assert.Equal(RabbitMqPublicationUnroutableException.DefaultMessage, ex.Message); + Assert.Contains("unroutable", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + /// + /// Generic invalid-operation publish failures are not typed as unroutable. + /// + [Fact] + public void Is_not_the_same_type_as_generic_invalid_operation() + { + Exception generic = new InvalidOperationException("RabbitMQ negatively acknowledged the publication."); + Exception unroutable = new RabbitMqPublicationUnroutableException(); + Assert.IsNotType(generic); + Assert.IsType(unroutable); + } + } +} From 004e71606b102108f056b2d23dc4b31304fbbe37 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Sun, 25 Oct 2026 19:19:01 +0200 Subject: [PATCH 2/3] fix(tests): resolve Linux Stress corpus and CodeQL findings Skip host-specific corpus assertions when the required Windows corpus is unavailable. Replace generic Stress exception handlers with typed handling, ensure reply-client cleanup in finally, and fix null-flow assertions flagged by CodeQL. --- .../Nntp/NntpProviderSessionTests.cs | 10 +- .../Stress/ArticleWorkStressClient.cs | 202 +++++++++++++----- .../Stress/BackFillerE2EStressHarness.cs | 133 ++++++++++-- .../Stress/BackFillerNntpStressClient.cs | 48 ++++- .../Stress/CorpusArticlePreflight.cs | 24 ++- .../Stress/StressCorpusIntegrityTests.cs | 7 + 6 files changed, 336 insertions(+), 88 deletions(-) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs index 609c27f5..c975a3a1 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs @@ -1532,10 +1532,12 @@ private static async Task AssertDisposeDuringArticlePayloadReadAsync(bool useCal Assert.Equal(ArticleRetrievalKind.ProviderFailure, finished.Kind); Assert.False(finished.SessionReusable); Assert.Null(finished.Article); - Assert.StartsWith(nameof(InvalidOperationException) + ":", finished.Reason, StringComparison.Ordinal); - Assert.False(string.IsNullOrWhiteSpace(finished.Reason[(nameof(InvalidOperationException) + ": ").Length..])); - Assert.DoesNotContain("NullReference", finished.Reason ?? string.Empty, StringComparison.Ordinal); - Assert.DoesNotContain("ArgumentNull", finished.Reason ?? string.Empty, StringComparison.Ordinal); + var reason = finished.Reason ?? string.Empty; + var prefix = nameof(InvalidOperationException) + ":"; + Assert.StartsWith(prefix, reason, StringComparison.Ordinal); + Assert.False(string.IsNullOrWhiteSpace(reason[prefix.Length..].TrimStart())); + Assert.DoesNotContain("NullReference", reason, StringComparison.Ordinal); + Assert.DoesNotContain("ArgumentNull", reason, StringComparison.Ordinal); Assert.Equal(0, Volatile.Read(ref consumed)); Assert.Null(SessionField(session, "_stream")); Assert.Null(SessionField(session, "_reader")); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs index 106bd2e8..3cc21877 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs @@ -4,6 +4,7 @@ using System.Text.Json; using RabbitMQ.Client; using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; using VectorNNTPD.BackFillerService.ArticleWork; namespace VectorNNTPD.BackFillerService.Tests.Stress @@ -212,24 +213,30 @@ await _publishChannel.BasicPublishAsync( .ConfigureAwait(false); publishCompleted = Stopwatch.GetTimestamp(); } - // codeql[cs/catch-of-all-exceptions] Publish must return a broker-error outcome for every transport/AMQP failure type. - catch (Exception ex) + catch (AlreadyClosedException ex) { - CompleteBrokerOutstanding(correlationId, clearWaiting: true); - return new ArticleWorkReply( - ArticleWorkStressOutcome.BrokerError, - backbone, - messageId, - requestId, - null, - null, - null, - ex.GetType().Name) - { - CorrelationId = correlationId, - PublishStartedTimestamp = publishStarted, - PublishCompletedTimestamp = Stopwatch.GetTimestamp(), - }; + return PublishBrokerError(ex, correlationId, backbone, messageId, requestId, publishStarted); + } + catch (OperationInterruptedException ex) + { + return PublishBrokerError(ex, correlationId, backbone, messageId, requestId, publishStarted); + } + catch (IOException ex) + { + // Includes BrokerUnreachableException. + return PublishBrokerError(ex, correlationId, backbone, messageId, requestId, publishStarted); + } + catch (TimeoutException ex) + { + return PublishBrokerError(ex, correlationId, backbone, messageId, requestId, publishStarted); + } + catch (ObjectDisposedException ex) + { + return PublishBrokerError(ex, correlationId, backbone, messageId, requestId, publishStarted); + } + catch (InvalidOperationException ex) + { + return PublishBrokerError(ex, correlationId, backbone, messageId, requestId, publishStarted); } using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -357,25 +364,76 @@ private async Task OnReplyAsync(object sender, BasicDeliverEventArgs args) pending.Waiter.TrySetResult(reply with { CorrelationId = correlationId }); } } - // codeql[cs/catch-of-all-exceptions] Consumer callback must complete the waiter for any parse/delivery fault. - catch (Exception ex) + catch (JsonException ex) { - if (wasWaiting) - { - pending.Waiter.TrySetResult(new ArticleWorkReply( - ArticleWorkStressOutcome.BrokerError, - Backbone: string.Empty, - MessageId: string.Empty, - RequestId: null, - Fqdn: null, - ListenerPort: null, - ArticleIdHex: null, - Error: ex.GetType().Name) - { - CorrelationId = correlationId, - }); - } + CompleteReplyParseFault(wasWaiting, pending, correlationId, ex); + } + catch (KeyNotFoundException ex) + { + CompleteReplyParseFault(wasWaiting, pending, correlationId, ex); + } + catch (InvalidOperationException ex) + { + CompleteReplyParseFault(wasWaiting, pending, correlationId, ex); + } + catch (FormatException ex) + { + CompleteReplyParseFault(wasWaiting, pending, correlationId, ex); + } + catch (ArgumentException ex) + { + CompleteReplyParseFault(wasWaiting, pending, correlationId, ex); + } + } + + private ArticleWorkReply PublishBrokerError( + Exception ex, + string correlationId, + string backbone, + string messageId, + Guid requestId, + long publishStarted) + { + CompleteBrokerOutstanding(correlationId, clearWaiting: true); + return new ArticleWorkReply( + ArticleWorkStressOutcome.BrokerError, + backbone, + messageId, + requestId, + null, + null, + null, + ex.GetType().Name) + { + CorrelationId = correlationId, + PublishStartedTimestamp = publishStarted, + PublishCompletedTimestamp = Stopwatch.GetTimestamp(), + }; + } + + private static void CompleteReplyParseFault( + bool wasWaiting, + PendingRpc pending, + string correlationId, + Exception ex) + { + if (!wasWaiting) + { + return; } + + pending.Waiter.TrySetResult(new ArticleWorkReply( + ArticleWorkStressOutcome.BrokerError, + Backbone: string.Empty, + MessageId: string.Empty, + RequestId: null, + Fqdn: null, + ListenerPort: null, + ArticleIdHex: null, + Error: ex.GetType().Name) + { + CorrelationId = correlationId, + }); } private void CompleteBrokerOutstanding(string correlationId, bool clearWaiting) @@ -487,39 +545,85 @@ public async ValueTask DisposeAsync() Interlocked.Exchange(ref _waitingCount, 0); Interlocked.Exchange(ref _brokerOutstandingCount, 0); + await CloseIgnoreFaultsAsync(_consumeChannel).ConfigureAwait(false); + await CloseIgnoreFaultsAsync(_publishChannel).ConfigureAwait(false); + await CloseIgnoreFaultsAsync(_connection).ConfigureAwait(false); + + await _consumeChannel.DisposeAsync().ConfigureAwait(false); + await _publishChannel.DisposeAsync().ConfigureAwait(false); + await _connection.DisposeAsync().ConfigureAwait(false); + } + + private static async Task CloseIgnoreFaultsAsync(IChannel channel) + { try { - await _consumeChannel.CloseAsync().ConfigureAwait(false); + await channel.CloseAsync().ConfigureAwait(false); } - // codeql[cs/catch-of-all-exceptions] Dispose must continue closing remaining RabbitMQ resources after any close fault. - catch (Exception) + catch (AlreadyClosedException) { // ignored on dispose } - - try + catch (OperationInterruptedException) + { + // ignored on dispose + } + catch (IOException) { - await _publishChannel.CloseAsync().ConfigureAwait(false); + // ignored on dispose } - // codeql[cs/catch-of-all-exceptions] Dispose must continue closing remaining RabbitMQ resources after any close fault. - catch (Exception) + catch (ObjectDisposedException) + { + // ignored on dispose + } + catch (InvalidOperationException) + { + // ignored on dispose + } + catch (TimeoutException) + { + // ignored on dispose + } + catch (OperationCanceledException) { // ignored on dispose } + } + private static async Task CloseIgnoreFaultsAsync(IConnection connection) + { try { - await _connection.CloseAsync().ConfigureAwait(false); + await connection.CloseAsync().ConfigureAwait(false); } - // codeql[cs/catch-of-all-exceptions] Dispose must finish even when the broker connection close faults. - catch (Exception) + catch (AlreadyClosedException) + { + // ignored on dispose + } + catch (OperationInterruptedException) + { + // ignored on dispose + } + catch (IOException) + { + // ignored on dispose + } + catch (ObjectDisposedException) + { + // ignored on dispose + } + catch (InvalidOperationException) + { + // ignored on dispose + } + catch (TimeoutException) + { + // ignored on dispose + } + catch (OperationCanceledException) { // ignored on dispose } - - await _consumeChannel.DisposeAsync().ConfigureAwait(false); - await _publishChannel.DisposeAsync().ConfigureAwait(false); - await _connection.DisposeAsync().ConfigureAwait(false); } private sealed class PendingRpc(TaskCompletionSource waiter) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs index 6f404e27..a336f817 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs @@ -1,8 +1,10 @@ using System.Diagnostics; using System.Net.Http.Headers; +using System.Security.Authentication; using System.Text; using System.Text.Json; using System.Threading.Channels; +using RabbitMQ.Client.Exceptions; namespace VectorNNTPD.BackFillerService.Tests.Stress { @@ -170,6 +172,7 @@ await File.WriteAllTextAsync( .ConfigureAwait(false); workClient.StressRunId = runId; + var disposeReplyClientOnFailure = true; try { // Live retrieval preflight runs after the reply consumer is ready and before any throughput publish. @@ -373,14 +376,30 @@ await deliveryChannel.Writer.WriteAsync( { break; } - // codeql[cs/catch-of-all-exceptions] Publisher worker must record any broker/RPC failure and keep flooding other items. - catch (Exception) + catch (AlreadyClosedException) { - metrics.RecordFailure( - item.Backbone, - StressFailureClass.Other, - (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds); - MaybeAbort(metrics, runCts); + RecordPublisherWorkerFault(metrics, item, runCts); + } + catch (OperationInterruptedException) + { + RecordPublisherWorkerFault(metrics, item, runCts); + } + catch (IOException) + { + // Includes BrokerUnreachableException. + RecordPublisherWorkerFault(metrics, item, runCts); + } + catch (ObjectDisposedException) + { + RecordPublisherWorkerFault(metrics, item, runCts); + } + catch (InvalidOperationException) + { + RecordPublisherWorkerFault(metrics, item, runCts); + } + catch (TimeoutException) + { + RecordPublisherWorkerFault(metrics, item, runCts); } finally { @@ -480,14 +499,25 @@ or NntpResponseKind.NotAvailable (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds); MaybeAbort(metrics, runCts); } - // codeql[cs/catch-of-all-exceptions] Delivery worker must classify any NNTP/transport failure and return the leased client. - catch (Exception) + catch (IOException) { - metrics.RecordFailure( - item.Backbone, - StressFailureClass.Disconnect, - (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds); - MaybeAbort(metrics, runCts); + RecordDeliveryWorkerFault(metrics, item, runCts); + } + catch (ObjectDisposedException) + { + RecordDeliveryWorkerFault(metrics, item, runCts); + } + catch (AuthenticationException) + { + RecordDeliveryWorkerFault(metrics, item, runCts); + } + catch (InvalidOperationException) + { + RecordDeliveryWorkerFault(metrics, item, runCts); + } + catch (TimeoutException) + { + RecordDeliveryWorkerFault(metrics, item, runCts); } finally { @@ -528,10 +558,18 @@ or NntpResponseKind.NotAvailable { await sampler.ConfigureAwait(false); } - // codeql[cs/catch-of-all-exceptions] Queue-depth sampler is optional observability and must not fail the soak. - catch (Exception) + catch (OperationCanceledException) + { + // Sampler cancelled with the measured run. + } + catch (ObjectDisposedException) + { + // Sampler raced disposal during teardown. + } + catch (AggregateException ex) when (ex.InnerExceptions.All(static e => + e is OperationCanceledException or ObjectDisposedException)) { - // Sampler faults are non-fatal. + // Task.WhenAll-style aggregates of expected teardown faults. } metrics.SteadyStateEndTimestamp = Stopwatch.GetTimestamp(); @@ -563,6 +601,7 @@ or NntpResponseKind.NotAvailable } await workClient.DisposeAsync().ConfigureAwait(false); + disposeReplyClientOnFailure = false; metrics.SetReplyLifecycleCounters( workClient.LateRepliesAfterTimeout, workClient.UnresolvedAtDispose, @@ -578,11 +617,12 @@ await File.AppendAllTextAsync( .ConfigureAwait(false); return summary; } - // codeql[cs/catch-of-all-exceptions] Run failure must still dispose the exclusive ReplyTo client before rethrowing. - catch (Exception) + finally { - await workClient.DisposeAsync().ConfigureAwait(false); - throw; + if (disposeReplyClientOnFailure) + { + await workClient.DisposeAsync().ConfigureAwait(false); + } } } @@ -618,11 +658,58 @@ private async Task SampleQueueDepthSafeAsync( var consumers = root.TryGetProperty("consumers", out var c) ? c.GetInt64() : 0; metrics.SampleQueueDepth(Stopwatch.GetTimestamp(), queueName, messages, ready, unacked, consumers); } - // codeql[cs/catch-of-all-exceptions] Management API sampling is optional; any fault must not abort the soak. - catch (Exception) + catch (OperationCanceledException) + { + // Optional observability; never fail the run. + } + catch (HttpRequestException) + { + // Optional observability; never fail the run. + } + catch (JsonException) + { + // Optional observability; never fail the run. + } + catch (IOException) + { + // Optional observability; never fail the run. + } + catch (ObjectDisposedException) + { + // Optional observability; never fail the run. + } + catch (UriFormatException) { // Optional observability; never fail the run. } + catch (FormatException) + { + // Optional observability; never fail the run. + } + } + + private void RecordPublisherWorkerFault( + StressMetrics metrics, + WorkItem item, + CancellationTokenSource runCts) + { + metrics.RecordFailure( + item.Backbone, + StressFailureClass.Other, + (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds); + MaybeAbort(metrics, runCts); + } + + private void RecordDeliveryWorkerFault( + StressMetrics metrics, + DeliveryItem item, + CancellationTokenSource runCts) + { + metrics.RecordFailure( + item.Backbone, + StressFailureClass.Disconnect, + (long)Stopwatch.GetElapsedTime(item.StartedTimestamp).TotalMilliseconds); + MaybeAbort(metrics, runCts); } private void MaybeAbort(StressMetrics metrics, CancellationTokenSource runCts) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs index 8455d542..8d79c50a 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs @@ -137,11 +137,22 @@ public async Task QuitAsync(CancellationToken cancellationToken) { throw; } - // codeql[cs/catch-of-all-exceptions] QUIT under load may fail with any transport fault; dispose must still run. - catch (Exception) + catch (IOException) { // Peer may close without 205 under load; still dispose. } + catch (ObjectDisposedException) + { + // Transport already closed during QUIT; still dispose. + } + catch (AuthenticationException) + { + // TLS fault during QUIT write/read; still dispose. + } + catch (InvalidOperationException) + { + // Stream/state fault during QUIT; still dispose. + } finally { await DisposeAsync().ConfigureAwait(false); @@ -217,17 +228,19 @@ public async Task RentAsync( Interlocked.Decrement(ref _created); // Wait briefly for a return under saturation. await Task.Delay(1, cancellationToken).ConfigureAwait(false); - if (_idle.TryTake(out var recycled) && recycled.IsConnected) + if (!_idle.TryTake(out var recycled) || recycled is null) { - return recycled; + continue; } - if (recycled is not null) + if (recycled.IsConnected) { - await recycled.DisposeAsync().ConfigureAwait(false); - Interlocked.Decrement(ref _created); - metrics.ConnectionClosed(); + return recycled; } + + await recycled.DisposeAsync().ConfigureAwait(false); + Interlocked.Decrement(ref _created); + metrics.ConnectionClosed(); } } @@ -258,8 +271,23 @@ public async ValueTask DisposeAsync() using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); await client.QuitAsync(cts.Token).ConfigureAwait(false); } - // codeql[cs/catch-of-all-exceptions] Pool dispose must force-close idle clients after any QUIT failure. - catch (Exception) + catch (OperationCanceledException) + { + await client.DisposeAsync().ConfigureAwait(false); + } + catch (IOException) + { + await client.DisposeAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + await client.DisposeAsync().ConfigureAwait(false); + } + catch (AuthenticationException) + { + await client.DisposeAsync().ConfigureAwait(false); + } + catch (InvalidOperationException) { await client.DisposeAsync().ConfigureAwait(false); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs index 962cebae..585981f0 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs @@ -73,8 +73,28 @@ public static async Task> RunLiveRetrieva }; results.Add(new CorpusIdPreflightResult(id, kind, reply.Error ?? reply.Outcome.ToString())); } - // codeql[cs/catch-of-all-exceptions] Live preflight must record every retrieval fault as an explicit per-ID outcome. - catch (Exception ex) when (ex is not OperationCanceledException) + catch (RabbitMQ.Client.Exceptions.AlreadyClosedException ex) + { + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.RetrievalFailed, ex.GetType().Name)); + } + catch (RabbitMQ.Client.Exceptions.OperationInterruptedException ex) + { + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.RetrievalFailed, ex.GetType().Name)); + } + catch (IOException ex) + { + // Includes BrokerUnreachableException. + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.RetrievalFailed, ex.GetType().Name)); + } + catch (ObjectDisposedException ex) + { + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.RetrievalFailed, ex.GetType().Name)); + } + catch (InvalidOperationException ex) + { + results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.RetrievalFailed, ex.GetType().Name)); + } + catch (TimeoutException ex) { results.Add(new CorpusIdPreflightResult(id, CorpusIdPreflightKind.RetrievalFailed, ex.GetType().Name)); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs index 5852eca6..9dd6fb5e 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs @@ -16,6 +16,13 @@ public void Required_corpus_path_is_the_vectornntp_newslogs_directory() [Fact] public void Required_corpus_path_resolves_to_usable_logs_on_this_host() { + // Absolute corpus path is machine-local (see StressCorpusPaths). CI agents do not mount it. + // Live stress (VECTORNNTPD_BACKFILLER_E2E_STRESS=1) still requires the directory when enabled. + if (!Directory.Exists(StressCorpusPaths.RequiredNewsLogDirectory)) + { + return; + } + // After the verified junction (or a real VectorNNTP tree), discovery must see *.log files. NewsLogMessageIdDiscovery.RequireUsableCorpusDirectory(StressCorpusPaths.RequiredNewsLogDirectory); Assert.NotEmpty(NewsLogMessageIdDiscovery.DiscoverLogFiles(StressCorpusPaths.RequiredNewsLogDirectory)); From 053c9544e59bfbb592fc0bfb1542ed416ba8fe23 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Sun, 25 Oct 2026 19:35:23 +0200 Subject: [PATCH 3/3] docs(backfiller): complete XML documentation audit Document all applicable production, test, helper, and Stress declarations, including private and internal members. Correct stale documentation and close missing XML documentation gaps. Record the namespace documentation limitation (CS1587) in the audit report. Release x64 builds pass with zero warnings or errors. --- .../ArticleWork/ArticleWorkConsumerSession.cs | 13 +- .../ArticleWorkDeliveryPipeline.cs | 5 + .../ArticleWork/ArticleWorkDisposition.cs | 4 +- .../ArticleWorkDispositionPlanner.cs | 10 +- .../ArticleWork/ArticleWorkOutcome.cs | 31 +++-- .../ArticleWorkQuorumManagementHealthProbe.cs | 16 +++ .../Nntp/NntpExceptionDiagnostics.cs | 8 ++ .../Nntp/NntpSessionPool.cs | 3 + .../AcmeCertificateApplicationServiceTests.cs | 12 ++ ...leFailedDiagnosticsIsolatedRuntimeTests.cs | 9 ++ .../ArticleWork/ArticleProcessingLogTests.cs | 5 + .../ArticleWorkChannelFailureRecoveryTests.cs | 19 +++ ...leWorkQuorumHealthAndDeliveryLimitTests.cs | 28 ++++ .../ArticleWorkResponseWireProtocolTests.cs | 3 + ...leWorkTopologyDeclarationIsolationTests.cs | 8 ++ .../ProviderFailureTerminalSettlementTests.cs | 1 + ...expectedFailureRpcErrorDiagnosticsTests.cs | 6 + .../BackFillerOptionsValidatorTests.cs | 1 + .../BackFillerRuntimeFqdnProjectionTests.cs | 8 ++ .../Listener/NntpCacheListenerSessionTests.cs | 3 + .../Nntp/NntpCommandAllocationBaseline.cs | 45 +++++++ .../Nntp/NntpCommandBudgetTests.cs | 5 + .../Nntp/NntpExceptionDiagnosticsTests.cs | 3 + .../Nntp/NntpProviderSessionTests.cs | 1 + .../Nntp/NntpSessionPoolTests.cs | 11 ++ .../Nntp/NntpStatusLineTests.cs | 50 +++++++ .../ProviderSessionEstablishmentGateTests.cs | 37 ++++++ .../RabbitMqBlockedConnectionTimeoutTests.cs | 29 ++++ .../CanonicalArticleBufferRetentionTests.cs | 23 ++++ .../ArticleWorkRpcLifecycleContractTests.cs | 11 ++ .../ArticleWorkRpcOutstandingTracker.cs | 15 +++ .../ArticleWorkRpcOutstandingTrackerTests.cs | 5 + .../Stress/ArticleWorkStressClient.cs | 49 +++++++ ...eWorkStressClientRequestPropertiesTests.cs | 5 + .../Stress/BackFillerE2EStressBench.cs | 9 ++ .../Stress/BackFillerE2EStressHarness.cs | 12 ++ .../Stress/BackFillerNntpStressClient.cs | 37 ++++++ .../Stress/CorpusArticlePreflight.cs | 15 +++ .../Stress/NewsLogMessageIdDiscovery.cs | 3 + .../Stress/NewsLogMessageIdDiscoveryTests.cs | 10 ++ .../Stress/NntpArticleResponseReader.cs | 58 ++++++++ .../Stress/NntpArticleResponseReaderTests.cs | 6 + .../Stress/StageLatencyStats.cs | 5 + .../Stress/StressAbortDecisionTests.cs | 4 + .../Stress/StressCorpusIntegrityTests.cs | 14 ++ .../Stress/StressHarnessOptions.cs | 58 ++++++++ .../Stress/StressMetrics.cs | 124 ++++++++++++++++++ .../Stress/StressPublishGateTests.cs | 5 + .../Stress/StressQueuePreflight.cs | 1 + .../Stress/StressQueuePreflightTests.cs | 3 + .../Stress/WorkloadCorpusManifest.cs | 16 +++ .../TestDoubles/SequenceDbDataReader.cs | 13 ++ 52 files changed, 860 insertions(+), 15 deletions(-) diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs index b81106c3..ccab9478 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkConsumerSession.cs @@ -745,8 +745,19 @@ bool ChannelStillCurrent() => } /// - /// True when a retryable outcome was settled after the broker delivery count reached the configured limit. + /// Whether is one of the failure classifications that participates in + /// delivery-limit exhaustion logging when the effective broker count is at or above the configured limit. /// + /// Delivery whose count and redelivered flag feed the effective count. + /// Handler/pipeline outcome observed for this delivery. + /// + /// when the effective count is exhausted and is + /// ProviderFailure, Cancelled, UnexpectedFailure, or RetentionRejected. + /// + /// + /// Used for structured logging only. Settlement itself follows ; + /// ProviderFailure, UnexpectedFailure, and RetentionRejected ACK with a reply rather than NACK-requeue. + /// private bool IsDeliveryLimitExhausted(RabbitMqManualAckDelivery delivery, ArticleWorkOutcome outcome) { var effectiveCount = ArticleWorkDispositionPlanner.ResolveEffectiveDeliveryCount( diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs index 97639748..1da7fbb9 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryPipeline.cs @@ -321,6 +321,11 @@ await lease.TrySettleAsync( /// /// Builds NACK requeue/discard for publish-path retries (not handler ). /// + /// Broker x-delivery-count when present; otherwise null. + /// AMQP redelivered flag for missing-count handling. + /// + /// ACK false, publish false, and requeue while the effective count is below . + /// /// /// Must not use the planner's terminal UnexpectedFailure arm (ACK + reply). A failed or incomplete /// reply publish must leave the delivery eligible for requeue while retries remain. diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDisposition.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDisposition.cs index 67ab4dba..201b738c 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDisposition.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDisposition.cs @@ -10,7 +10,9 @@ namespace VectorNNTPD.BackFillerService.ArticleWork /// NACK requeue flag. Ignored when is . /// /// Whether a terminal RPC response should be attempted before settlement. - /// Retryable outcomes are never published. + /// and publish-path retry dispositions keep this + /// ; terminal failure replies (ProviderFailure, UnexpectedFailure, RetentionRejected) + /// set it with . /// internal readonly record struct ArticleWorkDisposition( bool Acknowledge, diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs index f041f817..cd90e879 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDispositionPlanner.cs @@ -51,7 +51,7 @@ internal static int ResolveEffectiveDeliveryCount( /// /// Creates the settlement plan for one classified result. /// - /// Processing or parse the outcome. + /// Classified processing or parse outcome. /// /// Whether AMQP CorrelationId and ReplyTo are both present. /// Only uses this to decide publication. @@ -121,7 +121,13 @@ or ArticleWorkOutcome.UnexpectedFailure }; } - /// NACK requeue while retries remain; NACK without requeue when the delivery limit is exhausted. + /// + /// Builds a non-publishing NACK disposition for cooperative cancel or unrecognized outcomes. + /// + /// + /// When , NACK without requeue; otherwise NACK with requeue. + /// + /// ACK false, false, and requeue inverted from exhaustion. private static ArticleWorkDisposition CreateRetryable(bool retriesExhausted) => new(Acknowledge: false, Requeue: !retriesExhausted, PublishResponse: false); } diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkOutcome.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkOutcome.cs index c49a987f..2dc34b36 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkOutcome.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkOutcome.cs @@ -4,17 +4,20 @@ namespace VectorNNTPD.BackFillerService.ArticleWork /// Article-work classification used for disposition planning. /// /// - /// Terminal protocol outcomes are , , - /// , and . Retryable internal - /// outcomes are , , - /// , and . + /// Terminal ACK + correlated reply outcomes are , , + /// , (when replyable), + /// , , and . + /// is the cooperative cancel/shutdown path: NACK requeue while the delivery + /// limit allows, without a terminal RPC reply. See . /// internal enum ArticleWorkOutcome { - /// Article recovered and retained for NNTP delivery. + /// Article recovered and retained for NNTP delivery. ACK + Success reply. Success = 0, - /// Provider did not have the article. + /// + /// Provider reported a definitive miss (NNTP 430 or ARTICLE 451). ACK + ArticleNotFound reply. + /// ArticleNotFound = 1, /// @@ -23,22 +26,28 @@ internal enum ArticleWorkOutcome /// InvalidArticle = 2, - /// Malformed protocol payload or AMQP metadata. + /// Malformed protocol payload or AMQP metadata. ACK; publish when replyable. InvalidRequest = 3, - /// Transient provider/infrastructure failure. No terminal RPC response. + /// + /// Provider or infrastructure failure for this lookup. ACK + ProviderFailure reply (not NACK-requeued). + /// ProviderFailure = 4, - /// Processing cancelled. No terminal RPC response. + /// + /// Processing cancelled or shut down before a terminal reply. NACK requeue while retries remain; no reply. + /// Cancelled = 5, - /// Unexpected failure. No terminal RPC response. + /// + /// Unexpected handler or pipeline failure for this delivery. ACK + UnexpectedFailure reply (not NACK-requeued). + /// UnexpectedFailure = 6, /// /// Retrieval succeeded, but retention could not admit the payload. /// Distinct from and . - /// Settled as NACK requeue without a terminal RPC response. + /// ACK + RetentionRejected reply (not NACK-requeued). /// RetentionRejected = 7, } diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementHealthProbe.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementHealthProbe.cs index 8ea79bbd..f77300a8 100644 --- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementHealthProbe.cs +++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkQuorumManagementHealthProbe.cs @@ -21,7 +21,10 @@ internal sealed class ArticleWorkQuorumManagementHealthProbe : IArticleWorkQuoru /// Named client for Management calls. internal const string HttpClientName = "VectorNNTPD.ArticleWork.QuorumManagement"; + /// Factory that creates the named Management HTTP client for each inspection. private readonly IHttpClientFactory _httpClientFactory; + + /// Bound RabbitMQ options, including Management base URL, vhost, and credentials. private readonly IOptions _options; /// @@ -155,6 +158,9 @@ public async Task InspectAsync( } /// Maps a Management queue document to a health result. + /// Deserialized Management queue payload for the provider queue. + /// Configured membership floor compared to observed members and online counts. + /// Healthy, insufficient, missing, or unverifiable classification with a non-secret reason. private static ArticleWorkQuorumHealthResult Evaluate( ArticleWorkQuorumManagementQueueDocument document, int requiredMembers) @@ -217,6 +223,10 @@ private static ArticleWorkQuorumHealthResult Evaluate( } /// Builds an absolute Management queue URL without credentials. + /// Management HTTP base URL (scheme and authority; trailing slash optional). + /// AMQP virtual host; URI-encoded into the path. + /// Queue name; URI-encoded into the path. + /// Absolute /api/queues/{vhost}/{queue} URI under . internal static Uri BuildQueueUri(string baseUrl, string vhost, string queueName) { var root = baseUrl.TrimEnd('/'); @@ -225,6 +235,12 @@ internal static Uri BuildQueueUri(string baseUrl, string vhost, string queueName return new Uri($"{root}/api/queues/{encodedVhost}/{encodedQueue}"); } + /// + /// Builds an result with null membership observations. + /// + /// Configured membership floor copied onto the result. + /// Operator-facing explanation of why membership could not be verified. + /// An unverifiable health result with no observed member, online, or leader counts. private static ArticleWorkQuorumHealthResult Unverifiable(int requiredMembers, string reason) => new( ArticleWorkQuorumHealthKind.Unverifiable, diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpExceptionDiagnostics.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpExceptionDiagnostics.cs index b71b5cc1..b9eef3dc 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpExceptionDiagnostics.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpExceptionDiagnostics.cs @@ -127,10 +127,18 @@ private static string Truncate(string reason) /// Reference equality for exception cycle detection along chains. private sealed class ReferenceEqualityComparer : IEqualityComparer { + /// Shared comparer instance used by . internal static readonly ReferenceEqualityComparer Instance = new(); + /// Returns whether and are the same object instance. + /// First exception, or . + /// Second exception, or . + /// when both references are identical. public bool Equals(Exception? x, Exception? y) => ReferenceEquals(x, y); + /// Returns the runtime identity hash for . + /// Exception whose identity is hashed. Must not be null for use. + /// A hash derived from object identity, not . public int GetHashCode(Exception obj) => System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj); } } diff --git a/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionPool.cs b/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionPool.cs index a4ac444c..63254d37 100644 --- a/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionPool.cs +++ b/src/VectorNNTPD.BackFillerService/Nntp/NntpSessionPool.cs @@ -915,6 +915,9 @@ await RetireSessionAsync( /// private struct PermitWaitLink : IDisposable { + /// + /// Linked caller-and-shutdown source created on the first blocking wait; null on the fast path. + /// private CancellationTokenSource? _linked; /// Returns when the caller token does not need a separate link. Otherwise returns the linked token. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs index 8aa16c25..ca82bae3 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs @@ -459,6 +459,14 @@ private static PasswordRenewal CreatePasswordRenewal(DateTimeOffset clock, strin return store.Load(); } + /// + /// Builds a started-ready harness with manual clock, injectable issuer, and optional on-disk certificate seed. + /// + /// Initial UTC instant for . + /// Issuer used by the wired . + /// Certificate validity end when is true; defaults to ninety days after . + /// When true, writes exportable PFX material into the temp ACME directory before constructing the manager. + /// Harness whose is constructed but not started. private static ObservedRenewal CreateObservedRenewal( DateTimeOffset clock, ICertificateIssuer issuer, @@ -814,6 +822,7 @@ private sealed class UnusedHttpClientFactory : IHttpClientFactory /// Manual-clock renewal service, watchdog health, and the publisher used to swap certificate leases. private sealed class ObservedRenewal : IAsyncDisposable { + /// Temp ACME state directory disposed after the service and publisher stop. private readonly TempAcmeDir _directory; /// Stores the started collaborators. The caller starts the service. @@ -856,7 +865,10 @@ public async ValueTask DisposeAsync() /// Issuer whose result depends on the 1-based call count. private sealed class ScriptedIssuer : ICertificateIssuer { + /// Produces material for the 1-based invocation number. private readonly Func, string, CertificateMaterial> _next; + + /// Number of completed calls. private int _calls; /// Stores the script. Call 1 is the first . diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleFailedDiagnosticsIsolatedRuntimeTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleFailedDiagnosticsIsolatedRuntimeTests.cs index 044cd5a7..6e5862cb 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleFailedDiagnosticsIsolatedRuntimeTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleFailedDiagnosticsIsolatedRuntimeTests.cs @@ -157,6 +157,9 @@ public async Task Isolated_retrieve_exception_emits_failed_event_then_rethrows() Assert.Null(entry.Exception); } + /// Builds a minimal work item with a fake settlement lease for isolated-runtime diagnostics tests. + /// NNTP Message-ID header value. + /// Provider backbone name on the request. private static ArticleWorkItem Item(string messageId, string backbone) => new( new ArticleWorkRequest(1, Guid.Parse(ArticleWorkTestDeliveries.CanonicalRequestId), messageId, backbone), @@ -164,6 +167,7 @@ private static ArticleWorkItem Item(string messageId, string backbone) => "reply", new ArticleWorkSettlementLease(new FakeBackFillerRabbitMqChannel(1), 1, 1)); + /// Creates a small in-memory retention authority for Article Work handler tests. private static ArticleRetentionAuthority CreateRetention() => new( new BackFillerArticleRetentionRuntimeOptions(1024 * 1024, TimeSpan.FromHours(1), TimeSpan.FromMinutes(1), 256), @@ -172,8 +176,10 @@ private static ArticleRetentionAuthority CreateRetention() => TimeProvider.System, NullLogger.Instance); + /// that completes with a fault carrying the configured exception. private sealed class ThrowingRetriever(Exception exception) : INntpArticleRetriever { + /// public Task RetrieveAsync( ArticleWorkItem item, int maxArticleBytes, @@ -182,10 +188,13 @@ public Task RetrieveAsync( Task.FromException(exception); } + /// Captures Serilog events in memory for assertion. private sealed class ListSink : ILogEventSink { + /// Events emitted through . public List Events { get; } = new(); + /// public void Emit(LogEvent logEvent) => Events.Add(logEvent); } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleProcessingLogTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleProcessingLogTests.cs index b698b05c..72b2cfd0 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleProcessingLogTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleProcessingLogTests.cs @@ -479,6 +479,11 @@ private sealed class ScriptedRetriever : INntpArticleRetriever /// Optional physical connection identity stamped onto the returned result. private readonly Guid _connectionId; + /// Configures scripted retrieval inputs for elapsed and outcome logging tests. + /// Article bytes for the callback, or null to skip consumption (failures). + /// Base retrieval result (kind, code, session reuse). + /// Value assigned to on return. + /// Optional connection identity stamped onto the returned result. internal ScriptedRetriever( byte[]? payload, ArticleRetrievalResult result, diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs index d084b7b2..18b3154b 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkChannelFailureRecoveryTests.cs @@ -432,6 +432,7 @@ await ArticleWorkTestDeliveries.WaitUntilAsync( Assert.Single(context.PublishChannel.Publications); } + /// Canonical success response intent used when exercising publish recovery paths. private static ArticleWorkResponseIntent SuccessIntent() => new( ArticleWorkOutcome.Success, @@ -445,6 +446,9 @@ private static ArticleWorkResponseIntent SuccessIntent() => ArticleWorkTestDeliveries.CanonicalListenerPort, ArticleWorkTestDeliveries.CanonicalArticleIdHex); + /// Starts a single-session Article Work consumer against a fake RabbitMQ factory. + /// Handler wired into the consumer; defaults to . + /// Harness holding factory, connections, and running consumer. private static async Task StartConsumerAsync(IArticleWorkHandler? handler = null) { var factory = new FakeBackFillerRabbitMqConnectionFactory(); @@ -476,6 +480,7 @@ await ArticleWorkTestDeliveries.WaitUntilAsync( return new ConsumerHarness(factory, connections, consumer); } + /// Starts a running with publish confirms enabled. private static async Task StartPublisherAsync() { var factory = new FakeBackFillerRabbitMqConnectionFactory @@ -494,6 +499,9 @@ private static async Task StartPublisherAsync() return new PublisherHarness(factory, connections, publisher); } + /// Publishes a single-backbone usable-capacity snapshot when is positive. + /// Backbone name key in the snapshot. + /// Active session count; zero omits the backbone from the snapshot. private static BackboneUsableCapacityState CreateCapacity(string backbone, int active) { var capacity = new BackboneUsableCapacityState(); @@ -508,17 +516,22 @@ private static BackboneUsableCapacityState CreateCapacity(string backbone, int a return capacity; } + /// Owns a started consumer and its RabbitMQ connections for channel-failure tests. private sealed class ConsumerHarness( FakeBackFillerRabbitMqConnectionFactory factory, RabbitMqService connections, ArticleWorkConsumerService consumer) : IAsyncDisposable { + /// Fake factory that created the test connection. public FakeBackFillerRabbitMqConnectionFactory Factory { get; } = factory; + /// Started instance. public RabbitMqService Connections { get; } = connections; + /// Running Article Work consumer under test. public ArticleWorkConsumerService Consumer { get; } = consumer; + /// Stops the consumer, then disposes connections. public async ValueTask DisposeAsync() { await Consumer.DisposeAsync(); @@ -526,20 +539,26 @@ public async ValueTask DisposeAsync() } } + /// Owns a started response publisher and exposes its fake publish channel. private sealed class PublisherHarness( FakeBackFillerRabbitMqConnectionFactory factory, RabbitMqService connections, ArticleWorkResponsePublisher publisher) : IAsyncDisposable { + /// Fake factory that created the test connection. public FakeBackFillerRabbitMqConnectionFactory Factory { get; } = factory; + /// Started instance. public RabbitMqService Connections { get; } = connections; + /// Running response publisher under test. public ArticleWorkResponsePublisher Publisher { get; } = publisher; + /// Fake publish channel backing . public FakeBackFillerRabbitMqPublishChannel PublishChannel => Assert.IsType(Publisher.Channel); + /// Stops the publisher, then disposes connections. public async ValueTask DisposeAsync() { await Publisher.DisposeAsync(); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs index 5517afae..5f39bb50 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkQuorumHealthAndDeliveryLimitTests.cs @@ -19,7 +19,10 @@ namespace VectorNNTPD.BackFillerService.Tests.ArticleWork /// public sealed class ArticleWorkQuorumHealthAndDeliveryLimitTests { + /// Structured log event id for quorum membership not ready. private const int QuorumMembershipNotReadyEventId = 5325; + + /// Structured log event id for quorum membership healthy. private const int QuorumMembershipHealthyEventId = 5326; /// @@ -744,6 +747,7 @@ public async Task Management_probe_missing_leader_is_unverifiable() } } + /// Builds a healthy quorum probe result for stubbed management responses. private static ArticleWorkQuorumHealthResult Healthy(int required, int observed, int online) => new( ArticleWorkQuorumHealthKind.Healthy, @@ -753,6 +757,7 @@ private static ArticleWorkQuorumHealthResult Healthy(int required, int observed, Leader: "rabbit@node-a", Reason: "Quorum membership and online availability meet the configured policy."); + /// Builds an insufficient-members quorum probe result for stubbed management responses. private static ArticleWorkQuorumHealthResult Deficient(int required, int observed, int online) => new( ArticleWorkQuorumHealthKind.InsufficientMembers, @@ -762,6 +767,7 @@ private static ArticleWorkQuorumHealthResult Deficient(int required, int observe Leader: "rabbit@node-a", Reason: "Configured quorum membership is below the required group size. Operator membership growth is required for pre-existing queues."); + /// Creates runtime options with the given RabbitMQ host count and Article Work delivery limit. private static BackFillerRuntimeOptions CreateRuntime(int hostCount, int deliveryLimit) { var rabbit = BackFillerTestOptions.CreateValidRabbitMqWithHostCount(hostCount); @@ -773,6 +779,7 @@ private static BackFillerRuntimeOptions CreateRuntime(int hostCount, int deliver rabbit); } + /// Constructs a consumer with deferred handler, recording publisher, and optional quorum probe. private static ArticleWorkConsumerService CreateConsumer( IRabbitMqService connections, IBackFillerProviderCatalog catalog, @@ -790,6 +797,7 @@ private static ArticleWorkConsumerService CreateConsumer( capacity, quorumHealth: quorumHealth); + /// Builds a static provider catalog from backbone and max-session pairs. private static StaticBackFillerProviderCatalog CreateCatalog( IReadOnlyList<(string Backbone, int MaxSessions)> providers) => new( @@ -805,6 +813,7 @@ private static StaticBackFillerProviderCatalog CreateCatalog( p.MaxSessions)), ]); + /// Publishes a usable-capacity snapshot omitting backbones with zero active sessions. private static BackboneUsableCapacityState CreateCapacity( IReadOnlyList<(string Backbone, int Active)> capacities) { @@ -823,6 +832,8 @@ private static BackboneUsableCapacityState CreateCapacity( return state; } + /// Wires a management health probe to a single-handler HTTP client factory. + /// Probe and disposable factory lifetime. private static (ArticleWorkQuorumManagementHealthProbe Probe, IDisposable Lifetime) CreateManagementProbe( HttpMessageHandler handler, string baseUrl) @@ -847,10 +858,13 @@ private static (ArticleWorkQuorumManagementHealthProbe Probe, IDisposable Lifeti /// Controllable Management stand-in for membership readiness tests. private sealed class ControllableArticleWorkQuorumHealthProbe : IArticleWorkQuorumHealthProbe { + /// Result returned from the next call. internal ArticleWorkQuorumHealthResult Next { get; set; } + /// Number of completed inspect calls. internal int InspectCount; + /// public Task InspectAsync( string queueName, int requiredMembers, @@ -862,29 +876,43 @@ public Task InspectAsync( } } + /// Returns one shared backed by the injected handler. private sealed class SingleHandlerHttpClientFactory : IHttpClientFactory, IDisposable { + /// Client disposed with the factory. private readonly HttpClient _client; + /// Creates a client that owns . + /// Handler used for all management API requests. public SingleHandlerHttpClientFactory(HttpMessageHandler handler) => _client = new HttpClient(handler, disposeHandler: true); + /// public HttpClient CreateClient(string name) => _client; + /// Disposes the shared client. public void Dispose() => _client.Dispose(); } + /// Returns a fixed status code and JSON body for every HTTP request. private sealed class StubHttpMessageHandler : HttpMessageHandler { + /// HTTP status returned from . private readonly HttpStatusCode _statusCode; + + /// UTF-8 response body bytes. private readonly byte[] _bodyBytes; + /// Captures the scripted management API response. + /// HTTP status for every request. + /// Response body text encoded as UTF-8. public StubHttpMessageHandler(HttpStatusCode statusCode, string body) { _statusCode = statusCode; _bodyBytes = Encoding.UTF8.GetBytes(body); } + /// protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponseWireProtocolTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponseWireProtocolTests.cs index e3856893..b04de531 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponseWireProtocolTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkResponseWireProtocolTests.cs @@ -19,6 +19,9 @@ public void Response_expiration_milliseconds_is_exactly_1000() Assert.Equal("1000", ArticleWorkResponseWireProtocol.ExpirationMilliseconds); } + /// + /// Verifies the response AMQP expiration constant is non-empty, fixed at one second, and not treated as a work-request TTL. + /// [Fact] public void Response_expiration_is_distinct_from_omitted_work_request_ttl() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs index 7acc02a2..71d02e40 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ArticleWorkTopologyDeclarationIsolationTests.cs @@ -315,6 +315,9 @@ public async Task Successful_declarations_reuse_one_open_topology_channel() await connections.DisposeAsync(); } + /// Hooks manual-ack channel creation so queue declare failures are scripted per queue name. + /// Factory whose last connection receives the hook. + /// Returns an exception for a queue name, or null to succeed. private static void ArmQueueDeclareFailure( FakeBackFillerRabbitMqConnectionFactory factory, Func failure) @@ -325,6 +328,8 @@ private static void ArmQueueDeclareFailure( }; } + /// Builds a RabbitMQ precondition-failed shutdown wrapped as . + /// Queue name embedded in the broker error text. private static OperationInterruptedException CreatePreconditionFailed(string queue) { var shutdown = new ShutdownEventArgs( @@ -336,6 +341,7 @@ private static OperationInterruptedException CreatePreconditionFailed(string que return new OperationInterruptedException(shutdown); } + /// Constructs a consumer with deferred handler and recording publisher for topology tests. private static ArticleWorkConsumerService CreateConsumer( IRabbitMqService connections, IBackFillerProviderCatalog catalog, @@ -350,6 +356,7 @@ private static ArticleWorkConsumerService CreateConsumer( catalog, capacity); + /// Builds a static provider catalog from backbone and max-session pairs. private static StaticBackFillerProviderCatalog CreateCatalog( IReadOnlyList<(string Backbone, int MaxSessions)> providers) => new( @@ -365,6 +372,7 @@ private static StaticBackFillerProviderCatalog CreateCatalog( p.MaxSessions)), ]); + /// Publishes a usable-capacity snapshot omitting backbones with zero active sessions. private static BackboneUsableCapacityState CreateCapacity( IReadOnlyList<(string Backbone, int Active)> capacities) { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ProviderFailureTerminalSettlementTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ProviderFailureTerminalSettlementTests.cs index dbb0108a..b923576c 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ProviderFailureTerminalSettlementTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/ProviderFailureTerminalSettlementTests.cs @@ -113,6 +113,7 @@ public void Established_terminal_and_cancel_dispositions_unchanged() AssertDisposition(ArticleWorkOutcome.Cancelled, acknowledge: false, requeue: true, publish: false); } + /// Asserts ACK/requeue/publish flags from for one terminal outcome. private static void AssertDisposition( ArticleWorkOutcome outcome, bool acknowledge, diff --git a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/UnexpectedFailureRpcErrorDiagnosticsTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/UnexpectedFailureRpcErrorDiagnosticsTests.cs index b2be500e..18c4697d 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/UnexpectedFailureRpcErrorDiagnosticsTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/ArticleWork/UnexpectedFailureRpcErrorDiagnosticsTests.cs @@ -299,6 +299,7 @@ public async Task Isolated_pipeline_retrieve_exception_preserves_distinct_inner( Assert.Equal(pipeline.LastEscapedHandlerFailureError, Assert.Single(publisher.Published).Error); } + /// Creates a small in-memory retention authority for RPC error diagnostics tests. private static ArticleRetentionAuthority CreateRetention() => new( new BackFillerArticleRetentionRuntimeOptions(1024 * 1024, TimeSpan.FromHours(1), TimeSpan.FromMinutes(1), 256), @@ -307,15 +308,20 @@ private static ArticleRetentionAuthority CreateRetention() => TimeProvider.System, NullLogger.Instance); + /// that synchronously throws the configured exception. private sealed class ThrowingRetriever : INntpArticleRetriever { + /// Exception propagated from . private readonly Exception _exception; + /// Stores the exception to throw on retrieve. + /// Fault surfaced to the Article Work pipeline. internal ThrowingRetriever(Exception exception) { _exception = exception; } + /// public Task RetrieveAsync( ArticleWorkItem item, int maxArticleBytes, diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerOptionsValidatorTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerOptionsValidatorTests.cs index 8b3c33d7..65c0ca72 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerOptionsValidatorTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerOptionsValidatorTests.cs @@ -17,6 +17,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Configuration public sealed class BackFillerOptionsValidatorTests { + /// Expected validation text when physical RAM is below the retention minimum. private static readonly string ExpectedMinimumPhysicalMemoryPhrase = $"at least {BackFillerArticleRetentionOptions.MinimumPhysicalMemoryGibibytes} GiB ({BackFillerArticleRetentionOptions.MinimumPhysicalMemoryBytes} bytes) of physical RAM"; diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeFqdnProjectionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeFqdnProjectionTests.cs index b2af1842..f0974390 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeFqdnProjectionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Configuration/BackFillerRuntimeFqdnProjectionTests.cs @@ -233,6 +233,9 @@ private static ArticleRecordCreateResult CreateMinimalArticle(string localFqdn) /// Hosts ProviderAccountConfigurationService for FQDN projection assertions. private sealed class ProjectionHarness : IAsyncDisposable { + /// Retains account configuration and provider registry for FQDN projection tests. + /// Started or constructed account configuration service. + /// Provider registry disposed with the harness. internal ProjectionHarness( ProviderAccountConfigurationService service, NntpProviderRegistry registry) @@ -241,10 +244,13 @@ internal ProjectionHarness( Registry = registry; } + /// Account configuration service under test. internal ProviderAccountConfigurationService Service { get; } + /// Provider registry wired to the service. internal NntpProviderRegistry Registry { get; } + /// Disposes the service and registry in order. public async ValueTask DisposeAsync() { await Service.DisposeAsync().ConfigureAwait(false); @@ -255,8 +261,10 @@ public async ValueTask DisposeAsync() /// Shared-configuration stub with a replaceable snapshot. private sealed class FixedSharedConfigurationSource : IBackFillerSharedConfigurationSource { + /// Snapshot returned from every call. public required NntpSharedConfiguration Value { get; set; } + /// public ValueTask ReadAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Listener/NntpCacheListenerSessionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Listener/NntpCacheListenerSessionTests.cs index f472a591..b18d98ac 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Listener/NntpCacheListenerSessionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Listener/NntpCacheListenerSessionTests.cs @@ -1228,7 +1228,10 @@ private static int IndexOf(byte[] buffer, byte[] pattern) /// Returns a different UTC instant from each call, in order. private sealed class SteppingTimeProvider : TimeProvider { + /// UTC instants returned in order; the last instant repeats when exhausted. private readonly DateTimeOffset[] _instants; + + /// Number of calls completed. private int _index; /// Stores the instants returned by successive clock reads. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandAllocationBaseline.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandAllocationBaseline.cs index 25247607..89bf617c 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandAllocationBaseline.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandAllocationBaseline.cs @@ -26,6 +26,7 @@ public async Task Measure_current_status_line_article_and_lease() await MeasureLeaseAsync(); } + /// Prints per-operation allocation for status parsing and a manual builder baseline. private static void MeasureStatusParse() { var manualBefore = GC.GetAllocatedBytesForCurrentThread(); @@ -63,6 +64,7 @@ private static void MeasureStatusParse() $"after status-parse n={iterations} alloc/op={allocated / iterations} gen0={GC.CollectionCount(0) - g0}"); } + /// Prints allocation and throughput for repeated status-line reads on a cycling wire buffer. private static async Task MeasureStatusLineAsync() { var wire = "220 0 article follows\n"u8.ToArray(); @@ -98,6 +100,7 @@ private static async Task MeasureStatusLineAsync() $"after status-line n={iterations} alloc/op={allocated / iterations} lines/s={iterations / elapsed.TotalSeconds:F0} gen0={GC.CollectionCount(0) - g0} capacity={reader.LineBufferCapacity}"); } + /// Prints caller and process allocation for scripted ARTICLE downloads on an immediate in-memory peer. private static async Task MeasureArticleAsync() { // The pipe-backed scripted peer completes reads on the thread pool. GetResult keeps this @@ -174,6 +177,7 @@ private static void MeasureArticleDebugText() $"after article-debug-text-disabled n={iterations} alloc/op={allocated / iterations} gen0={GC.CollectionCount(0) - g0}"); } + /// Prints allocation for repeated session-pool acquire and dispose cycles. private static async Task MeasureLeaseAsync() { var factory = new ScriptedNntpTransportFactory(); @@ -207,6 +211,7 @@ private static async Task MeasureLeaseAsync() /// Returns the already constructed stream. The article measurement owns its lifetime. private sealed class ImmediateNntpTransportFactory(ImmediateScriptedNntpStream stream) : INntpTransportFactory { + /// public Task ConnectAsync( BackFillerProviderDefinition provider, NntpSessionOptions options, @@ -224,57 +229,76 @@ public Task ConnectAsync( /// private sealed class ImmediateScriptedNntpStream : Stream { + /// Initial server greeting appended in the constructor. private static readonly byte[] Greeting = "200 posting allowed\r\n"u8.ToArray(); + /// CAPABILITIES multiline response. private static readonly byte[] Capabilities = "101 Capability list follows\r\nVERSION 2\r\nREADER\r\n.\r\n"u8.ToArray(); + /// ARTICLE success multiline payload. private static readonly byte[] Article = "220 0 article follows\r\nFrom: a@b\r\n\r\nbody\r\n.\r\n"u8.ToArray(); + /// Default positive single-line response for non-ARTICLE commands. private static readonly byte[] Ready = "201 ready\r\n"u8.ToArray(); + /// Buffered outbound bytes not yet consumed by reads. private readonly byte[] _inbound = new byte[1024]; + /// Accumulates the current client command line before dispatch. private readonly byte[] _command = new byte[256]; + /// Start offset of unread bytes in . private int _inboundOffset; + /// Number of unread bytes in . private int _inboundCount; + /// Length of the partial command accumulated in . private int _commandCount; + /// Buffers the server greeting for the first client read. public ImmediateScriptedNntpStream() { Append(Greeting); } + /// public override bool CanRead => true; + /// public override bool CanSeek => false; + /// public override bool CanWrite => true; + /// public override long Length => throw new NotSupportedException(); + /// public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + /// public override void Flush() { } + /// public override Task FlushAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); return Task.CompletedTask; } + /// public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + /// public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -295,6 +319,7 @@ public override ValueTask ReadAsync(Memory buffer, CancellationToken return new ValueTask(n); } + /// public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -316,12 +341,16 @@ public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationTo return ValueTask.CompletedTask; } + /// public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + /// public override void SetLength(long value) => throw new NotSupportedException(); + /// public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + /// Appends a command fragment into . private void CopyCommand(ReadOnlySpan data) { if (_commandCount + data.Length > _command.Length) @@ -333,6 +362,7 @@ private void CopyCommand(ReadOnlySpan data) _commandCount += data.Length; } + /// Maps a completed command line to a scripted response appended to the inbound buffer. private void DispatchCommand() { var line = _command.AsSpan(0, _commandCount); @@ -360,6 +390,7 @@ private void DispatchCommand() } } + /// Appends response bytes to the read-side buffer, compacting when necessary. private void Append(ReadOnlySpan data) { if (_inboundOffset > 0 && _inboundOffset + _inboundCount + data.Length > _inbound.Length) @@ -378,30 +409,40 @@ private void Append(ReadOnlySpan data) } } + /// Read-only stream that rewinds to the start after each full pass over . private sealed class CyclingByteStream(byte[] data) : Stream { + /// Current read offset; wraps to zero at end of . private int _pos; + /// public override bool CanRead => true; + /// public override bool CanSeek => false; + /// public override bool CanWrite => false; + /// public override long Length => data.Length; + /// public override long Position { get => _pos; set => throw new NotSupportedException(); } + /// public override void Flush() { } + /// public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + /// public override int Read(Span buffer) { if (_pos >= data.Length) @@ -415,16 +456,20 @@ public override int Read(Span buffer) return n; } + /// public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); return new ValueTask(Read(buffer.Span)); } + /// public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + /// public override void SetLength(long value) => throw new NotSupportedException(); + /// public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandBudgetTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandBudgetTests.cs index 19a1457d..dc7f8084 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandBudgetTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpCommandBudgetTests.cs @@ -304,6 +304,9 @@ public async Task Shutdown_drains_a_permit_waiter_still_queued_after_cancellatio Assert.True(sawBlockedWaiter, "shutdown cancellation did not leave a permit waiter queued"); } + /// + /// Runs one pool shutdown while a cancelled acquire and a shutdown acquire are queued; returns whether a blocked permit waiter was observed. + /// private static async Task RunConcurrentShutdownOnceAsync() { var factory = new ScriptedNntpTransportFactory(); @@ -337,6 +340,7 @@ private static async Task RunConcurrentShutdownOnceAsync() return sawBlockedWaiter; } + /// Connects a provider session to one enqueued scripted server with test timeouts. private static async Task ConnectAsync( ScriptedNntpServer server, TimeSpan? commandTimeout = null, @@ -357,6 +361,7 @@ NntpSessionOptions.Default with return session; } + /// Polls until true or a five-second safety timeout. private static async Task WaitUntilAsync(Func predicate) { using var safety = new CancellationTokenSource(TimeSpan.FromSeconds(5)); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpExceptionDiagnosticsTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpExceptionDiagnosticsTests.cs index cfd58fd0..835643f6 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpExceptionDiagnosticsTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpExceptionDiagnosticsTests.cs @@ -111,8 +111,11 @@ public void FormatReason_limits_inner_depth() Assert.DoesNotContain("level-5", reason, StringComparison.Ordinal); } + /// Exception type with a controllable message for blank-message format tests. private sealed class BlankMessageException : Exception { + /// Creates an exception with the given message text. + /// Message passed to the base exception. public BlankMessageException(string message) : base(message) { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs index c975a3a1..d81133a0 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpProviderSessionTests.cs @@ -2282,6 +2282,7 @@ public override void Return(byte[] array, bool clearArray = false) } } + /// Wraps a plaintext transport so the first handshake read can be stalled and cancellation observed. private sealed class StalledPlaintextStream : Stream { /// Transport wrapped for the scripted server. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpSessionPoolTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpSessionPoolTests.cs index 5cd78ac1..04352879 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpSessionPoolTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpSessionPoolTests.cs @@ -1454,6 +1454,17 @@ private static void ObserveExpectedCleanup(Exception exception) exception.Message); } + /// Builds a test session pool with standard timeouts and optional establishment gate. + /// Transport factory supplying scripted servers. + /// Maximum concurrent sessions for the provider definition. + /// Minimum warm sessions (provider definition). + /// KEEPALIVE interval passed to the provider definition. + /// Optional time provider for pool timers. + /// Optional command timeout override. + /// Optional establishment gate wired into new sessions. + /// Provider backbone name on the definition. + /// Optional pool logger. + /// Optional shutdown grace period. internal static NntpSessionPool CreatePool( ScriptedNntpTransportFactory factory, int max, diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpStatusLineTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpStatusLineTests.cs index d7aedc4e..3f33b753 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpStatusLineTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpStatusLineTests.cs @@ -293,10 +293,15 @@ private static void AssertParsed(string wire, int code, string text) Assert.Equal(text, NntpProtocolIo.DecodeStatusText(bytes)); } + /// Builds a reader over an in-memory wire buffer. + /// Bytes fed to the underlying stream. private static NntpStreamReader Reader(byte[] wire) => Reader(new MemoryStream(wire, writable: false)); + /// Builds a reader with the default 1 KiB line scratch capacity. + /// Transport supplying protocol bytes. private static NntpStreamReader Reader(Stream stream) => new(stream, 1024); + /// Reads one line using test defaults for max length and timeout. private static Task ReadAsync( NntpStreamReader reader, int maxBytes = 16 * 1024, @@ -304,30 +309,40 @@ private static Task ReadAsync( CancellationToken cancellationToken = default) => reader.ReadLineAsync(maxBytes, timeout ?? TimeSpan.FromSeconds(2), cancellationToken); + /// Returns successive read chunks exactly once, then EOF. private sealed class ChunkedStream(byte[][] chunks) : Stream { + /// Index of the next chunk to copy on read. private int _index; + /// public override bool CanRead => true; + /// public override bool CanSeek => false; + /// public override bool CanWrite => false; + /// public override long Length => throw new NotSupportedException(); + /// public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + /// public override void Flush() { } + /// public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + /// public override int Read(Span buffer) { if (_index >= chunks.Length) @@ -340,42 +355,57 @@ public override int Read(Span buffer) return chunk.Length; } + /// public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) => new(Read(buffer.Span)); + /// public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + /// public override void SetLength(long value) => throw new NotSupportedException(); + /// public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); } + /// Read-only stream that counts synchronous and asynchronous read calls. private sealed class CountingStream(byte[] data) : Stream { + /// Current offset into . private int _pos; + /// Number of completed read operations. public int Reads { get; private set; } + /// public override bool CanRead => true; + /// public override bool CanSeek => false; + /// public override bool CanWrite => false; + /// public override long Length => data.Length; + /// public override long Position { get => _pos; set => throw new NotSupportedException(); } + /// public override void Flush() { } + /// public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + /// public override int Read(Span buffer) { Reads++; @@ -390,43 +420,60 @@ public override int Read(Span buffer) return n; } + /// public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) => new(Read(buffer.Span)); + /// public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + /// public override void SetLength(long value) => throw new NotSupportedException(); + /// public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); } + /// Blocks the first asynchronous read until completes, then serves . private sealed class HoldThenDataStream(byte[] data, TaskCompletionSource release) : Stream { + /// 0 before the first read enters; 1 afterward. private int _phase; + + /// Current offset into after the hold phase. private int _pos; + /// Signaled when the first read enters the hold path. public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + /// public override bool CanRead => true; + /// public override bool CanSeek => false; + /// public override bool CanWrite => false; + /// public override long Length => data.Length; + /// public override long Position { get => _pos; set => throw new NotSupportedException(); } + /// public override void Flush() { } + /// public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + /// public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) { if (_phase == 0) @@ -449,10 +496,13 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation return n; } + /// public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + /// public override void SetLength(long value) => throw new NotSupportedException(); + /// public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/ProviderSessionEstablishmentGateTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/ProviderSessionEstablishmentGateTests.cs index 3f5ecb44..18a98940 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/ProviderSessionEstablishmentGateTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/ProviderSessionEstablishmentGateTests.cs @@ -416,14 +416,28 @@ private static void AppendPool( /// private sealed class EstablishmentAttemptLog : ILogger { + /// Structured event id for establishment waiting. private const int WaitingEventId = 5412; + + /// Structured event id for establishment started. private const int StartedEventId = 5413; + + /// Structured event id for establishment completed. private const int CompletedEventId = 5414; + + /// Structured event id for establishment cancelled. private const int CancelledEventId = 5415; + + /// Structured event id for establishment failed. private const int FailedEventId = 5416; + /// Protects . private readonly object _gate = new(); + + /// Elapsed time origin for recorded observations. private readonly Stopwatch _clock = Stopwatch.StartNew(); + + /// Structured establishment events captured from debug logging. private readonly List _events = []; /// @@ -498,6 +512,7 @@ internal string Describe() return builder.ToString(); } + /// Reads one named value from structured logging state when present. private static bool TryRead(object? state, string name, out object? value) { if (state is IEnumerable> pairs) @@ -516,12 +531,14 @@ private static bool TryRead(object? state, string name, out object? value) return false; } + /// Redacts AUTHINFO USER and PASS secrets from failure reason text. private static string Scrub(string reason) { reason = ScrubCommand(reason, "AUTHINFO USER "); return ScrubCommand(reason, "AUTHINFO PASS "); } + /// Redacts the token following when not already masked. private static string ScrubCommand(string reason, string marker) { var index = reason.IndexOf(marker, StringComparison.Ordinal); @@ -545,15 +562,29 @@ private static string ScrubCommand(string reason, string marker) return string.Concat(reason.AsSpan(0, valueStart), "***", reason.AsSpan(valueEnd)); } + /// One structured establishment log event for a connection number. + /// Stopwatch elapsed when the event was logged. + /// Structured log event identifier. + /// Provider connection number from log state. + /// Scrubbed failure reason when the event is a failure. private readonly record struct Observation(TimeSpan Elapsed, int EventId, int Connection, string? Reason); + /// Aggregates observations for one connection into a terminal result line. private sealed class Attempt(int connection) { + /// Elapsed time of the first observed event for this connection. private TimeSpan? _origin; + + /// Elapsed time when a terminal result was recorded. private TimeSpan? _terminal; + + /// Terminal result label (ready, cancelled, failed, or incomplete). private string _result = "incomplete"; + + /// Scrubbed failure reason when the attempt failed. private string? _reason; + /// Formats connection, result, optional reason, and elapsed milliseconds. internal string Summarize(TimeSpan now) { var end = _terminal ?? now; @@ -570,6 +601,7 @@ internal string Summarize(TimeSpan now) return text.ToString(); } + /// Records terminal state from one observation; first terminal event wins. internal void Apply(Observation observation) { _origin ??= observation.Elapsed; @@ -587,6 +619,7 @@ internal void Apply(Observation observation) } } + /// Sets terminal fields once; ignores subsequent terminal events. private void SetTerminal(string result, TimeSpan elapsed, string? reason) { if (_terminal is not null) @@ -600,6 +633,7 @@ private void SetTerminal(string result, TimeSpan elapsed, string? reason) } } + /// Rounds elapsed time to a non-negative millisecond count for log lines. private static string FormatMilliseconds(TimeSpan elapsed) { var milliseconds = elapsed.TotalMilliseconds; @@ -611,10 +645,13 @@ private static string FormatMilliseconds(TimeSpan elapsed) return Math.Round(milliseconds).ToString("0", CultureInfo.InvariantCulture); } + /// Empty logging scope returned from . private sealed class NoScope : IDisposable { + /// Shared singleton scope instance. internal static NoScope Instance { get; } = new(); + /// public void Dispose() { } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqBlockedConnectionTimeoutTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqBlockedConnectionTimeoutTests.cs index 1fd32d49..0998cd9a 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqBlockedConnectionTimeoutTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/RabbitMq/RabbitMqBlockedConnectionTimeoutTests.cs @@ -358,6 +358,7 @@ private static async Task WaitForLogEventAsync(CollectingLogger } } + /// Creates a with fast reconnect delays and configurable blocked-connection timeout. private static RabbitMqService CreateService( IRabbitMqConnectionFactory factory, ILogger? logger = null, @@ -381,20 +382,26 @@ private static RabbitMqService CreateService( /// private sealed class RecordingHostsConnectionFactory : IRabbitMqConnectionFactory { + /// Inner fake factory that performs connects. private readonly FakeBackFillerRabbitMqConnectionFactory _inner = new(); + /// Number of connect attempts observed on the inner factory. internal int ConnectCount => _inner.ConnectCount; + /// Last connection created by the inner factory. internal FakeBackFillerRabbitMqConnection? LastConnection => _inner.LastConnection; + /// Optional gate controlling when the inner factory completes a connect. internal TaskCompletionSource? Connected { get => _inner.Connected; set => _inner.Connected = value; } + /// passed to each call. internal List ConnectOptions { get; } = []; + /// public Task ConnectAsync( RabbitMqOptions options, string connectionName, @@ -425,14 +432,18 @@ public override ITimer CreateTimer( return new NoOpTimer(); } + /// Timer that fires immediately in and performs no delayed work. private sealed class NoOpTimer : ITimer { + /// public bool Change(TimeSpan dueTime, TimeSpan period) => true; + /// public void Dispose() { } + /// public ValueTask DisposeAsync() => ValueTask.CompletedTask; } } @@ -442,8 +453,13 @@ public void Dispose() /// private sealed class ControllableTimeProvider : TimeProvider { + /// Protects timer list and elapsed time. private readonly Lock _gate = new(); + + /// Virtual UTC offset from Unix epoch. private TimeSpan _elapsed; + + /// Pending one-shot timers not yet due. private readonly List _timers = []; /// @@ -497,6 +513,7 @@ public override ITimer CreateTimer( return timer; } + /// Registers or cancels a pending timer relative to the virtual clock. private void Schedule(PendingTimer timer, TimeSpan dueTime) { lock (_gate) @@ -520,6 +537,7 @@ private void Schedule(PendingTimer timer, TimeSpan dueTime) } } + /// Marks a timer cancelled and removes it from the pending list. private void Cancel(PendingTimer timer) { lock (_gate) @@ -529,10 +547,13 @@ private void Cancel(PendingTimer timer) } } + /// Virtual timer scheduled by . private sealed class PendingTimer : ITimer { + /// Owning virtual clock. private readonly ControllableTimeProvider _owner; + /// Creates a timer owned by . internal PendingTimer(ControllableTimeProvider owner, TimerCallback callback, object? state) { _owner = owner; @@ -540,16 +561,22 @@ internal PendingTimer(ControllableTimeProvider owner, TimerCallback callback, ob State = state; } + /// Callback invoked when the timer fires. internal TimerCallback Callback { get; } + /// Opaque state passed to . internal object? State { get; } + /// Virtual due time on the owner's clock. internal TimeSpan DueAt { get; set; } + /// True when the timer was cancelled or set to infinite due time. internal bool Cancelled { get; set; } + /// True when the callback should run synchronously from . internal bool FireImmediately { get; set; } + /// public bool Change(TimeSpan dueTime, TimeSpan period) { _ = period; @@ -564,8 +591,10 @@ public bool Change(TimeSpan dueTime, TimeSpan period) return true; } + /// public void Dispose() => _owner.Cancel(this); + /// public ValueTask DisposeAsync() { Dispose(); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Retention/CanonicalArticleBufferRetentionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Retention/CanonicalArticleBufferRetentionTests.cs index 73e6eae7..64100cec 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Retention/CanonicalArticleBufferRetentionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Retention/CanonicalArticleBufferRetentionTests.cs @@ -662,10 +662,19 @@ public Task RetrieveAsync( /// private sealed class BlockingBodyTransport : ICacheListenerTransport { + /// Client command bytes returned from reads. private readonly byte[] _inbound; + + /// Outbound bytes copied from completed writes. private readonly List _written = []; + + /// 1-based write index that waits on . private readonly int _blockOnWrite; + + /// Next read offset into . private int _offset; + + /// Number of completed write calls. private int _writeCalls; /// Feeds as the client command and pauses write number . @@ -725,10 +734,19 @@ public async ValueTask WriteAsync(ReadOnlyMemory buffer, Cancellation /// Accepts the first half of the retained body only after , then the remainder. private sealed class PartialBodyTransport : ICacheListenerTransport { + /// Client command bytes returned from reads. private readonly byte[] _inbound; + + /// Retained body prefix that triggers the partial-write pause. private readonly byte[] _body; + + /// Outbound bytes copied from completed writes. private readonly List _written = []; + + /// Next read offset into . private int _offset; + + /// True after the body write has entered the pause path. private bool _pausedBody; /// Feeds and pauses the write that begins with . @@ -804,8 +822,13 @@ private Span Append(int count) /// Pool that rents a longer array and throws if an array is returned twice. private sealed class TrackingBytePool : ArrayPool { + /// Rented arrays not yet returned, keyed by reference identity. private readonly ConcurrentDictionary _outstanding = new(ReferenceEqualityComparer.Instance); + + /// Total rent count. private int _rents; + + /// Total successful return count. private int _returns; /// Number of calls. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcLifecycleContractTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcLifecycleContractTests.cs index b0968b04..96eb24e3 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcLifecycleContractTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcLifecycleContractTests.cs @@ -7,6 +7,9 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class ArticleWorkRpcLifecycleContractTests { + /// + /// Asserts the publish path registers a correlation in _pending before publish so an immediate reply cannot miss the waiter. + /// [Fact] public void Waiter_registration_before_publish_prevents_immediate_reply_race() { @@ -20,6 +23,9 @@ public void Waiter_registration_before_publish_prevents_immediate_reply_race() Assert.True(pending.ContainsKey(correlationId)); } + /// + /// Two published correlations decrement independent waiter and broker-outstanding counts as replies arrive. + /// [Fact] public void Concurrent_correlations_complete_independent_waiters() { @@ -36,6 +42,7 @@ public void Concurrent_correlations_complete_independent_waiters() Assert.Equal(0, tracker.BrokerOutstandingCount); } + /// Caller timeout clears but leaves broker outstanding at one. [Fact] public void Caller_timeout_does_not_clear_broker_outstanding() { @@ -46,6 +53,7 @@ public void Caller_timeout_does_not_clear_broker_outstanding() Assert.Equal(1, tracker.BrokerOutstandingCount); } + /// Late reply increments without disturbing an still-waiting correlation. [Fact] public void Late_reply_after_timeout_is_accounted_without_corrupting_other_waiters() { @@ -65,6 +73,7 @@ public void Late_reply_after_timeout_is_accounted_without_corrupting_other_waite Assert.Equal(1, tracker.LateRepliesAfterTimeout); } + /// After timeout, broker outstanding remains positive—the value recorded as unresolved when dispose drain expires. [Fact] public void Bounded_drain_reports_unresolved_when_broker_outstanding_remains() { @@ -78,6 +87,7 @@ public void Bounded_drain_reports_unresolved_when_broker_outstanding_remains() Assert.True(unresolved > 0); } + /// omits AMQP Expiration on work requests. [Fact] public void Stress_work_requests_do_not_carry_per_message_expiration() { @@ -88,6 +98,7 @@ public void Stress_work_requests_do_not_carry_per_message_expiration() Assert.True(string.IsNullOrEmpty(props.Expiration)); } + /// and expose reply lifecycle counters. [Fact] public void Stress_summary_surfaces_reply_lifecycle_counters() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTracker.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTracker.cs index a6fc8dd4..680fcfc4 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTracker.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTracker.cs @@ -9,22 +9,33 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// internal sealed class ArticleWorkRpcOutstandingTracker { + /// Callers still awaiting a correlated reply. private int _waiting; + + /// Published correlations not yet settled by reply or publish failure. private int _brokerOutstanding; + + /// Replies received after the caller wait timed out. private int _lateReplies; + /// Number of active caller waiters. public int WaitingCount => Volatile.Read(ref _waiting); + /// Number of in-flight broker correlations (includes post-timeout late replies). public int BrokerOutstandingCount => Volatile.Read(ref _brokerOutstanding); + /// Cumulative late replies counted when runs with callerStillWaiting: false. public int LateRepliesAfterTimeout => Volatile.Read(ref _lateReplies); + /// Increments both waiter and broker-outstanding when a request is published. public void OnPublished() { Interlocked.Increment(ref _waiting); Interlocked.Increment(ref _brokerOutstanding); } + /// Removes the caller waiter without decrementing broker outstanding. + /// Waiting count would go negative. public void OnCallerTimeout() { var waiting = Interlocked.Decrement(ref _waiting); @@ -35,6 +46,9 @@ public void OnCallerTimeout() } } + /// Settles one correlation; increments late-reply counter when the caller already timed out. + /// When , the reply is treated as late after caller timeout. + /// An internal counter would go negative. public void OnReply(bool callerStillWaiting) { if (callerStillWaiting) @@ -59,6 +73,7 @@ public void OnReply(bool callerStillWaiting) } } + /// Rolls back both counters when publish fails before the broker accepts the message. public void OnPublishFailed() { Interlocked.Decrement(ref _waiting); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTrackerTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTrackerTests.cs index c584c791..a4d95675 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTrackerTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkRpcOutstandingTrackerTests.cs @@ -5,6 +5,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class ArticleWorkRpcOutstandingTrackerTests { + /// Caller timeout zeroes waiting count; broker outstanding clears only when the late reply arrives. [Fact] public void Timeout_clears_waiter_but_keeps_broker_outstanding_until_late_reply() { @@ -23,6 +24,7 @@ public void Timeout_clears_waiter_but_keeps_broker_outstanding_until_late_reply( Assert.Equal(1, tracker.LateRepliesAfterTimeout); } + /// OnReply while caller waits clears both counters with zero late replies. [Fact] public void Normal_reply_clears_waiter_and_broker_outstanding() { @@ -34,6 +36,7 @@ public void Normal_reply_clears_waiter_and_broker_outstanding() Assert.Equal(0, tracker.LateRepliesAfterTimeout); } + /// Mixed timeout and replies on two correlations leave correct independent counts. [Fact] public void Concurrent_requests_are_independent() { @@ -56,6 +59,7 @@ public void Concurrent_requests_are_independent() Assert.Equal(1, tracker.LateRepliesAfterTimeout); } + /// OnPublishFailed zeroes waiter and broker outstanding. [Fact] public void Publish_failure_clears_both_counters() { @@ -66,6 +70,7 @@ public void Publish_failure_clears_both_counters() Assert.Equal(0, tracker.BrokerOutstandingCount); } + /// New tracker starts with zero outstanding without requiring dispose. [Fact] public void Dispose_is_safe_when_broker_outstanding_already_zero() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs index 3cc21877..de5587cf 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClient.cs @@ -24,23 +24,53 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class ArticleWorkStressClient : IAsyncDisposable { + /// Owns the exclusive ReplyTo queue; disposed last after drain. private readonly IConnection _connection; + + /// Channel used for BasicPublish to provider fanout exchanges. private readonly IChannel _publishChannel; + + /// Channel that declares ReplyTo and runs the auto-ack reply consumer. private readonly IChannel _consumeChannel; + + /// Exclusive auto-delete queue name passed as AMQP ReplyTo. private readonly string _replyQueue; + + /// Outstanding correlations keyed by AMQP CorrelationId. private readonly ConcurrentDictionary _pending = new(StringComparer.Ordinal); + + /// Per-request waiter timeout configured from harness options. private readonly TimeSpan _replyTimeout; + + /// Maximum time to wait for broker-outstanding replies before closing ReplyTo. private readonly TimeSpan _disposeDrainTimeout; + + /// Non-zero after begins. private int _disposed; + + /// Active caller waiters (mirrors publish-side lifecycle accounting). private int _waitingCount; + + /// Published correlations not yet settled by reply, timeout path, or publish failure. private int _brokerOutstandingCount; + + /// Broker outstanding remaining when dispose drain timed out. private int _unresolvedAtDispose; + + /// Late replies received after caller timeout. private int _lateRepliesAfterTimeout; + + /// Set to 1 after ReplyTo consumer is registered in . private int _replyConsumerReady; + + /// Replies with unknown or stale CorrelationId. private int _unknownCorrelationReplies; + + /// Optional run id propagated on request headers when set. private string? _stressRunId; + /// Private ctor; use to construct a connected client. private ArticleWorkStressClient( IConnection connection, IChannel publishChannel, @@ -292,6 +322,7 @@ await _publishChannel.BasicPublishAsync( } } + /// Maps a backbone label to the fanout exchange/queue entity name backfiller.{backbone}. public static string ComposeProviderEntity(string backbone) { ArgumentException.ThrowIfNullOrWhiteSpace(backbone); @@ -334,6 +365,7 @@ public static BasicProperties CreateRequestProperties( }; } + /// Auto-ack reply handler: completes waiters or counts late/unknown correlations. private async Task OnReplyAsync(object sender, BasicDeliverEventArgs args) { await Task.Yield(); @@ -386,6 +418,7 @@ private async Task OnReplyAsync(object sender, BasicDeliverEventArgs args) } } + /// Rolls back outstanding state and returns a synthetic reply. private ArticleWorkReply PublishBrokerError( Exception ex, string correlationId, @@ -411,6 +444,7 @@ private ArticleWorkReply PublishBrokerError( }; } + /// Completes a still-waiting caller with broker-error when JSON parsing fails. private static void CompleteReplyParseFault( bool wasWaiting, PendingRpc pending, @@ -436,6 +470,7 @@ private static void CompleteReplyParseFault( }); } + /// Removes a pending correlation and decrements lifecycle counters (timeout/cancel/publish fault). private void CompleteBrokerOutstanding(string correlationId, bool clearWaiting) { if (!_pending.TryRemove(correlationId, out var pending)) @@ -451,6 +486,7 @@ private void CompleteBrokerOutstanding(string correlationId, bool clearWaiting) Interlocked.Decrement(ref _brokerOutstandingCount); } + /// UTF-8 JSON body for version-1 Article Work request (no article bytes). private static byte[] BuildRequestJson(Guid requestId, string messageId, string backbone) { var json = @@ -464,6 +500,7 @@ private static byte[] BuildRequestJson(Guid requestId, string messageId, string return Encoding.UTF8.GetBytes(json); } + /// Deserializes compact JSON reply metadata into . private static ArticleWorkReply ParseReply(ReadOnlySpan body) { using var doc = JsonDocument.Parse(body.ToArray()); @@ -513,6 +550,9 @@ private static ArticleWorkReply ParseReply(ReadOnlySpan body) return new ArticleWorkReply(outcome, backbone, messageId, requestId, fqdn, port, articleId, error); } + /// + /// Bounded drain on broker-outstanding work, then closes channels and cancels remaining waiters. + /// public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) != 0) @@ -554,6 +594,7 @@ public async ValueTask DisposeAsync() await _connection.DisposeAsync().ConfigureAwait(false); } + /// Best-effort channel close during dispose; transport faults are ignored. private static async Task CloseIgnoreFaultsAsync(IChannel channel) { try @@ -590,6 +631,7 @@ private static async Task CloseIgnoreFaultsAsync(IChannel channel) } } + /// Best-effort connection close during dispose; transport faults are ignored. private static async Task CloseIgnoreFaultsAsync(IConnection connection) { try @@ -626,8 +668,10 @@ private static async Task CloseIgnoreFaultsAsync(IConnection connection) } } + /// Per-correlation waiter state for one in-flight RPC. private sealed class PendingRpc(TaskCompletionSource waiter) { + /// Task completed when a reply arrives or parse fault is synthesized. public TaskCompletionSource Waiter { get; } = waiter; /// 1 while a caller is awaiting ; 0 after timeout or completion. @@ -658,8 +702,13 @@ public sealed record ArticleWorkReply( /// Milliseconds for the local BasicPublishAsync call. public long PublishMilliseconds { get; init; } + /// timestamp when publish began. public long PublishStartedTimestamp { get; init; } + + /// timestamp when BasicPublish completed. public long PublishCompletedTimestamp { get; init; } + + /// timestamp when the reply was received (or timeout path taken). public long ReplyReceivedTimestamp { get; init; } } } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs index 11548518..8ea11720 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/ArticleWorkStressClientRequestPropertiesTests.cs @@ -11,6 +11,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class ArticleWorkStressClientRequestPropertiesTests { + /// Stress work requests must not set AMQP Expiration. [Fact] public void CreateRequestProperties_omits_expiration_so_ready_work_can_buffer() { @@ -23,6 +24,7 @@ public void CreateRequestProperties_omits_expiration_so_ready_work_can_buffer() Assert.True(string.IsNullOrEmpty(props.Expiration)); } + /// Required AMQP fields and RequestId header are populated for RPC. [Fact] public void CreateRequestProperties_sets_correlation_reply_to_content_type_and_request_id() { @@ -42,6 +44,7 @@ public void CreateRequestProperties_sets_correlation_reply_to_content_type_and_r Assert.True(string.IsNullOrEmpty(props.Expiration)); } + /// Production response TTL constant is not copied onto work requests. [Fact] public void Response_ttl_remains_one_second_and_is_not_applied_to_stress_requests() { @@ -55,6 +58,7 @@ public void Response_ttl_remains_one_second_and_is_not_applied_to_stress_request props.Expiration ?? string.Empty); } + /// Classic and quorum queue arguments must not declare x-message-ttl. [Fact] public void Provider_work_queue_arguments_do_not_declare_x_message_ttl() { @@ -84,6 +88,7 @@ public void Provider_work_queue_arguments_do_not_declare_x_message_ttl() Assert.True(quorum.ContainsKey(BackFillerArticleWorkTopology.DeliveryLimitArgumentName)); } + /// Whitespace correlation or reply-to throws . [Fact] public void CreateRequestProperties_rejects_missing_correlation_or_reply_to() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs index 6df58815..28812f2c 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressBench.cs @@ -12,6 +12,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class BackFillerE2EStressBench { + /// When stress is enabled, discovers ids from the configured corpus and asserts non-zero valid/unique counts. [Fact] public async Task Discover_workload_from_newslogs_when_enabled() { @@ -34,6 +35,7 @@ public async Task Discover_workload_from_newslogs_when_enabled() $"discovery files={stats.FileCount} entries={stats.TotalLogEntries} valid={stats.ValidIds} unique={stats.UniqueIds} malformed={stats.MalformedEntries} dupRate={stats.DuplicateRatePercent:F2}% giganews={stats.ProviderGiganews} eweka={stats.ProviderEweka}"); } + /// Phase A: low concurrency baseline run completes with zero outstanding at end. [Fact] public async Task PhaseA_functional_baseline_when_enabled() { @@ -61,6 +63,7 @@ public async Task PhaseA_functional_baseline_when_enabled() Assert.Equal(0, summary.OutstandingAtEnd); } + /// Phase B: ramps concurrency levels until timeout rate exceeds 25% of started requests. [Fact] public async Task PhaseB_concurrency_ramp_when_enabled() { @@ -104,6 +107,7 @@ public async Task PhaseB_concurrency_ramp_when_enabled() } } + /// Phase C: sustained main workload run logs whether 1 Gbps logical payload target is met. [Fact] public async Task PhaseC_sustained_throughput_when_enabled() { @@ -140,6 +144,7 @@ public async Task PhaseC_sustained_throughput_when_enabled() Console.WriteLine($"phaseC payloadGbps={summary.PayloadGbps:F3} target=1.0 achieved={(summary.PayloadGbps >= 1.0 ? "yes" : "no")}"); } + /// Phase D: repeatable workload soak asserts drain completes or timeouts were recorded. [Fact] public async Task PhaseD_stability_soak_when_enabled() { @@ -175,6 +180,7 @@ public async Task PhaseD_stability_soak_when_enabled() Assert.True(summary.OutstandingAtEnd == 0 || summary.Timeouts > 0); } + /// Provider phase: runs warmup workload separately for Giganews and Eweka backbones. [Fact] public async Task Isolated_backbone_runs_when_enabled() { @@ -517,6 +523,7 @@ await StressQueuePreflight.RequireCleanAsync( } } + /// Appends one tab-separated concurrency sweep row to the report directory table file. private static void WriteSweepRow(string reportDirectory, StressRunSummary summary, int knownMaxSessions) { var path = Path.Join(reportDirectory, "concurrency-sweep-table.txt"); @@ -525,6 +532,7 @@ private static void WriteSweepRow(string reportDirectory, StressRunSummary summa File.AppendAllText(path, line + Environment.NewLine); } + /// Blocks until workload manifest and SHA provenance checks pass. private static void EnsureWorkload(StressHarnessOptions options) { // Synchronous gate used by opt-in benches; provenance check is authoritative. @@ -533,6 +541,7 @@ private static void EnsureWorkload(StressHarnessOptions options) .GetResult(); } + /// Parses phase-B concurrency list from env or returns default ramp steps. private static int[] ParseConcurrencyLevels() { var raw = Environment.GetEnvironmentVariable(StressHarnessOptions.ConcurrencyEnvironmentVariable); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs index a336f817..59d91f8a 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerE2EStressHarness.cs @@ -19,13 +19,16 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class BackFillerE2EStressHarness { + /// Immutable harness configuration for this orchestrator instance. private readonly StressHarnessOptions _options; + /// Captures harness options; does not connect to broker until run methods execute. public BackFillerE2EStressHarness(StressHarnessOptions options) { _options = options ?? throw new ArgumentNullException(nameof(options)); } + /// Discovers Message-IDs from the configured corpus, writes workload files and manifest. public async Task DiscoverAndWriteWorkloadAsync( CancellationToken cancellationToken) { @@ -100,6 +103,7 @@ await File.WriteAllTextAsync( return stats; } + /// Executes one end-to-end stress run with publish, reply, and NNTP delivery workers. public async Task RunAsync( string runName, string backbone, @@ -626,6 +630,7 @@ await File.AppendAllTextAsync( } } + /// Best-effort Management API queue sample; swallows errors so observability never fails the run. private async Task SampleQueueDepthSafeAsync( StressMetrics metrics, string queueName, @@ -688,6 +693,7 @@ private async Task SampleQueueDepthSafeAsync( } } + /// Records publisher-side fault and may cancel the run via abort thresholds. private void RecordPublisherWorkerFault( StressMetrics metrics, WorkItem item, @@ -700,6 +706,7 @@ private void RecordPublisherWorkerFault( MaybeAbort(metrics, runCts); } + /// Records NNTP delivery disconnect fault and may cancel the run. private void RecordDeliveryWorkerFault( StressMetrics metrics, DeliveryItem item, @@ -712,6 +719,7 @@ private void RecordDeliveryWorkerFault( MaybeAbort(metrics, runCts); } + /// Cancels the run token when completed-reply abort thresholds are exceeded. private void MaybeAbort(StressMetrics metrics, CancellationTokenSource runCts) { var summary = metrics.BuildSummary(); @@ -730,6 +738,7 @@ private void MaybeAbort(StressMetrics metrics, CancellationTokenSource runCts) } } + /// Loads non-empty angle-bracket Message-ID lines from a workload file. private static async Task> LoadMessageIdsAsync( string path, CancellationToken cancellationToken) @@ -740,6 +749,7 @@ private static async Task> LoadMessageIdsAsync( .ToArray(); } + /// Replaces invalid path characters in for report filenames. private static string SanitizeFileName(string runName) { var invalid = Path.GetInvalidFileNameChars(); @@ -747,8 +757,10 @@ private static string SanitizeFileName(string runName) return new string(chars); } + /// One scheduled Article Work publish item. private readonly record struct WorkItem(string MessageId, string Backbone, long StartedTimestamp); + /// One NNTP delivery item after a successful work reply. private readonly record struct DeliveryItem( string MessageId, string Backbone, diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs index 8d79c50a..296405f9 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/BackFillerNntpStressClient.cs @@ -11,17 +11,37 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class BackFillerNntpStressClient : IAsyncDisposable { + /// Listener hostname for TLS SNI and connect. private readonly string _host; + + /// Listener TLS port. private readonly int _port; + + /// When true, accepts any server certificate (stress/lab only). private readonly bool _acceptAnyServerCertificate; + + /// Underlying TCP connection. private TcpClient? _tcp; + + /// TLS wrapper over the TCP stream. private SslStream? _ssl; + + /// Active IO stream (TLS when connected). private Stream? _stream; + + /// Scratch buffer for status line reads. private readonly byte[] _lineBuffer = new byte[512]; + + /// Scratch buffer for multiline article body reads. private readonly byte[] _readBuffer = new byte[64 * 1024]; + + /// Reusable buffer for ARTICLE command bytes (512-octet limit). private readonly byte[] _commandBuffer = new byte[512]; + + /// Non-zero after dispose. private int _disposed; + /// Creates a client for the given listener host/port (not yet connected). public BackFillerNntpStressClient(string host, int port, bool acceptAnyServerCertificate = true) { ArgumentException.ThrowIfNullOrWhiteSpace(host); @@ -32,6 +52,7 @@ public BackFillerNntpStressClient(string host, int port, bool acceptAnyServerCer _acceptAnyServerCertificate = acceptAnyServerCertificate; } + /// when TLS stream is open and the TCP socket is connected. public bool IsConnected => _stream is not null && _tcp is { Connected: true }; /// Connects, completes TLS, and consumes the 201 greeting. @@ -159,6 +180,7 @@ public async Task QuitAsync(CancellationToken cancellationToken) } } + /// Disposes TLS and TCP without sending QUIT (use for graceful shutdown). public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) != 0) @@ -181,13 +203,25 @@ public async ValueTask DisposeAsync() /// Pool of reusable TLS NNTP sessions with bounded size. public sealed class BackFillerNntpConnectionPool : IAsyncDisposable { + /// Shared listener host for rented clients. private readonly string _host; + + /// Shared listener port. private readonly int _port; + + /// Maximum concurrently created sessions. private readonly int _capacity; + + /// Idle connected clients available for reuse. private readonly ConcurrentBag _idle = []; + + /// Count of clients currently allocated (idle + in use). private int _created; + + /// Non-zero after pool dispose. private int _disposed; + /// Creates a pool with at most concurrent TLS sessions. public BackFillerNntpConnectionPool(string host, int port, int capacity) { ArgumentException.ThrowIfNullOrWhiteSpace(host); @@ -197,6 +231,7 @@ public BackFillerNntpConnectionPool(string host, int port, int capacity) _capacity = capacity; } + /// Returns a connected client from the idle bag or creates one until capacity is reached. public async Task RentAsync( StressMetrics metrics, CancellationToken cancellationToken) @@ -244,6 +279,7 @@ public async Task RentAsync( } } + /// Recycles a healthy connected client or disposes it and updates metrics. public void Return(BackFillerNntpStressClient client, StressMetrics metrics, bool healthy) { if (_disposed != 0 || !healthy || !client.IsConnected) @@ -257,6 +293,7 @@ public void Return(BackFillerNntpStressClient client, StressMetrics metrics, boo _idle.Add(client); } + /// Gracefully QUITs idle clients then disposes the pool. public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposed, 1) != 0) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs index 585981f0..90cf668e 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/CorpusArticlePreflight.cs @@ -144,16 +144,31 @@ public static void ThrowIfPreflightBlocksPublish(IReadOnlyListClassification for one Message-ID in corpus or live preflight. public enum CorpusIdPreflightKind { + /// Id appears in the discovered corpus set. InCorpus, + + /// Id is canonical but not present in the corpus set. NotInCorpus, + + /// Id failed canonical Message-ID validation. InvalidMessageId, + + /// Live Article Work preflight retrieved the article successfully. Retrieved, + + /// Live preflight completed but article was not found upstream. NotFound, + + /// Live preflight timed out waiting for work reply. Timeout, + + /// Live preflight failed with a non-timeout error. RetrievalFailed, } + /// One preflight row for a selected Message-ID. public sealed record CorpusIdPreflightResult(string MessageId, CorpusIdPreflightKind Kind, string? Detail); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscovery.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscovery.cs index 7b1635f4..8ff8a1a8 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscovery.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscovery.cs @@ -276,6 +276,7 @@ public static WorkloadPartitionCounts WriteWorkloadSets( return new WorkloadPartitionCounts(warmup.Length, main.Length, repeatable.Length, seed); } + /// Copies up to ids starting at . private static string[] TakeSlice(string[] ids, int start, int count) { if (start >= ids.Length || count <= 0) @@ -289,6 +290,7 @@ private static string[] TakeSlice(string[] ids, int start, int count) return slice; } + /// Fisher–Yates shuffle using for reproducible workloads. private static void Shuffle(string[] ids, int seed) { var rng = new Random(seed); @@ -299,6 +301,7 @@ private static void Shuffle(string[] ids, int seed) } } + /// Streams lines from a large log file asynchronously with shared read access. private static async IAsyncEnumerable ReadLinesAsync( string path, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscoveryTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscoveryTests.cs index e189e053..1c14e5b9 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscoveryTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NewsLogMessageIdDiscoveryTests.cs @@ -1,7 +1,9 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress { + /// Regression tests for news-log parsing, discovery, corpus gates, and workload partitioning. public sealed class NewsLogMessageIdDiscoveryTests { + /// Known VectorNNTP log line shapes parse to status, provider, Message-ID, and size. [Theory] [InlineData( "Oct 17 00:00:00.026 + giganews 740121 ?", @@ -35,6 +37,7 @@ public void TryParseLine_accepts_known_formats( Assert.Equal(size, entry.LoggedSize); } + /// Non-log and malformed lines do not parse. [Theory] [InlineData("")] [InlineData("not a log line")] @@ -44,6 +47,7 @@ public void TryParseLine_rejects_malformed(string line) Assert.False(NewsLogMessageIdDiscovery.TryParseLine(line.AsSpan(), out _)); } + /// Discovery stats and unique set reflect deduplication and malformed rejection. [Fact] public async Task DiscoverAsync_streams_and_deduplicates() { @@ -92,6 +96,7 @@ await File.WriteAllLinesAsync( } } + /// Missing directory throws with required corpus path in message. [Fact] public void RequireUsableCorpusDirectory_fails_when_missing() { @@ -102,6 +107,7 @@ public void RequireUsableCorpusDirectory_fails_when_missing() Assert.Contains(StressCorpusPaths.RequiredNewsLogDirectory, ex.Message, StringComparison.Ordinal); } + /// Directory without log files throws. [Fact] public void RequireUsableCorpusDirectory_fails_when_empty() { @@ -119,6 +125,7 @@ public void RequireUsableCorpusDirectory_fails_when_empty() } } + /// Quoted local-parts are rejected and excluded from the unique set. [Fact] public async Task DiscoverAsync_rejects_non_canonical_message_ids_without_fabricating() { @@ -148,6 +155,7 @@ await File.WriteAllLinesAsync( } } + /// All-garbage corpus throws and leaves the unique set empty. [Fact] public async Task DiscoverAsync_fails_when_corpus_yields_no_usable_ids() { @@ -170,6 +178,7 @@ await File.WriteAllLinesAsync( } } + /// Identical seed produces identical main workload file contents. [Fact] public void WriteWorkloadSets_is_reproducible_for_same_seed() { @@ -198,6 +207,7 @@ public void WriteWorkloadSets_is_reproducible_for_same_seed() } } + /// Empty id input throws without generating synthetic ids. [Fact] public void WriteWorkloadSets_fails_on_empty_input_without_synthetic_fallback() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReader.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReader.cs index 7b675d92..6bd2f46f 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReader.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReader.cs @@ -262,51 +262,109 @@ public static StressFailureClass ClassifyFailure( }; } + /// Returns whether is an ASCII decimal digit. private static bool IsDigit(byte value) => value is >= (byte)'0' and <= (byte)'9'; } /// Listener ARTICLE / multiline outcome kinds used by the stress harness. public enum NntpResponseKind { + /// 220/221 style status before multiline body consumption. ArticleFollows, + + /// Complete multiline article received and destuffed. ArticleDelivered, + + /// 430-class article not found on listener. NotFound, + + /// 435-class article temporarily unavailable. NotAvailable, + + /// 411-class newsgroup missing (not used for ARTICLE-by-message-id success path). NoNewsgroup, + + /// 502 unknown command response. UnknownCommand, + + /// 501 syntax error response. SyntaxError, + + /// Unrecognized numeric status code. UnexpectedStatus, + + /// Framing or parsing violation while reading NNTP. ProtocolError, + + /// Peer closed before multiline terminator. Incomplete, + + /// Transport closed unexpectedly. Disconnect, } /// Article Work reply outcomes observable by the stress publisher (wire names). public enum ArticleWorkStressOutcome { + /// Work reply indicates listener routing metadata for delivery. Success, + + /// Upstream/article missing for the requested Message-ID. ArticleNotFound, + + /// Article failed validation/retention rules. InvalidArticle, + + /// Malformed work request rejected by BackFiller. InvalidRequest, + + /// Provider/upstream failure reported in reply JSON. ProviderFailure, + + /// Unexpected BackFiller failure outcome. UnexpectedFailure, + + /// Retention policy rejected the article. RetentionRejected, + + /// Harness waiter timed out (synthetic outcome). NoResponse, + + /// Publish/parse/broker fault on the stress client path. BrokerError, } /// Stable failure buckets for reports. public enum StressFailureClass { + /// Not a failure (success path). None, + + /// Article missing or expired (work or NNTP 430/404 class). MissingOrExpired, + + /// 435 not available. NotAvailable, + + /// Harness or work reply timeout. Timeout, + + /// NNTP transport disconnect during delivery. Disconnect, + + /// Multiline body ended early. IncompleteResponse, + + /// NNTP protocol/framing error. ProtocolError, + + /// BackFiller validation/provider failures. UpstreamOrValidation, + + /// RabbitMQ publish/reply/client faults. RabbitMqError, + + /// Uncategorized failure bucket. Other, } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReaderTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReaderTests.cs index 53084e51..693c1915 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReaderTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/NntpArticleResponseReaderTests.cs @@ -2,8 +2,10 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress { + /// Unit tests for NNTP multiline read, failure classification, and provider entity naming. public sealed class NntpArticleResponseReaderTests { + /// 220 multiline path counts wire vs logical bytes after dot-stuffing. [Fact] public async Task ReadArticleExchange_counts_destuffed_payload_without_framing() { @@ -25,6 +27,7 @@ public async Task ReadArticleExchange_counts_destuffed_payload_without_framing() Assert.Equal(5, result.LogicalArticleBytes); // ".hi\r\n" } + /// 430 response yields zero payload bytes. [Fact] public async Task ReadArticleExchange_not_found_has_zero_payload() { @@ -39,6 +42,7 @@ public async Task ReadArticleExchange_not_found_has_zero_payload() Assert.Equal(0, result.WireArticleBytes); } + /// Body read without terminator yields . [Fact] public async Task ConsumeArticleBody_incomplete_when_peer_closes() { @@ -51,6 +55,7 @@ public async Task ConsumeArticleBody_incomplete_when_peer_closes() Assert.True(result.WireArticleBytes > 0); } + /// ClassifyFailure maps work, NNTP, timeout, and disconnect inputs to . [Theory] [InlineData(ArticleWorkStressOutcome.ArticleNotFound, null, false, false, StressFailureClass.MissingOrExpired)] [InlineData(ArticleWorkStressOutcome.ProviderFailure, null, false, false, StressFailureClass.UpstreamOrValidation)] @@ -71,6 +76,7 @@ public void ClassifyFailure_maps_buckets( Assert.Equal(expected, actual); } + /// ComposeProviderEntity lowercases backbone in the backfiller entity name. [Fact] public void ComposeProviderEntity_normalizes_backbone() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StageLatencyStats.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StageLatencyStats.cs index 0a2dd6fd..678e489c 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StageLatencyStats.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StageLatencyStats.cs @@ -19,6 +19,10 @@ public sealed record StageLatencyStats( double P99Ms, double MaxMs) { + /// Builds stage statistics from unsorted millisecond samples; empty bag yields zeros. + /// Short stage identifier used in reports. + /// Human-readable scope of what the samples measure. + /// Concurrent sample bag populated during the run. public static StageLatencyStats From(string name, string description, ConcurrentBag bag) { if (bag.IsEmpty) @@ -46,6 +50,7 @@ public static StageLatencyStats From(string name, string description, Concurrent values[^1]); } + /// Order-statistic at on a sorted ascending array. private static double At(long[] sorted, double quantile) { var index = (int)Math.Clamp(Math.Ceiling(quantile * sorted.Length) - 1, 0, sorted.Length - 1); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecisionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecisionTests.cs index d5ecc0a2..35a96c03 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecisionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressAbortDecisionTests.cs @@ -5,6 +5,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class StressAbortDecisionTests { + /// High started count with few completions must not trigger abort (DualSoak regression). [Fact] public void ShouldAbort_false_during_flood_ramp_when_few_rpcs_completed() { @@ -23,6 +24,7 @@ public void ShouldAbort_false_during_flood_ramp_when_few_rpcs_completed() abortOnFailureRate: 0.50)); } + /// Abort when completed failure rate exceeds threshold after minimum completions. [Fact] public void ShouldAbort_true_when_completed_failure_rate_exceeds_threshold() { @@ -39,6 +41,7 @@ public void ShouldAbort_true_when_completed_failure_rate_exceeds_threshold() abortOnFailureRate: 0.50)); } + /// Abort when completed timeout rate exceeds threshold. [Fact] public void ShouldAbort_true_when_completed_timeout_rate_exceeds_threshold() { @@ -55,6 +58,7 @@ public void ShouldAbort_true_when_completed_timeout_rate_exceeds_threshold() abortOnFailureRate: 0.50)); } + /// Abort thresholds are ignored until AbortAfterRequests completions exist. [Fact] public void ShouldAbort_false_when_below_completed_threshold() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs index 9dd6fb5e..2a032afe 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressCorpusIntegrityTests.cs @@ -5,6 +5,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class StressCorpusIntegrityTests { + /// RequiredNewsLogDirectory matches the committed absolute VectorNNTP corpus path. [Fact] public void Required_corpus_path_is_the_vectornntp_newslogs_directory() { @@ -13,6 +14,7 @@ public void Required_corpus_path_is_the_vectornntp_newslogs_directory() StressCorpusPaths.RequiredNewsLogDirectory); } + /// When the corpus exists locally, discovery finds log files including news-20261019.log. [Fact] public void Required_corpus_path_resolves_to_usable_logs_on_this_host() { @@ -31,6 +33,7 @@ public void Required_corpus_path_resolves_to_usable_logs_on_this_host() Path.Join(StressCorpusPaths.RequiredNewsLogDirectory, "news-20261019.log"))); } + /// NotInCorpus preflight row causes ThrowIfPreflightBlocksPublish to throw. [Fact] public void Preflight_blocks_publish_on_invalid_or_non_corpus_ids() { @@ -43,6 +46,7 @@ public void Preflight_blocks_publish_on_invalid_or_non_corpus_ids() Assert.Contains("blocked publish", ex.Message, StringComparison.OrdinalIgnoreCase); } + /// All failed live preflight rows block publish. [Fact] public void Preflight_blocks_publish_when_zero_ids_retrieved() { @@ -56,6 +60,7 @@ public void Preflight_blocks_publish_when_zero_ids_retrieved() Assert.Contains("none of the", ex.Message, StringComparison.OrdinalIgnoreCase); } + /// A single Retrieved row allows the publish gate to pass. [Fact] public void Preflight_allows_publish_when_at_least_one_retrieved() { @@ -67,6 +72,7 @@ public void Preflight_allows_publish_when_at_least_one_retrieved() CorpusArticlePreflight.ThrowIfPreflightBlocksPublish(results); } + /// Manifest SHA mismatch fails the provenance guard. [Fact] public async Task Workload_guard_rejects_hash_mismatch() { @@ -114,6 +120,7 @@ await WorkloadCorpusManifest.WriteAsync( } } + /// FromEnvironment uses RequiredNewsLogDirectory when NEWSLOGS env is unset. [Fact] public void FromEnvironment_defaults_to_required_corpus_path() { @@ -145,6 +152,7 @@ public void FromEnvironment_defaults_to_required_corpus_path() } } + /// NEWSLOGS environment override is reflected in options. [Fact] public void FromEnvironment_honours_explicit_newslog_override_for_fixtures() { @@ -174,6 +182,7 @@ public void FromEnvironment_honours_explicit_newslog_override_for_fixtures() } } + /// Workload without manifest fails provenance check. [Fact] public async Task Workload_guard_rejects_missing_manifest_stale_cache() { @@ -201,6 +210,7 @@ public async Task Workload_guard_rejects_missing_manifest_stale_cache() } } + /// Manifest corpus directory must match configured NewsLogDirectory. [Fact] public async Task Workload_guard_rejects_cross_corpus_manifest() { @@ -248,6 +258,7 @@ await WorkloadCorpusManifest.WriteAsync( } } + /// ValidateSelectedAgainstCorpus marks missing ids NotInCorpus without substitution. [Fact] public void Preflight_flags_ids_not_in_corpus_without_replacement() { @@ -259,6 +270,7 @@ public void Preflight_flags_ids_not_in_corpus_without_replacement() Assert.Equal("", results[1].MessageId); } + /// RequireSelectedIdsSubset throws when selection is not a subset of the allowed set. [Fact] public void Selected_ids_must_be_subset_of_corpus() { @@ -267,6 +279,7 @@ public void Selected_ids_must_be_subset_of_corpus() () => StressWorkloadGuard.RequireSelectedIdsSubset(["", ""], allowed)); } + /// Request properties carry correlation, reply-to, RequestId, and StressRunId without Expiration. [Fact] public void CreateRequestProperties_preserves_exact_message_path_headers_without_request_ttl() { @@ -284,6 +297,7 @@ public void CreateRequestProperties_preserves_exact_message_path_headers_without Assert.Equal(runId, props.Headers["StressRunId"]); } + /// WriteWorkloadSets deduplicates ids before partitioning. [Fact] public void Duplicate_ids_are_collapsed_before_workload_write() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs index a9734fd1..bbd559a4 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressHarnessOptions.cs @@ -9,45 +9,97 @@ public sealed class StressHarnessOptions /// Master enable flag. Must be 1 for live stress tests to run. public const string EnableEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS"; + /// Phase selector for opt-in bench methods (A, B, C, D, provider, ALL). public const string PhaseEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_PHASE"; + + /// Publisher concurrency override (scalar or comma-separated list). public const string ConcurrencyEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_CONCURRENCY"; + + /// Run duration override in seconds. public const string DurationSecondsEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_DURATION_SECONDS"; + + /// Provider backbone label (e.g. Giganews, Eweka). public const string BackboneEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_BACKBONE"; + + /// In-flight work request cap override. public const string MaxInFlightEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_MAX_INFLIGHT"; + + /// News-log corpus directory override (fixtures only). public const string NewsLogDirectoryEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_NEWSLOGS"; + + /// Directory for workload id lists and manifest. public const string WorkloadDirectoryEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_WORKLOAD"; + + /// Directory for JSON reports and preflight snapshots. public const string ReportDirectoryEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_REPORT"; + + /// Overrides BackFiller listener host from Article Work success replies. public const string ListenerHostEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_LISTENER_HOST"; + + /// Overrides BackFiller listener TLS port from success replies. public const string ListenerPortEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_LISTENER_PORT"; + + /// Deterministic shuffle seed for workload partitioning. public const string SeedEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_SEED"; + + /// Live Article Work preflight probe count override. public const string ArticlePreflightCountEnvironmentVariable = "VECTORNNTPD_BACKFILLER_E2E_STRESS_ARTICLE_PREFLIGHT_COUNT"; + /// RabbitMQ broker hostname (Management API uses same host on 15672). public required string RabbitMqHost { get; init; } + + /// AMQP port. public int RabbitMqPort { get; init; } = 5672; + + /// Virtual host for Article Work exchanges. public string RabbitMqVirtualHost { get; init; } = "/"; + + /// Broker username (never logged). public required string RabbitMqUsername { get; init; } + + /// Broker password (never logged). public required string RabbitMqPassword { get; init; } + + /// Per-request Article Work reply waiter timeout. public TimeSpan WorkReplyTimeout { get; init; } = TimeSpan.FromSeconds(60); /// /// Bounded wait for broker-outstanding RPCs (including timed-out callers) before disposing /// the exclusive ReplyTo connection. /// public TimeSpan ReplyQueueDisposeDrainTimeout { get; init; } = TimeSpan.FromSeconds(120); + /// NNTP ARTICLE command timeout for listener delivery. public TimeSpan ArticleTimeout { get; init; } = TimeSpan.FromSeconds(30); + + /// Absolute path to news-log corpus used for discovery and provenance. public string NewsLogDirectory { get; init; } = string.Empty; + + /// Directory containing workload id files and corpus manifest. public string WorkloadDirectory { get; init; } = string.Empty; + + /// Output directory for run summaries and preflight captures. public string ReportDirectory { get; init; } = string.Empty; + + /// When set, NNTP client connects here instead of reply FQDN. public string? ListenerHostOverride { get; init; } + + /// When set, NNTP client uses this port instead of reply listener port. public int? ListenerPortOverride { get; init; } + + /// Seed for deterministic workload shuffles. public int Seed { get; init; } = 42; /// /// Number of selected Message-IDs to probe via Article Work before throughput publishing. /// Zero disables live retrieval preflight (membership/canonical checks still apply). /// public int ArticlePreflightCount { get; init; } = 5; + /// Completed-reply timeout rate threshold for early run abort. public double AbortOnTimeoutRate { get; init; } = 0.25; + + /// Completed-reply failure rate threshold for early run abort. public double AbortOnFailureRate { get; init; } = 0.50; + + /// Minimum completed replies before abort thresholds are evaluated. public int AbortAfterRequests { get; init; } = 200; /// Returns when the opt-in environment flag is set to 1. @@ -126,9 +178,11 @@ public static StressHarnessOptions FromEnvironment() }; } + /// Reads phase env var or returns . public static string GetPhaseOrDefault(string defaultPhase) => Environment.GetEnvironmentVariable(PhaseEnvironmentVariable) ?? defaultPhase; + /// Positive concurrency from env or . public static int GetConcurrencyOrDefault(int defaultConcurrency) { if (int.TryParse(Environment.GetEnvironmentVariable(ConcurrencyEnvironmentVariable), out var value) @@ -140,6 +194,7 @@ public static int GetConcurrencyOrDefault(int defaultConcurrency) return defaultConcurrency; } + /// Positive duration seconds from env or . public static int GetDurationSecondsOrDefault(int defaultSeconds) { if (int.TryParse(Environment.GetEnvironmentVariable(DurationSecondsEnvironmentVariable), out var value) @@ -151,6 +206,7 @@ public static int GetDurationSecondsOrDefault(int defaultSeconds) return defaultSeconds; } + /// Positive max-in-flight from env or . public static int GetMaxInFlightOrDefault(int defaultMax) { if (int.TryParse(Environment.GetEnvironmentVariable(MaxInFlightEnvironmentVariable), out var value) @@ -162,6 +218,7 @@ public static int GetMaxInFlightOrDefault(int defaultMax) return defaultMax; } + /// Backbone label from env or . public static string GetBackboneOrDefault(string defaultBackbone) => Environment.GetEnvironmentVariable(BackboneEnvironmentVariable) ?? defaultBackbone; @@ -188,6 +245,7 @@ public object ToRedactedReportObject() => AbortAfterRequests, }; + /// Walks upward from the test base directory to locate VectorNNTPD.slnx. private static string FindRepositoryRoot() { var dir = new DirectoryInfo(AppContext.BaseDirectory); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressMetrics.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressMetrics.cs index 5669a568..ce062d92 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressMetrics.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressMetrics.cs @@ -8,50 +8,133 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// Thread-safe counters and latency samples for one stress run. public sealed class StressMetrics { + /// End-to-end success latency samples (schedule to NNTP complete). private readonly ConcurrentBag _endToEndSuccessLatenciesMs = []; + + /// End-to-end failure latency samples. private readonly ConcurrentBag _endToEndFailureLatenciesMs = []; + + /// Harness enqueue-to-publish-complete samples. private readonly ConcurrentBag _scheduleToPublishMs = []; + + /// BasicPublishAsync duration samples. private readonly ConcurrentBag _publishCallMs = []; + + /// Publish-complete to reply-received samples. private readonly ConcurrentBag _publishToReplyMs = []; + + /// Success reply to ARTICLE-start samples. private readonly ConcurrentBag _replyToNntpStartMs = []; + + /// NNTP ARTICLE multiline consume duration samples. private readonly ConcurrentBag _nntpDeliveryLatenciesMs = []; + + /// Logical article byte sizes for delivered articles. private readonly ConcurrentBag _articleSizeBytes = []; + + /// Counts keyed by name. private readonly ConcurrentDictionary _failureBuckets = new(StringComparer.Ordinal); + + /// Per-backbone counter shards. private readonly ConcurrentDictionary _byBackbone = new(StringComparer.OrdinalIgnoreCase); + + /// Time series of throughput snapshots. private readonly ConcurrentQueue _throughputSamples = new(); + + /// Time series of queue depth snapshots. private readonly ConcurrentQueue _queueDepthSamples = new(); + + /// Total work requests started. private long _requestsStarted; + + /// Article Work replies with success outcome. private long _workSuccess; + + /// Article Work article-not-found outcomes. private long _workNotFound; + + /// Other Article Work failure outcomes. private long _workOtherFailure; + + /// Successful NNTP listener deliveries. private long _articlesDelivered; + + /// Sum of logical payload bytes delivered. private long _logicalPayloadBytes; + + /// Sum of on-wire payload bytes delivered. private long _wirePayloadBytes; + + /// NNTP 430/404 class not-found deliveries. private long _nntpNotFound; + + /// NNTP errors excluding clean not-found. private long _nntpErrors; + + /// Work or delivery timeouts recorded. private long _timeouts; + + /// Disconnect failures during delivery. private long _disconnects; + + /// Current publisher-side outstanding requests. private long _outstanding; + + /// Peak publisher-side outstanding. private long _peakOutstanding; + + /// Current NNTP deliveries in flight. private long _deliveryOutstanding; + + /// Peak NNTP deliveries in flight. private long _peakDeliveryOutstanding; + + /// Running sum of outstanding for mean calculation. private long _outstandingSum; + + /// Number of outstanding samples taken. private long _outstandingSamples; + + /// Active pooled NNTP connections. private long _activeConnections; + + /// Peak active pooled connections. private long _peakConnections; + + /// Incomplete multiline NNTP responses. private long _incompleteResponses; + + /// ReplyTo dispose drain duration in milliseconds. private long _drainMilliseconds; + + /// Late replies after caller timeout (from stress client). private long _lateRepliesAfterTimeout; + + /// Broker outstanding at dispose when drain timed out. private long _unresolvedAtDispose; + + /// Broker outstanding recorded at end of drain window. private long _brokerOutstandingAtDrainEnd; + /// Human-readable run label (phase, backbone, concurrency). public required string RunName { get; init; } + + /// Provider backbone under test, when single-backbone. public string? Backbone { get; init; } + + /// Configured publisher worker count for this run. public int ConfiguredConcurrency { get; init; } + + /// Configured in-flight request cap (semaphore limit). public int ConfiguredMaxInFlight { get; init; } + + /// timestamp marking steady-state measurement start. public long SteadyStateStartTimestamp { get; set; } + + /// timestamp marking steady-state measurement end. public long SteadyStateEndTimestamp { get; set; } + /// Increments started count and current outstanding; updates peak outstanding. public void RequestStarted() { Interlocked.Increment(ref _requestsStarted); @@ -59,28 +142,36 @@ public void RequestStarted() UpdatePeak(ref _peakOutstanding, outstanding); } + /// Decrements publisher-side outstanding when a work request completes. public void RequestFinished() => Interlocked.Decrement(ref _outstanding); + /// Current in-flight work requests (publisher side). public long Outstanding => Interlocked.Read(ref _outstanding); + /// Increments NNTP delivery outstanding and peak. public void DeliveryStarted() { var value = Interlocked.Increment(ref _deliveryOutstanding); UpdatePeak(ref _peakDeliveryOutstanding, value); } + /// Decrements NNTP delivery outstanding after ARTICLE completes. public void DeliveryFinished() => Interlocked.Decrement(ref _deliveryOutstanding); + /// Current in-flight NNTP ARTICLE deliveries. public long DeliveryOutstanding => Interlocked.Read(ref _deliveryOutstanding); + /// Increments active pooled NNTP connections and peak. public void ConnectionOpened() { var active = Interlocked.Increment(ref _activeConnections); UpdatePeak(ref _peakConnections, active); } + /// Decrements active pooled NNTP connections when a session is disposed. public void ConnectionClosed() => Interlocked.Decrement(ref _activeConnections); + /// Records ReplyTo dispose drain duration for the run summary. public void SetDrainMilliseconds(long milliseconds) => Interlocked.Exchange(ref _drainMilliseconds, milliseconds); @@ -98,6 +189,7 @@ public void SetReplyLifecycleCounters( Interlocked.Exchange(ref _brokerOutstandingAtDrainEnd, brokerOutstandingAtDrainEnd); } + /// Increments aggregate and per-backbone Article Work outcome counters. public void RecordWorkReply(string backbone, ArticleWorkStressOutcome outcome) { var counters = _byBackbone.GetOrAdd(backbone, static _ => new BackboneCounters()); @@ -118,6 +210,7 @@ public void RecordWorkReply(string backbone, ArticleWorkStressOutcome outcome) } } + /// Appends harness stage latency samples for later percentile aggregation. public void RecordStageTimings( long scheduleToPublishMs, long publishCallMs, @@ -155,6 +248,7 @@ public void RecordStageTimings( } } + /// Counts a successful listener delivery and payload byte totals. public void RecordArticleDelivered( string backbone, long logicalBytes, @@ -173,6 +267,7 @@ public void RecordArticleDelivered( _ = nntpDeliveryMs; } + /// Increments failure buckets, timeout/disconnect counters, and per-backbone failures. public void RecordFailure( string backbone, StressFailureClass failure, @@ -212,6 +307,7 @@ public void RecordFailure( Interlocked.Increment(ref counters.Failures); } + /// Enqueues a throughput snapshot with current counters and outstanding counts. public void SampleThroughput(long timestamp) { var outstanding = Interlocked.Read(ref _outstanding); @@ -225,11 +321,13 @@ public void SampleThroughput(long timestamp) Interlocked.Read(ref _deliveryOutstanding))); } + /// Enqueues one Management API queue depth observation. public void SampleQueueDepth(long timestamp, string queueName, long messages, long ready, long unacked, long consumers) { _queueDepthSamples.Enqueue(new QueueDepthSample(timestamp, queueName, messages, ready, unacked, consumers)); } + /// Materializes immutable run summary with percentiles, stages, and optional process stats. public StressRunSummary BuildSummary(Process? process = null) { var start = SteadyStateStartTimestamp; @@ -333,6 +431,7 @@ public StressRunSummary BuildSummary(Process? process = null) process?.TotalProcessorTime.TotalSeconds); } + /// Writes JSON to using source-generated context. public void WriteJson(string path, Process? process = null) { var summary = BuildSummary(process); @@ -341,6 +440,7 @@ public void WriteJson(string path, Process? process = null) File.WriteAllText(path, json); } + /// Lock-free max update for peak counters. private static void UpdatePeak(ref long peak, long candidate) { while (true) @@ -358,16 +458,31 @@ private static void UpdatePeak(ref long peak, long candidate) } } + /// Per-backbone counter shard updated concurrently during a run. private sealed class BackboneCounters { + /// Successful work replies for this backbone. public long WorkSuccess; + + /// Article-not-found work replies. public long WorkNotFound; + + /// Other work reply failures. public long WorkOtherFailure; + + /// NNTP deliveries completed. public long ArticlesDelivered; + + /// Logical bytes delivered on this backbone. public long LogicalPayloadBytes; + + /// Wire bytes delivered on this backbone. public long WirePayloadBytes; + + /// Delivery or work failures attributed to this backbone. public long Failures; + /// Volatile read snapshot for summary aggregation. public BackboneSummary Snapshot() => new( Interlocked.Read(ref WorkSuccess), @@ -380,6 +495,7 @@ public BackboneSummary Snapshot() => } } + /// Point-in-time throughput counters sampled during a run. public readonly record struct ThroughputSample( long Timestamp, long ArticlesDelivered, @@ -387,6 +503,7 @@ public readonly record struct ThroughputSample( long Outstanding, long DeliveryOutstanding); + /// One RabbitMQ queue depth observation from the Management API. public readonly record struct QueueDepthSample( long Timestamp, string QueueName, @@ -395,8 +512,10 @@ public readonly record struct QueueDepthSample( long Unacked, long Consumers); + /// Order statistics for a latency or size sample bag. public sealed record Percentiles(long Count, double Mean, double P50, double P90, double P95, double P99, double Max) { + /// Computes percentiles from unsorted samples; empty bag yields zeros. public static Percentiles From(ConcurrentBag bag) { if (bag.IsEmpty) @@ -422,6 +541,7 @@ public static Percentiles From(ConcurrentBag bag) values[^1]); } + /// Quantile on sorted ascending values. private static double At(long[] sorted, double quantile) { var index = (int)Math.Clamp(Math.Ceiling(quantile * sorted.Length) - 1, 0, sorted.Length - 1); @@ -429,6 +549,7 @@ private static double At(long[] sorted, double quantile) } } + /// Aggregate counters for one provider backbone. public sealed record BackboneSummary( long WorkSuccess, long WorkNotFound, @@ -438,6 +559,7 @@ public sealed record BackboneSummary( long WirePayloadBytes, long Failures); + /// Immutable end-of-run report for one stress execution. public sealed record StressRunSummary( string RunName, string? Backbone, @@ -482,6 +604,7 @@ public sealed record StressRunSummary( long? WorkingSetBytes, double? ProcessCpuSeconds) { + /// Single-line console/table row for sweep comparisons. public string FormatTableRow() { return string.Create( @@ -490,6 +613,7 @@ public string FormatTableRow() } } + /// Source-generated JSON serializer context for stress report types. [System.Text.Json.Serialization.JsonSerializable(typeof(StressRunSummary))] [System.Text.Json.Serialization.JsonSerializable(typeof(StageLatencyStats))] [System.Text.Json.Serialization.JsonSerializable(typeof(Percentiles))] diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressPublishGateTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressPublishGateTests.cs index e485c480..3447d6d1 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressPublishGateTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressPublishGateTests.cs @@ -5,6 +5,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class StressPublishGateTests { + /// Missing corpus directory fails RequireUsableCorpusDirectory before workload acceptance. [Fact] public async Task Missing_corpus_fails_before_workload_acceptance() { @@ -15,6 +16,7 @@ public async Task Missing_corpus_fails_before_workload_acceptance() await Task.CompletedTask; } + /// Workload without manifest fails provenance guard without broker I/O. [Fact] public async Task Unprovenanced_workload_is_rejected_without_needing_a_broker() { @@ -42,6 +44,7 @@ await Assert.ThrowsAsync( } } + /// ThrowIfPreflightBlocksPublish throws when no probed id retrieved successfully. [Fact] public void Live_preflight_failure_blocks_throughput_publish_decision() { @@ -54,6 +57,7 @@ public void Live_preflight_failure_blocks_throughput_publish_decision() () => CorpusArticlePreflight.ThrowIfPreflightBlocksPublish(failed)); } + /// Documents ConnectAsync-before-RequestAsync reply consumer readiness contract. [Fact] public void Reply_consumer_readiness_is_required_before_request_publish() { @@ -65,6 +69,7 @@ public void Reply_consumer_readiness_is_required_before_request_publish() StringComparison.Ordinal); } + /// StressRunId header is set when provided; Expiration remains unset. [Fact] public void StressRunId_is_optional_until_set_then_propagated_on_properties() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflight.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflight.cs index 00f14bcc..647c85db 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflight.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflight.cs @@ -160,6 +160,7 @@ await RequireCleanAsync(options, backbone, label + "-timeout", cancellationToken return last!; } + /// Reads a Management API message_stats.*_details.rate value when present; otherwise zero. private static double TryRate(JsonElement root, string detailsName) { if (!root.TryGetProperty("message_stats", out var stats)) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflightTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflightTests.cs index 9b2649b6..2fd7e3f0 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflightTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/StressQueuePreflightTests.cs @@ -3,6 +3,7 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// Deterministic cleanliness thresholds for stress preflight (no broker I/O). public sealed class StressQueuePreflightTests { + /// Default limits accept zero ready and unacked up to two. [Fact] public void IsClean_true_when_ready_and_unacked_within_limits() { @@ -10,6 +11,7 @@ public void IsClean_true_when_ready_and_unacked_within_limits() Assert.True(StressQueuePreflight.IsClean(ready: 0, unacked: 2)); } + /// Non-zero ready or excessive unacked fails cleanliness. [Fact] public void IsClean_false_when_ready_or_unacked_exceed_limits() { @@ -19,6 +21,7 @@ public void IsClean_false_when_ready_or_unacked_exceed_limits() Assert.False(StressQueuePreflight.IsClean(ready: 0, unacked: 25)); } + /// Custom maxReady and maxUnacked bounds are inclusive. [Fact] public void IsClean_respects_custom_limits() { diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Stress/WorkloadCorpusManifest.cs b/tests/VectorNNTPD.BackFillerService.Tests/Stress/WorkloadCorpusManifest.cs index 23670533..e81f7744 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Stress/WorkloadCorpusManifest.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Stress/WorkloadCorpusManifest.cs @@ -9,32 +9,46 @@ namespace VectorNNTPD.BackFillerService.Tests.Stress /// public sealed class WorkloadCorpusManifest { + /// Absolute news-log directory used for discovery (provenance gate). public required string CorpusDirectory { get; init; } + /// Number of *.log files scanned. public required int FileCount { get; init; } + /// Total log lines examined during discovery. public required long LogRecordsExamined { get; init; } + /// Lines that parsed to a canonical Message-ID. public required long ValidParsedIds { get; init; } + /// Lines rejected for non-canonical Message-ID syntax. public required long RejectedInvalidMessageIds { get; init; } + /// Distinct canonical ids after deduplication. public required long UniqueValidIds { get; init; } + /// Count written to warmup workload file. public required int WarmupSelected { get; init; } + /// Count written to main workload file. public required int MainSelected { get; init; } + /// Count written to repeatable workload file. public required int RepeatableSelected { get; init; } + /// Shuffle seed used when partitioning ids. public required int Seed { get; init; } + /// SHA-256 hex of main-message-ids.txt at creation time. public required string MainIdsSha256 { get; init; } + /// UTC timestamp when the manifest was written. public required DateTimeOffset CreatedUtc { get; init; } + /// Optional human notes (e.g. corpus policy reminders). public string? Notes { get; init; } + /// Serializes and writes the manifest JSON into the workload directory. public static async Task WriteAsync( string workloadDirectory, WorkloadCorpusManifest manifest, @@ -51,6 +65,7 @@ await File.WriteAllTextAsync( .ConfigureAwait(false); } + /// Reads manifest JSON when present; otherwise . public static async Task TryReadAsync( string workloadDirectory, CancellationToken cancellationToken) @@ -68,6 +83,7 @@ await File.WriteAllTextAsync( .ConfigureAwait(false); } + /// Computes lowercase hex SHA-256 of a file (used for main workload integrity). public static string ComputeFileSha256(string filePath) { ArgumentException.ThrowIfNullOrWhiteSpace(filePath); diff --git a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/SequenceDbDataReader.cs b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/SequenceDbDataReader.cs index 9a286cbe..1c8a1ec0 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/SequenceDbDataReader.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/TestDoubles/SequenceDbDataReader.cs @@ -8,9 +8,16 @@ namespace VectorNNTPD.BackFillerService.Tests.TestDoubles /// internal sealed class SequenceDbDataReader : DbDataReader { + /// Column names in ordinal order matching each row array. private readonly string[] _names; + + /// Row value arrays; cells may be for SQL NULL. private readonly object[][] _rows; + + /// Current row ordinal; -1 before the first successful . private int _index = -1; + + /// Set by ; subsequent reads throw. private bool _closed; /// Builds a reader over named columns and row value arrays (use for SQL NULL). @@ -179,6 +186,8 @@ public override Task ReadAsync(CancellationToken cancellationToken) /// public override void Close() => _closed = true; + /// Gets the row at or throws when no row is current. + /// No successful has positioned a row. private object[] CurrentRow { get @@ -192,6 +201,8 @@ private object[] CurrentRow } } + /// Throws when the reader was closed via . + /// The reader is closed. private void EnsureOpen() { if (_closed) @@ -200,6 +211,8 @@ private void EnsureOpen() } } + /// Ensures the reader is open and a current row exists before cell access. + /// Closed reader or no current row. private void EnsureOpenRow() { EnsureOpen();