diff --git a/docs/architecture.md b/docs/architecture.md index 12306d69..f5d34438 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,7 +12,7 @@ The worker consumes RabbitMQ Article Work, retrieves articles from upstream NNTP After start, non-null `IApplicationService.Execution` tasks are watched. A fault, a successful completion, or cancellation of that execution raises `UnexpectedServiceTermination`. `BackFillerApplicationHostedService` marks `BackFillerApplicationHealth` unhealthy and calls `IHostApplicationLifetime.StopApplication`. -`SystemdLifecycleNotifier` sends `READY=1` at most once after `IHostApplicationLifetime.ApplicationStarted`. Startup failure does not report ready. If watchdog health is not yet true at that moment, or if a handled transient notify failure occurs, readiness is retried until it succeeds, shutdown begins, or the notifier is disposed. Once a successful `READY=1` is sent, or once shutdown begins, further readiness notifications are suppressed. `SystemdWatchdogService` sends keep-alives only while `IApplicationHealth.IsHealthyForWatchdog` is true. That flag is also false while the published TLS certificate is missing, unreadable, expired, or still inside the renewal window after a renewal attempt that did not replace it. A later successful publication of a usable certificate clears that condition. Unexpected supervised-service termination stays sticky. Readiness is not inferred from DI registration order. +`SystemdLifecycleNotifier` sends `READY=1` at most once after `IHostApplicationLifetime.ApplicationStarted`. Startup failure does not report ready. If watchdog health is not yet true at that moment, or if a handled transient notify failure occurs, readiness is retried until it succeeds, shutdown begins, or the notifier is disposed. Once a successful `READY=1` is sent, or once shutdown begins, further readiness notifications are suppressed. `SystemdWatchdogService` sends keep-alives only while `IApplicationHealth.IsHealthyForWatchdog` is true. That flag is also false while the published TLS certificate is missing, unreadable, or expired. Being inside the renewal window alone does not block keep-alives while the current material remains usable and unexpired; a recoverable renewal failure keeps serving that material. Each health read compares the recorded certificate not-after to the process clock so expiry during a prolonged ACME outage fails closed without waiting for a new observation. A later successful publication of a usable certificate clears the block. Unexpected supervised-service termination stays sticky. Readiness is not inferred from DI registration order. Article Work shutdown uses `BackFillerShutdownOptions`: diff --git a/src/VectorNNTPD.BackFillerService/Acme/AcmeCertificateWatchdog.cs b/src/VectorNNTPD.BackFillerService/Acme/AcmeCertificateWatchdog.cs index f566ff05..ad853466 100644 --- a/src/VectorNNTPD.BackFillerService/Acme/AcmeCertificateWatchdog.cs +++ b/src/VectorNNTPD.BackFillerService/Acme/AcmeCertificateWatchdog.cs @@ -14,54 +14,82 @@ internal sealed class AcmeCertificateWatchdog : IAcmeCertificateObservation /// Logger for the block and restore events. private readonly ILogger _logger; + /// Clock used when re-checking material not-after against a possibly stale status snapshot. + private readonly TimeProvider _time; + /// One while keep-alives are blocked for the certificate. Used so the log is written only on a change. private int _blocked; - /// Retains the health flag and logger. + /// Retains the health flag, logger, and clock. /// Watchdog health updated by each observation. /// Logger for events 1924 and 1925. - public AcmeCertificateWatchdog(BackFillerApplicationHealth health, ILogger logger) + /// + /// Clock for not-after checks. Null uses . + /// Should match the clock injected into . + /// + public AcmeCertificateWatchdog( + BackFillerApplicationHealth health, + ILogger logger, + TimeProvider? time = null) { ArgumentNullException.ThrowIfNull(health); ArgumentNullException.ThrowIfNull(logger); _health = health; _logger = logger; + _time = time ?? TimeProvider.System; } /// /// True when watchdog keep-alives must stop for this certificate snapshot. - /// A usable certificate outside the renewal window does not block. A usable certificate inside the window - /// blocks only when this attempt did not replace it. Missing, unreadable, and expired material always blocks. + /// A usable, unexpired certificate never blocks, including when renewal is due and the last attempt + /// did not replace it. Missing, unreadable, and expired material always blocks. Expiry is evaluated + /// against (or the system clock) so a status whose + /// was computed earlier cannot keep keep-alives alive past . /// - /// Fresh validation snapshot. - /// True when this attempt persisted and published a replacement. + /// Validation snapshot. + /// + /// True when this attempt persisted and published a replacement. Unused while is usable; + /// retained for callers that already report replacement outcomes. + /// + /// + /// Instant used for the not-after check. Null uses . + /// /// True when keep-alives must stop. - internal static bool CertificateRequiresWatchdogBlock(CertificateStatus status, bool renewalReplacedCertificate) + internal static bool CertificateRequiresWatchdogBlock( + CertificateStatus status, + bool renewalReplacedCertificate, + DateTimeOffset? utcNow = null) { ArgumentNullException.ThrowIfNull(status); + _ = renewalReplacedCertificate; if (!status.Usable || status.Material is null) { return true; } - return status.DueForRenewal && !renewalReplacedCertificate; + var now = utcNow ?? DateTimeOffset.UtcNow; + return now >= status.Material.NotAfter; } /// public void OnObserved(CertificateStatus status, bool renewalReplacedCertificate) { - if (CertificateRequiresWatchdogBlock(status, renewalReplacedCertificate)) + var now = _time.GetUtcNow(); + if (CertificateRequiresWatchdogBlock(status, renewalReplacedCertificate, now)) { _health.MarkCertificateUnavailable(); if (Interlocked.Exchange(ref _blocked, 1) == 0) { - AcmeCertificateWatchdogLogMessages.CertificateWatchdogBlocked(_logger, status.Reason); + var reason = status.Usable && status.Material is not null && now >= status.Material.NotAfter + ? "expired" + : status.Reason; + AcmeCertificateWatchdogLogMessages.CertificateWatchdogBlocked(_logger, reason); } return; } - _health.MarkCertificateAvailable(); + _health.MarkCertificateAvailable(status.Material!.NotAfter); if (Interlocked.Exchange(ref _blocked, 0) == 1) { AcmeCertificateWatchdogLogMessages.CertificateWatchdogRestored(_logger); diff --git a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs index 8ce034be..21953de7 100644 --- a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs +++ b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs @@ -132,7 +132,8 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication builder.Services.AddSingleton(static provider => new AcmeCertificateWatchdog( provider.GetRequiredService(), - provider.GetRequiredService>())); + provider.GetRequiredService>(), + provider.GetRequiredService())); VectorNNTPD.Common.Hosting.AcmeCloudflareServiceCollectionExtensions.AddAcmeCloudflareInfrastructure( builder.Services); VectorNNTPD.Common.Hosting.NntpDbServiceCollectionExtensions.AddNntpDbOptions(builder.Services); @@ -159,7 +160,8 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication // Register systemd lifecycle notifier before other hosted services so it is // constructed early and observes ApplicationStarted/Stopping for READY/STOPPING. - builder.Services.AddSingleton(); + builder.Services.AddSingleton(static provider => + new BackFillerApplicationHealth(provider.GetRequiredService())); builder.Services.AddSingleton(static provider => provider.GetRequiredService()); builder.Services.TryAddSingleton(); diff --git a/src/VectorNNTPD.BackFillerService/Hosting/Systemd/BackFillerApplicationHealth.cs b/src/VectorNNTPD.BackFillerService/Hosting/Systemd/BackFillerApplicationHealth.cs index 7874aa02..5836533b 100644 --- a/src/VectorNNTPD.BackFillerService/Hosting/Systemd/BackFillerApplicationHealth.cs +++ b/src/VectorNNTPD.BackFillerService/Hosting/Systemd/BackFillerApplicationHealth.cs @@ -8,9 +8,15 @@ namespace VectorNNTPD.BackFillerService.Hosting.Systemd /// has not begun, no unexpected supervised-service termination has been recorded, and the published TLS /// certificate is not blocking keep-alives. Start, shutdown, and unexpected termination are sticky. /// is cleared by . + /// When a not-after is recorded with , each + /// read compares it to the injected clock so expiry is not sticky + /// across a stale observation snapshot. /// internal sealed class BackFillerApplicationHealth : IApplicationHealth { + /// Clock used to evaluate recorded certificate not-after against current time. + private readonly TimeProvider _time; + /// Non-zero after . private int _started; @@ -21,17 +27,35 @@ internal sealed class BackFillerApplicationHealth : IApplicationHealth private int _unexpectedTermination; /// - /// Non-zero while the published certificate is missing, unusable, expired, or inside the renewal window - /// after a renewal that did not replace it. Cleared by . + /// Non-zero while the published certificate is missing, unusable, or explicitly marked unavailable. + /// Cleared by . Expiry after a recorded not-after is checked + /// separately via . /// private int _certificateUnavailable; + /// + /// UTC ticks of the published certificate's not-after when keep-alives were last cleared. + /// Zero means no expiry bound (tests that mark available without a not-after, or after unavailable). + /// + private long _certificateNotAfterUtcTicks; + + /// Creates health that evaluates certificate expiry with . + /// + /// Clock for not-after checks. Null uses . + /// Production registers ; tests may inject a manual clock. + /// + public BackFillerApplicationHealth(TimeProvider? time = null) + { + _time = time ?? TimeProvider.System; + } + /// public bool IsHealthyForWatchdog => Volatile.Read(ref _started) == 1 && Volatile.Read(ref _stopping) == 0 && Volatile.Read(ref _unexpectedTermination) == 0 - && Volatile.Read(ref _certificateUnavailable) == 0; + && Volatile.Read(ref _certificateUnavailable) == 0 + && CertificateNotExpired(); /// /// Raised after clears the certificate block. @@ -40,16 +64,33 @@ internal sealed class BackFillerApplicationHealth : IApplicationHealth internal event Action? WatchdogHealthMayHaveImproved; /// Stops watchdog keep-alives until . - /// Reversible. Does not mark unexpected termination and does not stop the host by itself. - internal void MarkCertificateUnavailable() => Interlocked.Exchange(ref _certificateUnavailable, 1); + /// + /// Reversible. Does not mark unexpected termination and does not stop the host by itself. + /// Sets the unavailable flag before clearing the recorded not-after so a concurrent + /// read cannot observe a cleared expiry bound while still available + /// (which would briefly treat an expired certificate as healthy). + /// + internal void MarkCertificateUnavailable() + { + Interlocked.Exchange(ref _certificateUnavailable, 1); + Volatile.Write(ref _certificateNotAfterUtcTicks, 0); + } /// Allows watchdog keep-alives again after a usable certificate is published. + /// + /// Exclusive UTC end of validity for the published material. Null leaves no expiry bound on this flag + /// (callers that only exercise start/stop health). Production certificate observations always pass the + /// material not-after so keep-alives stop when the clock reaches expiry without a new observation. + /// /// /// Does not clear or . + /// Writes the not-after before clearing the unavailable flag so a concurrent health read cannot + /// observe available without the replacement expiry bound. /// Raises after clearing the certificate block. /// - internal void MarkCertificateAvailable() + internal void MarkCertificateAvailable(DateTimeOffset? notAfter = null) { + Volatile.Write(ref _certificateNotAfterUtcTicks, notAfter?.UtcTicks ?? 0); Interlocked.Exchange(ref _certificateUnavailable, 0); WatchdogHealthMayHaveImproved?.Invoke(); } @@ -67,5 +108,19 @@ internal void MarkCertificateAvailable() /// Sticky. Callers do not distinguish a faulted execution from one that completed or was canceled. /// internal void MarkUnexpectedTermination() => Interlocked.Exchange(ref _unexpectedTermination, 1); + + /// + /// True when no not-after was recorded, or the clock is still before that instant. + /// + private bool CertificateNotExpired() + { + var notAfterTicks = Volatile.Read(ref _certificateNotAfterUtcTicks); + if (notAfterTicks == 0) + { + return true; + } + + return _time.GetUtcNow().UtcTicks < notAfterTicks; + } } } diff --git a/src/VectorNNTPD.Common/Acme/AcmeLogMessages.cs b/src/VectorNNTPD.Common/Acme/AcmeLogMessages.cs index 854a790d..aeef30e8 100644 --- a/src/VectorNNTPD.Common/Acme/AcmeLogMessages.cs +++ b/src/VectorNNTPD.Common/Acme/AcmeLogMessages.cs @@ -176,12 +176,13 @@ internal static partial void AuthoritativeTxtLookupFailed( /// /// Written when the on-disk certificate is still usable but is true, before replacement issuance. + /// Startup and the renewal loop keep the existing certificate when that issuance fails. /// /// Logger that receives the event. [LoggerMessage( EventId = 1915, Level = LogLevel.Information, - Message = "Existing certificate due for renewal; issuing replacement")] + Message = "Existing certificate due for renewal; attempting replacement")] internal static partial void ExistingCertificateDueForRenewal(ILogger logger); /// diff --git a/src/VectorNNTPD.Common/Acme/CertificateManager.cs b/src/VectorNNTPD.Common/Acme/CertificateManager.cs index 4ebdd0f8..367d5d37 100644 --- a/src/VectorNNTPD.Common/Acme/CertificateManager.cs +++ b/src/VectorNNTPD.Common/Acme/CertificateManager.cs @@ -1,4 +1,8 @@ +using System.Net.Http; +using System.Net.Sockets; +using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; +using VectorNNTPD.Common.Acme.Protocol; namespace VectorNNTPD.Common.Acme { @@ -162,6 +166,11 @@ private CertificateStatus EvaluateExistingCore(Func load, /// Reuses a usable certificate or issues a new one (startup path). /// Cancels the in-process gate wait, the issuance lock wait, and issuance. /// The reused or newly stored material. + /// + /// A usable, unexpired certificate that is due for renewal still starts the process when replacement + /// issuance fails. Cancellation is not treated as a renewal failure. Missing, unreadable, or expired + /// material still requires successful issuance. + /// internal async Task EnsureCertificateAsync(CancellationToken cancellationToken) { await _gate.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -178,15 +187,30 @@ internal async Task EnsureCertificateAsync(CancellationToke return status.Material; } - if (status is { Usable: true, DueForRenewal: true }) + if (status is { Usable: true, Material: not null, DueForRenewal: true }) { AcmeLogMessages.ExistingCertificateDueForRenewal(_logger); - } - else - { - AcmeLogMessages.NoUsableCertificate(_logger, status.Reason); + var prior = status.Material; + var publishedBefore = Volatile.Read(ref _published); + try + { + return await IssueAndPersistAsync(cancellationToken, _pfxPassword).ConfigureAwait(false); + } + catch (Exception ex) when (IsRecoverableIssuanceFailure(ex, cancellationToken)) + { + if (!ReferenceEquals(Volatile.Read(ref _published), publishedBefore)) + { + return Volatile.Read(ref _published)!.Material; + } + + _current = prior; + PublishCredential(prior, _store.CurrentPfxPassword); + AcmeLogMessages.RenewalFailedPreservingExisting(_logger, AcmeFailureSanitizer.Sanitize(ex)); + return prior; + } } + AcmeLogMessages.NoUsableCertificate(_logger, status.Reason); return await IssueAndPersistAsync(cancellationToken, _pfxPassword).ConfigureAwait(false); } finally @@ -233,7 +257,7 @@ internal async Task RenewIfDueAsync( _ = await IssueAndPersistAsync(cancellationToken, issuePassword).ConfigureAwait(false); return true; } - catch (Exception ex) when (prior is not null && ex is not OperationCanceledException) + catch (Exception ex) when (prior is not null && IsRecoverableIssuanceFailure(ex, cancellationToken)) { if (!ReferenceEquals(Volatile.Read(ref _published), publishedBefore)) { @@ -241,6 +265,7 @@ internal async Task RenewIfDueAsync( } _current = prior; + PublishCredential(prior, _store.CurrentPfxPassword); AcmeLogMessages.RenewalFailedPreservingExisting(_logger, AcmeFailureSanitizer.Sanitize(ex)); return false; } @@ -334,6 +359,39 @@ private async Task IssueAndPersistAsync(CancellationToken c } } + /// + /// True when a failed replacement may keep a still-usable prior certificate. + /// Caller or host cancellation always propagates. Local store and configuration failures + /// and unexpected programming errors also propagate. Remote ACME, DNS, and transport failures + /// are recoverable. + /// + /// Exception thrown from . + /// Token that was passed to issuance; when cancelled, cancel exceptions are not recoverable. + /// True when the preserve-on-failure path may run. + private static bool IsRecoverableIssuanceFailure(Exception ex, CancellationToken cancellationToken) + { + if (ex is OperationCanceledException) + { + return !cancellationToken.IsCancellationRequested; + } + + if (ex is AcmeStorageException or AcmeConfigurationException) + { + return false; + } + + if (ex is AcmeException or AcmeCaException) + { + return true; + } + + return ex is HttpRequestException + or IOException + or SocketException + or TimeoutException + or CryptographicException; + } + /// Publishes and together for . /// PFX that can decrypt. /// Password for . Not logged. diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs index 9c136111..8aa16c25 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs @@ -121,60 +121,130 @@ public async Task Transient_renewal_fault_outside_the_window_stays_healthy() } /// - /// Repeated renewal failure inside the safety window blocks watchdog keep-alives. A later replacement clears the block. + /// Renewal failure inside the safety window keeps watchdog keep-alives while the current certificate remains usable. + /// A later successful replacement still leaves the process healthy. /// - /// - /// A second reader holds a certificate lease across both checks and reads watchdog health before releasing it. - /// The reader waits on a signal. It does not spin, so the renewal continuation and other tests can run. - /// [Fact] - public async Task Renewal_failure_inside_the_window_blocks_health_until_a_replacement_is_published() + public async Task Renewal_failure_inside_the_window_keeps_health_while_certificate_remains_usable() { var started = DateTimeOffset.UtcNow; var issuer = new ScriptedIssuer((call, domains, password) => call switch { 1 => CreateMaterial(domains, password, started.AddDays(20)), - 2 => throw new InvalidOperationException("renewal failed"), + 2 => throw new AcmeOrderException("finalize_failed", "ca rejected"), _ => CreateMaterial(domains, password, started.AddDays(90)), }); await using var observed = CreateObservedRenewal(started, issuer, notAfter: null, seedDisk: false); observed.Health.MarkStarted(); await observed.Service.StartAsync(CancellationToken.None); Assert.True(observed.Health.IsHealthyForWatchdog); - using (observed.Publisher.Acquire()) - { - var releaseReader = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var readerHolding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var reader = Task.Run(async () => - { - _ = observed.Health.IsHealthyForWatchdog; - using var current = observed.Publisher.Acquire(); - readerHolding.TrySetResult(); - await releaseReader.Task.ConfigureAwait(false); - _ = observed.Health.IsHealthyForWatchdog; - }); - - await readerHolding.Task.WaitAsync(TimeSpan.FromSeconds(2)); - await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalWaitsStarted == 1, TimeSpan.FromSeconds(2)); - observed.Time.Advance(TimeSpan.FromHours(6)); - await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalChecksCompleted == 1, TimeSpan.FromSeconds(2)); - Assert.False(observed.Health.IsHealthyForWatchdog); - - await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalWaitsStarted == 2, TimeSpan.FromSeconds(2)); - observed.Time.Advance(TimeSpan.FromHours(6)); - await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalChecksCompleted == 2, TimeSpan.FromSeconds(2)); - Assert.True(observed.Health.IsHealthyForWatchdog); - Assert.True(observed.Publisher.IsAvailable); - - releaseReader.TrySetResult(); - await reader.WaitAsync(TimeSpan.FromSeconds(2)); - } - - using (observed.Publisher.Acquire()) - { - } + await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalWaitsStarted == 1, TimeSpan.FromSeconds(2)); + observed.Time.Advance(TimeSpan.FromHours(6)); + await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalChecksCompleted == 1, TimeSpan.FromSeconds(2)); Assert.True(observed.Health.IsHealthyForWatchdog); + Assert.False(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock( + new CertificateStatus( + Usable: true, + DueForRenewal: true, + Material: CreateMaterial(["backfiller01.usenet.ninja"], "x", started.AddDays(20)), + Reason: "renewal_threshold"), + renewalReplacedCertificate: false, + utcNow: observed.Time.GetUtcNow())); + + await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalWaitsStarted == 2, TimeSpan.FromSeconds(2)); + observed.Time.Advance(TimeSpan.FromHours(6)); + await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalChecksCompleted == 2, TimeSpan.FromSeconds(2)); + Assert.True(observed.Health.IsHealthyForWatchdog); + Assert.True(observed.Publisher.IsAvailable); + } + + /// + /// After a usable observation, advancing the clock past not-after fails closed without a new ACME observation. + /// Health is also false at the exact not-after instant (UtcNow >= NotAfter). + /// + [Fact] + public void Watchdog_health_expires_with_the_clock_after_a_usable_observation() + { + var started = DateTimeOffset.UtcNow; + var time = new ManualTimeProvider(started); + var health = new BackFillerApplicationHealth(time); + health.MarkStarted(); + var watchdog = new AcmeCertificateWatchdog( + health, + NullLogger.Instance, + time); + var notAfter = started.AddHours(1); + var material = CreateMaterial(["backfiller01.usenet.ninja"], "x", notAfter); + + watchdog.OnObserved( + new CertificateStatus(Usable: true, DueForRenewal: false, Material: material, Reason: "ok"), + renewalReplacedCertificate: false); + Assert.True(health.IsHealthyForWatchdog); + Assert.False(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock( + new CertificateStatus(Usable: true, DueForRenewal: false, Material: material, Reason: "ok"), + renewalReplacedCertificate: false, + utcNow: time.GetUtcNow())); + + time.Advance(TimeSpan.FromHours(1)); + Assert.Equal(notAfter, time.GetUtcNow()); + Assert.False(health.IsHealthyForWatchdog); + Assert.True(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock( + new CertificateStatus(Usable: true, DueForRenewal: false, Material: material, Reason: "ok"), + renewalReplacedCertificate: false, + utcNow: time.GetUtcNow())); + + time.Advance(TimeSpan.FromHours(1)); + Assert.False(health.IsHealthyForWatchdog); + } + + /// + /// Marking a certificate unavailable while its recorded not-after is already past never reports healthy. + /// + [Fact] + public void Watchdog_health_stays_unhealthy_when_unavailable_after_expiry() + { + var started = DateTimeOffset.UtcNow; + var time = new ManualTimeProvider(started); + var health = new BackFillerApplicationHealth(time); + health.MarkStarted(); + var notAfter = started.AddMinutes(30); + health.MarkCertificateAvailable(notAfter); + Assert.True(health.IsHealthyForWatchdog); + + time.Advance(TimeSpan.FromHours(1)); + Assert.False(health.IsHealthyForWatchdog); + health.MarkCertificateUnavailable(); + Assert.False(health.IsHealthyForWatchdog); + } + + /// + /// A successful replacement observation records the new not-after for subsequent health reads. + /// + [Fact] + public void Watchdog_health_follows_replacement_certificate_not_after() + { + var started = DateTimeOffset.UtcNow; + var time = new ManualTimeProvider(started); + var health = new BackFillerApplicationHealth(time); + health.MarkStarted(); + var watchdog = new AcmeCertificateWatchdog( + health, + NullLogger.Instance, + time); + var shortLived = CreateMaterial(["backfiller01.usenet.ninja"], "x", started.AddHours(1)); + var replacement = CreateMaterial(["backfiller01.usenet.ninja"], "x", started.AddDays(90)); + + watchdog.OnObserved( + new CertificateStatus(Usable: true, DueForRenewal: true, Material: shortLived, Reason: "renewal_threshold"), + renewalReplacedCertificate: false); + Assert.True(health.IsHealthyForWatchdog); + + watchdog.OnObserved( + new CertificateStatus(Usable: true, DueForRenewal: false, Material: replacement, Reason: "ok"), + renewalReplacedCertificate: true); + time.Advance(TimeSpan.FromHours(2)); + Assert.True(health.IsHealthyForWatchdog); } /// @@ -187,7 +257,7 @@ public async Task Expired_certificate_is_not_healthy() var expires = started.AddHours(5); var issuer = new ScriptedIssuer((call, domains, password) => call == 1 ? CreateMaterial(domains, password, expires) - : throw new InvalidOperationException("renewal failed")); + : throw new AcmeOrderException("finalize_failed", "ca rejected")); await using var observed = CreateObservedRenewal(started, issuer, notAfter: null, seedDisk: false); observed.Health.MarkStarted(); await observed.Service.StartAsync(CancellationToken.None); @@ -200,10 +270,12 @@ public async Task Expired_certificate_is_not_healthy() Assert.False(observed.Health.IsHealthyForWatchdog); Assert.True(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock( new CertificateStatus(Usable: false, DueForRenewal: true, Material: null, Reason: "expired"), - renewalReplacedCertificate: false)); + renewalReplacedCertificate: false, + utcNow: observed.Time.GetUtcNow())); Assert.True(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock( new CertificateStatus(Usable: false, DueForRenewal: true, Material: null, Reason: "absent"), - renewalReplacedCertificate: true)); + renewalReplacedCertificate: true, + utcNow: observed.Time.GetUtcNow())); } /// @@ -423,7 +495,7 @@ private static ObservedRenewal CreateObservedRenewal( includeNewsHostname: false); var readiness = new AcmeCertificateReadiness(); var publisher = new TlsCertificateContextProvider(NullLogger.Instance); - var health = new BackFillerApplicationHealth(); + var health = new BackFillerApplicationHealth(time); var service = new AcmeCertificateService( Options.Create(options), new InjectedManagerFactory(manager, Options.Create(options)), @@ -431,7 +503,7 @@ private static ObservedRenewal CreateObservedRenewal( readiness, NullLogger.Instance, time, - new AcmeCertificateWatchdog(health, NullLogger.Instance)); + new AcmeCertificateWatchdog(health, NullLogger.Instance, time)); return new ObservedRenewal(service, health, time, publisher, dir); } diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs index d9288642..1b7ce2e0 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs @@ -62,6 +62,8 @@ public void AddBackFillerHosting_registers_rabbitmq_then_article_work_consumers( Assert.Same( host.Services.GetRequiredService(), host.Services.GetRequiredService()); + Assert.IsType( + host.Services.GetRequiredService()); Assert.NotNull(host.Services.GetRequiredService()); Assert.NotNull(host.Services.GetRequiredService()); Assert.DoesNotContain( diff --git a/tests/VectorNNTPD.Common.Tests/Acme/CertificateManagerLifecycleTests.cs b/tests/VectorNNTPD.Common.Tests/Acme/CertificateManagerLifecycleTests.cs new file mode 100644 index 00000000..7a7c60d1 --- /dev/null +++ b/tests/VectorNNTPD.Common.Tests/Acme/CertificateManagerLifecycleTests.cs @@ -0,0 +1,396 @@ +using Microsoft.Extensions.Logging.Abstractions; +using VectorNNTPD.Common.Acme; +using VectorNNTPD.Common.Configuration; +using VectorNNTPD.Common.Tests.TestDoubles; + +namespace VectorNNTPD.Common.Tests.Acme +{ + /// + /// Startup and renewal behaviour for when a usable certificate is due for renewal. + /// + public sealed class CertificateManagerLifecycleTests + { + private const string Fqdn = "backfiller01.usenet.ninja"; + private const string DirectoryUrl = AcmeCloudflareOptions.DefaultAcmeDirectoryUrl; + + /// + /// A usable certificate outside the renewal window is reused without contacting the issuer. + /// + [Fact] + public async Task EnsureCertificate_reuses_usable_certificate_outside_renewal_window() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(90)); + store.Save(seeded); + var issuer = new CountingIssuer(); + var manager = CreateManager(dir.Path, store, issuer); + + var material = await manager.EnsureCertificateAsync(CancellationToken.None); + + Assert.Equal(0, issuer.Calls); + AssertSameCertificateInstant(seeded.NotAfter, material.NotAfter); + Assert.NotNull(manager.CurrentMaterial); + } + + /// + /// Startup keeps a usable certificate when replacement issuance fails inside the renewal window. + /// + [Fact] + public async Task EnsureCertificate_preserves_usable_certificate_when_due_renewal_fails() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)); + store.Save(seeded); + var issuer = new CountingIssuer + { + ThrowOnIssue = new AcmeOrderException("finalize_failed", "ca rejected"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + var material = await manager.EnsureCertificateAsync(CancellationToken.None); + + Assert.Equal(1, issuer.Calls); + AssertSameCertificateInstant(seeded.NotAfter, material.NotAfter); + AssertSameCertificateInstant(seeded.NotAfter, manager.CurrentMaterial!.NotAfter); + using var tls = manager.CreateTlsCertificate(); + Assert.True(tls.NotAfter.ToUniversalTime() > DateTime.UtcNow); + } + + /// + /// Startup replaces a due certificate when issuance succeeds. + /// + [Fact] + public async Task EnsureCertificate_replaces_usable_certificate_when_due_renewal_succeeds() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)); + store.Save(seeded); + var replacementNotAfter = DateTimeOffset.UtcNow.AddDays(90); + var issuer = new CountingIssuer { FixedNotAfter = replacementNotAfter }; + var manager = CreateManager(dir.Path, store, issuer); + + var material = await manager.EnsureCertificateAsync(CancellationToken.None); + + Assert.Equal(1, issuer.Calls); + AssertSameCertificateInstant(replacementNotAfter, material.NotAfter); + Assert.NotEqual(TruncateToSeconds(seeded.NotAfter), TruncateToSeconds(material.NotAfter)); + } + + /// + /// Missing material still fails closed when issuance fails. + /// + [Fact] + public async Task EnsureCertificate_fails_closed_when_absent_and_issuance_fails() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + var issuer = new CountingIssuer + { + ThrowOnIssue = new AcmeOrderException("finalize_failed", "ca rejected"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + var ex = await Assert.ThrowsAsync( + () => manager.EnsureCertificateAsync(CancellationToken.None)); + Assert.Equal("finalize_failed", ex.Category); + Assert.Equal(1, issuer.Calls); + Assert.Null(manager.CurrentMaterial); + } + + /// + /// Caller or host cancellation during due renewal propagates and is not treated as a renewal outage. + /// + [Fact] + public async Task EnsureCertificate_propagates_caller_cancellation_during_due_renewal() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20))); + using var cts = new CancellationTokenSource(); + var issuer = new CountingIssuer + { + CancelOnIssue = cts, + ThrowOnIssue = new OperationCanceledException(cts.Token), + }; + var manager = CreateManager(dir.Path, store, issuer); + + await Assert.ThrowsAnyAsync( + () => manager.EnsureCertificateAsync(cts.Token)); + Assert.Equal(1, issuer.Calls); + } + + /// + /// A transport cancellation that is not tied to the caller token preserves a usable certificate. + /// + [Fact] + public async Task EnsureCertificate_preserves_usable_certificate_on_transport_cancellation() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)); + store.Save(seeded); + var issuer = new CountingIssuer + { + ThrowOnIssue = new OperationCanceledException("http timeout"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + var material = await manager.EnsureCertificateAsync(CancellationToken.None); + + Assert.Equal(1, issuer.Calls); + AssertSameCertificateInstant(seeded.NotAfter, material.NotAfter); + } + + /// + /// Periodic renewal failure preserves the usable certificate and returns false. + /// + [Fact] + public async Task RenewIfDue_preserves_usable_certificate_when_issuance_fails() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)); + store.Save(seeded); + var issuer = new CountingIssuer + { + ThrowOnIssue = new AcmeOrderException("finalize_failed", "ca rejected"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + Assert.False(await manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal(1, issuer.Calls); + AssertSameCertificateInstant(seeded.NotAfter, manager.CurrentMaterial!.NotAfter); + } + + /// + /// A recoverable HTTP transport failure preserves an existing usable certificate. + /// + [Fact] + public async Task EnsureCertificate_preserves_usable_certificate_on_http_request_failure() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)); + store.Save(seeded); + var issuer = new CountingIssuer + { + ThrowOnIssue = new System.Net.Http.HttpRequestException("acme unreachable"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + var material = await manager.EnsureCertificateAsync(CancellationToken.None); + + Assert.Equal(1, issuer.Calls); + AssertSameCertificateInstant(seeded.NotAfter, material.NotAfter); + } + + /// + /// An unexpected programming error during due renewal is not treated as a renewal outage. + /// + [Fact] + public async Task EnsureCertificate_propagates_unexpected_programming_exception() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20))); + var issuer = new CountingIssuer + { + ThrowOnIssue = new InvalidOperationException("bug in issuer"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + var ex = await Assert.ThrowsAsync( + () => manager.EnsureCertificateAsync(CancellationToken.None)); + Assert.Equal("bug in issuer", ex.Message); + Assert.Equal(1, issuer.Calls); + } + + /// + /// A certificate-store persistence failure is not classified as an ordinary CA renewal outage. + /// + [Fact] + public async Task EnsureCertificate_propagates_storage_exception_during_due_renewal() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20))); + var issuer = new CountingIssuer + { + ThrowOnIssue = new AcmeStorageException("certificate_persist_failed", "disk full"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + var ex = await Assert.ThrowsAsync( + () => manager.EnsureCertificateAsync(CancellationToken.None)); + Assert.Equal("certificate_persist_failed", ex.Category); + Assert.Equal(1, issuer.Calls); + } + + /// + /// Periodic renewal propagates unexpected programming errors instead of preserving. + /// + [Fact] + public async Task RenewIfDue_propagates_unexpected_programming_exception() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20))); + var issuer = new CountingIssuer + { + ThrowOnIssue = new NullReferenceException("unexpected null"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + await Assert.ThrowsAsync( + () => manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal(1, issuer.Calls); + } + + /// + /// Periodic renewal propagates storage failures instead of treating them as CA outages. + /// + [Fact] + public async Task RenewIfDue_propagates_storage_exception() + { + using var dir = new TempStateDir(); + var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false); + var store = new CertificateStore( + dir.Path, + Fqdn, + TestCertificateFactory.Password, + names, + TimeSpan.FromDays(30)); + store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20))); + var issuer = new CountingIssuer + { + ThrowOnIssue = new AcmeStorageException("certificate_persist_failed", "disk full"), + }; + var manager = CreateManager(dir.Path, store, issuer); + + var ex = await Assert.ThrowsAsync( + () => manager.RenewIfDueAsync(CancellationToken.None)); + Assert.Equal("certificate_persist_failed", ex.Category); + } + + private static void AssertSameCertificateInstant(DateTimeOffset expected, DateTimeOffset actual) => + Assert.Equal(TruncateToSeconds(expected), TruncateToSeconds(actual)); + + private static DateTimeOffset TruncateToSeconds(DateTimeOffset value) => + new(value.Year, value.Month, value.Day, value.Hour, value.Minute, value.Second, value.Offset); + + private static CertificateManager CreateManager( + string stateDir, + CertificateStore store, + ICertificateIssuer issuer) => + new( + Fqdn, + stateDir, + DirectoryUrl, + store, + issuer, + TestCertificateFactory.Password, + TimeSpan.FromDays(30), + NullLogger.Instance, + includeNewsHostname: false); + + /// Issuer that counts calls and optionally throws or returns a fixed not-after. + private sealed class CountingIssuer : ICertificateIssuer + { + public int Calls { get; private set; } + + public Exception? ThrowOnIssue { get; set; } + + public DateTimeOffset? FixedNotAfter { get; set; } + + /// When set, cancelled after the call is counted so issuance sees a cancelled caller token. + public CancellationTokenSource? CancelOnIssue { get; set; } + + public Task IssueAsync( + IReadOnlyList domains, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Calls++; + CancelOnIssue?.Cancel(); + if (ThrowOnIssue is not null) + { + throw ThrowOnIssue; + } + + return Task.FromResult( + TestCertificateFactory.CreateMaterial( + domains, + FixedNotAfter ?? DateTimeOffset.UtcNow.AddDays(90))); + } + } + } +}