diff --git a/docs/architecture.md b/docs/architecture.md
index 12306d69..f5d34438 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -12,7 +12,7 @@ The worker consumes RabbitMQ Article Work, retrieves articles from upstream NNTP
After start, non-null `IApplicationService.Execution` tasks are watched. A fault, a successful completion, or cancellation of that execution raises `UnexpectedServiceTermination`. `BackFillerApplicationHostedService` marks `BackFillerApplicationHealth` unhealthy and calls `IHostApplicationLifetime.StopApplication`.
-`SystemdLifecycleNotifier` sends `READY=1` at most once after `IHostApplicationLifetime.ApplicationStarted`. Startup failure does not report ready. If watchdog health is not yet true at that moment, or if a handled transient notify failure occurs, readiness is retried until it succeeds, shutdown begins, or the notifier is disposed. Once a successful `READY=1` is sent, or once shutdown begins, further readiness notifications are suppressed. `SystemdWatchdogService` sends keep-alives only while `IApplicationHealth.IsHealthyForWatchdog` is true. That flag is also false while the published TLS certificate is missing, unreadable, expired, or still inside the renewal window after a renewal attempt that did not replace it. A later successful publication of a usable certificate clears that condition. Unexpected supervised-service termination stays sticky. Readiness is not inferred from DI registration order.
+`SystemdLifecycleNotifier` sends `READY=1` at most once after `IHostApplicationLifetime.ApplicationStarted`. Startup failure does not report ready. If watchdog health is not yet true at that moment, or if a handled transient notify failure occurs, readiness is retried until it succeeds, shutdown begins, or the notifier is disposed. Once a successful `READY=1` is sent, or once shutdown begins, further readiness notifications are suppressed. `SystemdWatchdogService` sends keep-alives only while `IApplicationHealth.IsHealthyForWatchdog` is true. That flag is also false while the published TLS certificate is missing, unreadable, or expired. Being inside the renewal window alone does not block keep-alives while the current material remains usable and unexpired; a recoverable renewal failure keeps serving that material. Each health read compares the recorded certificate not-after to the process clock so expiry during a prolonged ACME outage fails closed without waiting for a new observation. A later successful publication of a usable certificate clears the block. Unexpected supervised-service termination stays sticky. Readiness is not inferred from DI registration order.
Article Work shutdown uses `BackFillerShutdownOptions`:
diff --git a/src/VectorNNTPD.BackFillerService/Acme/AcmeCertificateWatchdog.cs b/src/VectorNNTPD.BackFillerService/Acme/AcmeCertificateWatchdog.cs
index f566ff05..ad853466 100644
--- a/src/VectorNNTPD.BackFillerService/Acme/AcmeCertificateWatchdog.cs
+++ b/src/VectorNNTPD.BackFillerService/Acme/AcmeCertificateWatchdog.cs
@@ -14,54 +14,82 @@ internal sealed class AcmeCertificateWatchdog : IAcmeCertificateObservation
/// Logger for the block and restore events.
private readonly ILogger _logger;
+ /// Clock used when re-checking material not-after against a possibly stale status snapshot.
+ private readonly TimeProvider _time;
+
/// One while keep-alives are blocked for the certificate. Used so the log is written only on a change.
private int _blocked;
- /// Retains the health flag and logger.
+ /// Retains the health flag, logger, and clock.
/// Watchdog health updated by each observation.
/// Logger for events 1924 and 1925.
- public AcmeCertificateWatchdog(BackFillerApplicationHealth health, ILogger logger)
+ ///
+ /// Clock for not-after checks. Null uses .
+ /// Should match the clock injected into .
+ ///
+ public AcmeCertificateWatchdog(
+ BackFillerApplicationHealth health,
+ ILogger logger,
+ TimeProvider? time = null)
{
ArgumentNullException.ThrowIfNull(health);
ArgumentNullException.ThrowIfNull(logger);
_health = health;
_logger = logger;
+ _time = time ?? TimeProvider.System;
}
///
/// True when watchdog keep-alives must stop for this certificate snapshot.
- /// A usable certificate outside the renewal window does not block. A usable certificate inside the window
- /// blocks only when this attempt did not replace it. Missing, unreadable, and expired material always blocks.
+ /// A usable, unexpired certificate never blocks, including when renewal is due and the last attempt
+ /// did not replace it. Missing, unreadable, and expired material always blocks. Expiry is evaluated
+ /// against (or the system clock) so a status whose
+ /// was computed earlier cannot keep keep-alives alive past .
///
- /// Fresh validation snapshot.
- /// True when this attempt persisted and published a replacement.
+ /// Validation snapshot.
+ ///
+ /// True when this attempt persisted and published a replacement. Unused while is usable;
+ /// retained for callers that already report replacement outcomes.
+ ///
+ ///
+ /// Instant used for the not-after check. Null uses .
+ ///
/// True when keep-alives must stop.
- internal static bool CertificateRequiresWatchdogBlock(CertificateStatus status, bool renewalReplacedCertificate)
+ internal static bool CertificateRequiresWatchdogBlock(
+ CertificateStatus status,
+ bool renewalReplacedCertificate,
+ DateTimeOffset? utcNow = null)
{
ArgumentNullException.ThrowIfNull(status);
+ _ = renewalReplacedCertificate;
if (!status.Usable || status.Material is null)
{
return true;
}
- return status.DueForRenewal && !renewalReplacedCertificate;
+ var now = utcNow ?? DateTimeOffset.UtcNow;
+ return now >= status.Material.NotAfter;
}
///
public void OnObserved(CertificateStatus status, bool renewalReplacedCertificate)
{
- if (CertificateRequiresWatchdogBlock(status, renewalReplacedCertificate))
+ var now = _time.GetUtcNow();
+ if (CertificateRequiresWatchdogBlock(status, renewalReplacedCertificate, now))
{
_health.MarkCertificateUnavailable();
if (Interlocked.Exchange(ref _blocked, 1) == 0)
{
- AcmeCertificateWatchdogLogMessages.CertificateWatchdogBlocked(_logger, status.Reason);
+ var reason = status.Usable && status.Material is not null && now >= status.Material.NotAfter
+ ? "expired"
+ : status.Reason;
+ AcmeCertificateWatchdogLogMessages.CertificateWatchdogBlocked(_logger, reason);
}
return;
}
- _health.MarkCertificateAvailable();
+ _health.MarkCertificateAvailable(status.Material!.NotAfter);
if (Interlocked.Exchange(ref _blocked, 0) == 1)
{
AcmeCertificateWatchdogLogMessages.CertificateWatchdogRestored(_logger);
diff --git a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs
index 8ce034be..21953de7 100644
--- a/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs
+++ b/src/VectorNNTPD.BackFillerService/Hosting/BackFillerServiceCollectionExtensions.cs
@@ -132,7 +132,8 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication
builder.Services.AddSingleton(static provider =>
new AcmeCertificateWatchdog(
provider.GetRequiredService(),
- provider.GetRequiredService>()));
+ provider.GetRequiredService>(),
+ provider.GetRequiredService()));
VectorNNTPD.Common.Hosting.AcmeCloudflareServiceCollectionExtensions.AddAcmeCloudflareInfrastructure(
builder.Services);
VectorNNTPD.Common.Hosting.NntpDbServiceCollectionExtensions.AddNntpDbOptions(builder.Services);
@@ -159,7 +160,8 @@ internal static HostApplicationBuilder AddBackFillerHosting(this HostApplication
// Register systemd lifecycle notifier before other hosted services so it is
// constructed early and observes ApplicationStarted/Stopping for READY/STOPPING.
- builder.Services.AddSingleton();
+ builder.Services.AddSingleton(static provider =>
+ new BackFillerApplicationHealth(provider.GetRequiredService()));
builder.Services.AddSingleton(static provider =>
provider.GetRequiredService());
builder.Services.TryAddSingleton();
diff --git a/src/VectorNNTPD.BackFillerService/Hosting/Systemd/BackFillerApplicationHealth.cs b/src/VectorNNTPD.BackFillerService/Hosting/Systemd/BackFillerApplicationHealth.cs
index 7874aa02..5836533b 100644
--- a/src/VectorNNTPD.BackFillerService/Hosting/Systemd/BackFillerApplicationHealth.cs
+++ b/src/VectorNNTPD.BackFillerService/Hosting/Systemd/BackFillerApplicationHealth.cs
@@ -8,9 +8,15 @@ namespace VectorNNTPD.BackFillerService.Hosting.Systemd
/// has not begun, no unexpected supervised-service termination has been recorded, and the published TLS
/// certificate is not blocking keep-alives. Start, shutdown, and unexpected termination are sticky.
/// is cleared by .
+ /// When a not-after is recorded with , each
+ /// read compares it to the injected clock so expiry is not sticky
+ /// across a stale observation snapshot.
///
internal sealed class BackFillerApplicationHealth : IApplicationHealth
{
+ /// Clock used to evaluate recorded certificate not-after against current time.
+ private readonly TimeProvider _time;
+
/// Non-zero after .
private int _started;
@@ -21,17 +27,35 @@ internal sealed class BackFillerApplicationHealth : IApplicationHealth
private int _unexpectedTermination;
///
- /// Non-zero while the published certificate is missing, unusable, expired, or inside the renewal window
- /// after a renewal that did not replace it. Cleared by .
+ /// Non-zero while the published certificate is missing, unusable, or explicitly marked unavailable.
+ /// Cleared by . Expiry after a recorded not-after is checked
+ /// separately via .
///
private int _certificateUnavailable;
+ ///
+ /// UTC ticks of the published certificate's not-after when keep-alives were last cleared.
+ /// Zero means no expiry bound (tests that mark available without a not-after, or after unavailable).
+ ///
+ private long _certificateNotAfterUtcTicks;
+
+ /// Creates health that evaluates certificate expiry with .
+ ///
+ /// Clock for not-after checks. Null uses .
+ /// Production registers ; tests may inject a manual clock.
+ ///
+ public BackFillerApplicationHealth(TimeProvider? time = null)
+ {
+ _time = time ?? TimeProvider.System;
+ }
+
///
public bool IsHealthyForWatchdog =>
Volatile.Read(ref _started) == 1
&& Volatile.Read(ref _stopping) == 0
&& Volatile.Read(ref _unexpectedTermination) == 0
- && Volatile.Read(ref _certificateUnavailable) == 0;
+ && Volatile.Read(ref _certificateUnavailable) == 0
+ && CertificateNotExpired();
///
/// Raised after clears the certificate block.
@@ -40,16 +64,33 @@ internal sealed class BackFillerApplicationHealth : IApplicationHealth
internal event Action? WatchdogHealthMayHaveImproved;
/// Stops watchdog keep-alives until .
- /// Reversible. Does not mark unexpected termination and does not stop the host by itself.
- internal void MarkCertificateUnavailable() => Interlocked.Exchange(ref _certificateUnavailable, 1);
+ ///
+ /// Reversible. Does not mark unexpected termination and does not stop the host by itself.
+ /// Sets the unavailable flag before clearing the recorded not-after so a concurrent
+ /// read cannot observe a cleared expiry bound while still available
+ /// (which would briefly treat an expired certificate as healthy).
+ ///
+ internal void MarkCertificateUnavailable()
+ {
+ Interlocked.Exchange(ref _certificateUnavailable, 1);
+ Volatile.Write(ref _certificateNotAfterUtcTicks, 0);
+ }
/// Allows watchdog keep-alives again after a usable certificate is published.
+ ///
+ /// Exclusive UTC end of validity for the published material. Null leaves no expiry bound on this flag
+ /// (callers that only exercise start/stop health). Production certificate observations always pass the
+ /// material not-after so keep-alives stop when the clock reaches expiry without a new observation.
+ ///
///
/// Does not clear or .
+ /// Writes the not-after before clearing the unavailable flag so a concurrent health read cannot
+ /// observe available without the replacement expiry bound.
/// Raises after clearing the certificate block.
///
- internal void MarkCertificateAvailable()
+ internal void MarkCertificateAvailable(DateTimeOffset? notAfter = null)
{
+ Volatile.Write(ref _certificateNotAfterUtcTicks, notAfter?.UtcTicks ?? 0);
Interlocked.Exchange(ref _certificateUnavailable, 0);
WatchdogHealthMayHaveImproved?.Invoke();
}
@@ -67,5 +108,19 @@ internal void MarkCertificateAvailable()
/// Sticky. Callers do not distinguish a faulted execution from one that completed or was canceled.
///
internal void MarkUnexpectedTermination() => Interlocked.Exchange(ref _unexpectedTermination, 1);
+
+ ///
+ /// True when no not-after was recorded, or the clock is still before that instant.
+ ///
+ private bool CertificateNotExpired()
+ {
+ var notAfterTicks = Volatile.Read(ref _certificateNotAfterUtcTicks);
+ if (notAfterTicks == 0)
+ {
+ return true;
+ }
+
+ return _time.GetUtcNow().UtcTicks < notAfterTicks;
+ }
}
}
diff --git a/src/VectorNNTPD.Common/Acme/AcmeLogMessages.cs b/src/VectorNNTPD.Common/Acme/AcmeLogMessages.cs
index 854a790d..aeef30e8 100644
--- a/src/VectorNNTPD.Common/Acme/AcmeLogMessages.cs
+++ b/src/VectorNNTPD.Common/Acme/AcmeLogMessages.cs
@@ -176,12 +176,13 @@ internal static partial void AuthoritativeTxtLookupFailed(
///
/// Written when the on-disk certificate is still usable but is true, before replacement issuance.
+ /// Startup and the renewal loop keep the existing certificate when that issuance fails.
///
/// Logger that receives the event.
[LoggerMessage(
EventId = 1915,
Level = LogLevel.Information,
- Message = "Existing certificate due for renewal; issuing replacement")]
+ Message = "Existing certificate due for renewal; attempting replacement")]
internal static partial void ExistingCertificateDueForRenewal(ILogger logger);
///
diff --git a/src/VectorNNTPD.Common/Acme/CertificateManager.cs b/src/VectorNNTPD.Common/Acme/CertificateManager.cs
index 4ebdd0f8..367d5d37 100644
--- a/src/VectorNNTPD.Common/Acme/CertificateManager.cs
+++ b/src/VectorNNTPD.Common/Acme/CertificateManager.cs
@@ -1,4 +1,8 @@
+using System.Net.Http;
+using System.Net.Sockets;
+using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
+using VectorNNTPD.Common.Acme.Protocol;
namespace VectorNNTPD.Common.Acme
{
@@ -162,6 +166,11 @@ private CertificateStatus EvaluateExistingCore(Func load,
/// Reuses a usable certificate or issues a new one (startup path).
/// Cancels the in-process gate wait, the issuance lock wait, and issuance.
/// The reused or newly stored material.
+ ///
+ /// A usable, unexpired certificate that is due for renewal still starts the process when replacement
+ /// issuance fails. Cancellation is not treated as a renewal failure. Missing, unreadable, or expired
+ /// material still requires successful issuance.
+ ///
internal async Task EnsureCertificateAsync(CancellationToken cancellationToken)
{
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
@@ -178,15 +187,30 @@ internal async Task EnsureCertificateAsync(CancellationToke
return status.Material;
}
- if (status is { Usable: true, DueForRenewal: true })
+ if (status is { Usable: true, Material: not null, DueForRenewal: true })
{
AcmeLogMessages.ExistingCertificateDueForRenewal(_logger);
- }
- else
- {
- AcmeLogMessages.NoUsableCertificate(_logger, status.Reason);
+ var prior = status.Material;
+ var publishedBefore = Volatile.Read(ref _published);
+ try
+ {
+ return await IssueAndPersistAsync(cancellationToken, _pfxPassword).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (IsRecoverableIssuanceFailure(ex, cancellationToken))
+ {
+ if (!ReferenceEquals(Volatile.Read(ref _published), publishedBefore))
+ {
+ return Volatile.Read(ref _published)!.Material;
+ }
+
+ _current = prior;
+ PublishCredential(prior, _store.CurrentPfxPassword);
+ AcmeLogMessages.RenewalFailedPreservingExisting(_logger, AcmeFailureSanitizer.Sanitize(ex));
+ return prior;
+ }
}
+ AcmeLogMessages.NoUsableCertificate(_logger, status.Reason);
return await IssueAndPersistAsync(cancellationToken, _pfxPassword).ConfigureAwait(false);
}
finally
@@ -233,7 +257,7 @@ internal async Task RenewIfDueAsync(
_ = await IssueAndPersistAsync(cancellationToken, issuePassword).ConfigureAwait(false);
return true;
}
- catch (Exception ex) when (prior is not null && ex is not OperationCanceledException)
+ catch (Exception ex) when (prior is not null && IsRecoverableIssuanceFailure(ex, cancellationToken))
{
if (!ReferenceEquals(Volatile.Read(ref _published), publishedBefore))
{
@@ -241,6 +265,7 @@ internal async Task RenewIfDueAsync(
}
_current = prior;
+ PublishCredential(prior, _store.CurrentPfxPassword);
AcmeLogMessages.RenewalFailedPreservingExisting(_logger, AcmeFailureSanitizer.Sanitize(ex));
return false;
}
@@ -334,6 +359,39 @@ private async Task IssueAndPersistAsync(CancellationToken c
}
}
+ ///
+ /// True when a failed replacement may keep a still-usable prior certificate.
+ /// Caller or host cancellation always propagates. Local store and configuration failures
+ /// and unexpected programming errors also propagate. Remote ACME, DNS, and transport failures
+ /// are recoverable.
+ ///
+ /// Exception thrown from .
+ /// Token that was passed to issuance; when cancelled, cancel exceptions are not recoverable.
+ /// True when the preserve-on-failure path may run.
+ private static bool IsRecoverableIssuanceFailure(Exception ex, CancellationToken cancellationToken)
+ {
+ if (ex is OperationCanceledException)
+ {
+ return !cancellationToken.IsCancellationRequested;
+ }
+
+ if (ex is AcmeStorageException or AcmeConfigurationException)
+ {
+ return false;
+ }
+
+ if (ex is AcmeException or AcmeCaException)
+ {
+ return true;
+ }
+
+ return ex is HttpRequestException
+ or IOException
+ or SocketException
+ or TimeoutException
+ or CryptographicException;
+ }
+
/// Publishes and together for .
/// PFX that can decrypt.
/// Password for . Not logged.
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs
index 9c136111..8aa16c25 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Acme/AcmeCertificateApplicationServiceTests.cs
@@ -121,60 +121,130 @@ public async Task Transient_renewal_fault_outside_the_window_stays_healthy()
}
///
- /// Repeated renewal failure inside the safety window blocks watchdog keep-alives. A later replacement clears the block.
+ /// Renewal failure inside the safety window keeps watchdog keep-alives while the current certificate remains usable.
+ /// A later successful replacement still leaves the process healthy.
///
- ///
- /// A second reader holds a certificate lease across both checks and reads watchdog health before releasing it.
- /// The reader waits on a signal. It does not spin, so the renewal continuation and other tests can run.
- ///
[Fact]
- public async Task Renewal_failure_inside_the_window_blocks_health_until_a_replacement_is_published()
+ public async Task Renewal_failure_inside_the_window_keeps_health_while_certificate_remains_usable()
{
var started = DateTimeOffset.UtcNow;
var issuer = new ScriptedIssuer((call, domains, password) => call switch
{
1 => CreateMaterial(domains, password, started.AddDays(20)),
- 2 => throw new InvalidOperationException("renewal failed"),
+ 2 => throw new AcmeOrderException("finalize_failed", "ca rejected"),
_ => CreateMaterial(domains, password, started.AddDays(90)),
});
await using var observed = CreateObservedRenewal(started, issuer, notAfter: null, seedDisk: false);
observed.Health.MarkStarted();
await observed.Service.StartAsync(CancellationToken.None);
Assert.True(observed.Health.IsHealthyForWatchdog);
- using (observed.Publisher.Acquire())
- {
- var releaseReader = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
- var readerHolding = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
- var reader = Task.Run(async () =>
- {
- _ = observed.Health.IsHealthyForWatchdog;
- using var current = observed.Publisher.Acquire();
- readerHolding.TrySetResult();
- await releaseReader.Task.ConfigureAwait(false);
- _ = observed.Health.IsHealthyForWatchdog;
- });
-
- await readerHolding.Task.WaitAsync(TimeSpan.FromSeconds(2));
- await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalWaitsStarted == 1, TimeSpan.FromSeconds(2));
- observed.Time.Advance(TimeSpan.FromHours(6));
- await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalChecksCompleted == 1, TimeSpan.FromSeconds(2));
- Assert.False(observed.Health.IsHealthyForWatchdog);
-
- await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalWaitsStarted == 2, TimeSpan.FromSeconds(2));
- observed.Time.Advance(TimeSpan.FromHours(6));
- await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalChecksCompleted == 2, TimeSpan.FromSeconds(2));
- Assert.True(observed.Health.IsHealthyForWatchdog);
- Assert.True(observed.Publisher.IsAvailable);
-
- releaseReader.TrySetResult();
- await reader.WaitAsync(TimeSpan.FromSeconds(2));
- }
-
- using (observed.Publisher.Acquire())
- {
- }
+ await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalWaitsStarted == 1, TimeSpan.FromSeconds(2));
+ observed.Time.Advance(TimeSpan.FromHours(6));
+ await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalChecksCompleted == 1, TimeSpan.FromSeconds(2));
Assert.True(observed.Health.IsHealthyForWatchdog);
+ Assert.False(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock(
+ new CertificateStatus(
+ Usable: true,
+ DueForRenewal: true,
+ Material: CreateMaterial(["backfiller01.usenet.ninja"], "x", started.AddDays(20)),
+ Reason: "renewal_threshold"),
+ renewalReplacedCertificate: false,
+ utcNow: observed.Time.GetUtcNow()));
+
+ await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalWaitsStarted == 2, TimeSpan.FromSeconds(2));
+ observed.Time.Advance(TimeSpan.FromHours(6));
+ await ArticleWorkTestDeliveries.WaitUntilAsync(() => observed.Service.RenewalChecksCompleted == 2, TimeSpan.FromSeconds(2));
+ Assert.True(observed.Health.IsHealthyForWatchdog);
+ Assert.True(observed.Publisher.IsAvailable);
+ }
+
+ ///
+ /// After a usable observation, advancing the clock past not-after fails closed without a new ACME observation.
+ /// Health is also false at the exact not-after instant (UtcNow >= NotAfter).
+ ///
+ [Fact]
+ public void Watchdog_health_expires_with_the_clock_after_a_usable_observation()
+ {
+ var started = DateTimeOffset.UtcNow;
+ var time = new ManualTimeProvider(started);
+ var health = new BackFillerApplicationHealth(time);
+ health.MarkStarted();
+ var watchdog = new AcmeCertificateWatchdog(
+ health,
+ NullLogger.Instance,
+ time);
+ var notAfter = started.AddHours(1);
+ var material = CreateMaterial(["backfiller01.usenet.ninja"], "x", notAfter);
+
+ watchdog.OnObserved(
+ new CertificateStatus(Usable: true, DueForRenewal: false, Material: material, Reason: "ok"),
+ renewalReplacedCertificate: false);
+ Assert.True(health.IsHealthyForWatchdog);
+ Assert.False(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock(
+ new CertificateStatus(Usable: true, DueForRenewal: false, Material: material, Reason: "ok"),
+ renewalReplacedCertificate: false,
+ utcNow: time.GetUtcNow()));
+
+ time.Advance(TimeSpan.FromHours(1));
+ Assert.Equal(notAfter, time.GetUtcNow());
+ Assert.False(health.IsHealthyForWatchdog);
+ Assert.True(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock(
+ new CertificateStatus(Usable: true, DueForRenewal: false, Material: material, Reason: "ok"),
+ renewalReplacedCertificate: false,
+ utcNow: time.GetUtcNow()));
+
+ time.Advance(TimeSpan.FromHours(1));
+ Assert.False(health.IsHealthyForWatchdog);
+ }
+
+ ///
+ /// Marking a certificate unavailable while its recorded not-after is already past never reports healthy.
+ ///
+ [Fact]
+ public void Watchdog_health_stays_unhealthy_when_unavailable_after_expiry()
+ {
+ var started = DateTimeOffset.UtcNow;
+ var time = new ManualTimeProvider(started);
+ var health = new BackFillerApplicationHealth(time);
+ health.MarkStarted();
+ var notAfter = started.AddMinutes(30);
+ health.MarkCertificateAvailable(notAfter);
+ Assert.True(health.IsHealthyForWatchdog);
+
+ time.Advance(TimeSpan.FromHours(1));
+ Assert.False(health.IsHealthyForWatchdog);
+ health.MarkCertificateUnavailable();
+ Assert.False(health.IsHealthyForWatchdog);
+ }
+
+ ///
+ /// A successful replacement observation records the new not-after for subsequent health reads.
+ ///
+ [Fact]
+ public void Watchdog_health_follows_replacement_certificate_not_after()
+ {
+ var started = DateTimeOffset.UtcNow;
+ var time = new ManualTimeProvider(started);
+ var health = new BackFillerApplicationHealth(time);
+ health.MarkStarted();
+ var watchdog = new AcmeCertificateWatchdog(
+ health,
+ NullLogger.Instance,
+ time);
+ var shortLived = CreateMaterial(["backfiller01.usenet.ninja"], "x", started.AddHours(1));
+ var replacement = CreateMaterial(["backfiller01.usenet.ninja"], "x", started.AddDays(90));
+
+ watchdog.OnObserved(
+ new CertificateStatus(Usable: true, DueForRenewal: true, Material: shortLived, Reason: "renewal_threshold"),
+ renewalReplacedCertificate: false);
+ Assert.True(health.IsHealthyForWatchdog);
+
+ watchdog.OnObserved(
+ new CertificateStatus(Usable: true, DueForRenewal: false, Material: replacement, Reason: "ok"),
+ renewalReplacedCertificate: true);
+ time.Advance(TimeSpan.FromHours(2));
+ Assert.True(health.IsHealthyForWatchdog);
}
///
@@ -187,7 +257,7 @@ public async Task Expired_certificate_is_not_healthy()
var expires = started.AddHours(5);
var issuer = new ScriptedIssuer((call, domains, password) => call == 1
? CreateMaterial(domains, password, expires)
- : throw new InvalidOperationException("renewal failed"));
+ : throw new AcmeOrderException("finalize_failed", "ca rejected"));
await using var observed = CreateObservedRenewal(started, issuer, notAfter: null, seedDisk: false);
observed.Health.MarkStarted();
await observed.Service.StartAsync(CancellationToken.None);
@@ -200,10 +270,12 @@ public async Task Expired_certificate_is_not_healthy()
Assert.False(observed.Health.IsHealthyForWatchdog);
Assert.True(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock(
new CertificateStatus(Usable: false, DueForRenewal: true, Material: null, Reason: "expired"),
- renewalReplacedCertificate: false));
+ renewalReplacedCertificate: false,
+ utcNow: observed.Time.GetUtcNow()));
Assert.True(AcmeCertificateWatchdog.CertificateRequiresWatchdogBlock(
new CertificateStatus(Usable: false, DueForRenewal: true, Material: null, Reason: "absent"),
- renewalReplacedCertificate: true));
+ renewalReplacedCertificate: true,
+ utcNow: observed.Time.GetUtcNow()));
}
///
@@ -423,7 +495,7 @@ private static ObservedRenewal CreateObservedRenewal(
includeNewsHostname: false);
var readiness = new AcmeCertificateReadiness();
var publisher = new TlsCertificateContextProvider(NullLogger.Instance);
- var health = new BackFillerApplicationHealth();
+ var health = new BackFillerApplicationHealth(time);
var service = new AcmeCertificateService(
Options.Create(options),
new InjectedManagerFactory(manager, Options.Create(options)),
@@ -431,7 +503,7 @@ private static ObservedRenewal CreateObservedRenewal(
readiness,
NullLogger.Instance,
time,
- new AcmeCertificateWatchdog(health, NullLogger.Instance));
+ new AcmeCertificateWatchdog(health, NullLogger.Instance, time));
return new ObservedRenewal(service, health, time, publisher, dir);
}
diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs
index d9288642..1b7ce2e0 100644
--- a/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs
+++ b/tests/VectorNNTPD.BackFillerService.Tests/Hosting/BackFillerHostCompositionTests.cs
@@ -62,6 +62,8 @@ public void AddBackFillerHosting_registers_rabbitmq_then_article_work_consumers(
Assert.Same(
host.Services.GetRequiredService(),
host.Services.GetRequiredService());
+ Assert.IsType(
+ host.Services.GetRequiredService());
Assert.NotNull(host.Services.GetRequiredService());
Assert.NotNull(host.Services.GetRequiredService());
Assert.DoesNotContain(
diff --git a/tests/VectorNNTPD.Common.Tests/Acme/CertificateManagerLifecycleTests.cs b/tests/VectorNNTPD.Common.Tests/Acme/CertificateManagerLifecycleTests.cs
new file mode 100644
index 00000000..7a7c60d1
--- /dev/null
+++ b/tests/VectorNNTPD.Common.Tests/Acme/CertificateManagerLifecycleTests.cs
@@ -0,0 +1,396 @@
+using Microsoft.Extensions.Logging.Abstractions;
+using VectorNNTPD.Common.Acme;
+using VectorNNTPD.Common.Configuration;
+using VectorNNTPD.Common.Tests.TestDoubles;
+
+namespace VectorNNTPD.Common.Tests.Acme
+{
+ ///
+ /// Startup and renewal behaviour for when a usable certificate is due for renewal.
+ ///
+ public sealed class CertificateManagerLifecycleTests
+ {
+ private const string Fqdn = "backfiller01.usenet.ninja";
+ private const string DirectoryUrl = AcmeCloudflareOptions.DefaultAcmeDirectoryUrl;
+
+ ///
+ /// A usable certificate outside the renewal window is reused without contacting the issuer.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_reuses_usable_certificate_outside_renewal_window()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(90));
+ store.Save(seeded);
+ var issuer = new CountingIssuer();
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var material = await manager.EnsureCertificateAsync(CancellationToken.None);
+
+ Assert.Equal(0, issuer.Calls);
+ AssertSameCertificateInstant(seeded.NotAfter, material.NotAfter);
+ Assert.NotNull(manager.CurrentMaterial);
+ }
+
+ ///
+ /// Startup keeps a usable certificate when replacement issuance fails inside the renewal window.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_preserves_usable_certificate_when_due_renewal_fails()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20));
+ store.Save(seeded);
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new AcmeOrderException("finalize_failed", "ca rejected"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var material = await manager.EnsureCertificateAsync(CancellationToken.None);
+
+ Assert.Equal(1, issuer.Calls);
+ AssertSameCertificateInstant(seeded.NotAfter, material.NotAfter);
+ AssertSameCertificateInstant(seeded.NotAfter, manager.CurrentMaterial!.NotAfter);
+ using var tls = manager.CreateTlsCertificate();
+ Assert.True(tls.NotAfter.ToUniversalTime() > DateTime.UtcNow);
+ }
+
+ ///
+ /// Startup replaces a due certificate when issuance succeeds.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_replaces_usable_certificate_when_due_renewal_succeeds()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20));
+ store.Save(seeded);
+ var replacementNotAfter = DateTimeOffset.UtcNow.AddDays(90);
+ var issuer = new CountingIssuer { FixedNotAfter = replacementNotAfter };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var material = await manager.EnsureCertificateAsync(CancellationToken.None);
+
+ Assert.Equal(1, issuer.Calls);
+ AssertSameCertificateInstant(replacementNotAfter, material.NotAfter);
+ Assert.NotEqual(TruncateToSeconds(seeded.NotAfter), TruncateToSeconds(material.NotAfter));
+ }
+
+ ///
+ /// Missing material still fails closed when issuance fails.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_fails_closed_when_absent_and_issuance_fails()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new AcmeOrderException("finalize_failed", "ca rejected"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var ex = await Assert.ThrowsAsync(
+ () => manager.EnsureCertificateAsync(CancellationToken.None));
+ Assert.Equal("finalize_failed", ex.Category);
+ Assert.Equal(1, issuer.Calls);
+ Assert.Null(manager.CurrentMaterial);
+ }
+
+ ///
+ /// Caller or host cancellation during due renewal propagates and is not treated as a renewal outage.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_propagates_caller_cancellation_during_due_renewal()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)));
+ using var cts = new CancellationTokenSource();
+ var issuer = new CountingIssuer
+ {
+ CancelOnIssue = cts,
+ ThrowOnIssue = new OperationCanceledException(cts.Token),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ await Assert.ThrowsAnyAsync(
+ () => manager.EnsureCertificateAsync(cts.Token));
+ Assert.Equal(1, issuer.Calls);
+ }
+
+ ///
+ /// A transport cancellation that is not tied to the caller token preserves a usable certificate.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_preserves_usable_certificate_on_transport_cancellation()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20));
+ store.Save(seeded);
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new OperationCanceledException("http timeout"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var material = await manager.EnsureCertificateAsync(CancellationToken.None);
+
+ Assert.Equal(1, issuer.Calls);
+ AssertSameCertificateInstant(seeded.NotAfter, material.NotAfter);
+ }
+
+ ///
+ /// Periodic renewal failure preserves the usable certificate and returns false.
+ ///
+ [Fact]
+ public async Task RenewIfDue_preserves_usable_certificate_when_issuance_fails()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20));
+ store.Save(seeded);
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new AcmeOrderException("finalize_failed", "ca rejected"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ Assert.False(await manager.RenewIfDueAsync(CancellationToken.None));
+ Assert.Equal(1, issuer.Calls);
+ AssertSameCertificateInstant(seeded.NotAfter, manager.CurrentMaterial!.NotAfter);
+ }
+
+ ///
+ /// A recoverable HTTP transport failure preserves an existing usable certificate.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_preserves_usable_certificate_on_http_request_failure()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ var seeded = TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20));
+ store.Save(seeded);
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new System.Net.Http.HttpRequestException("acme unreachable"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var material = await manager.EnsureCertificateAsync(CancellationToken.None);
+
+ Assert.Equal(1, issuer.Calls);
+ AssertSameCertificateInstant(seeded.NotAfter, material.NotAfter);
+ }
+
+ ///
+ /// An unexpected programming error during due renewal is not treated as a renewal outage.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_propagates_unexpected_programming_exception()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)));
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new InvalidOperationException("bug in issuer"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var ex = await Assert.ThrowsAsync(
+ () => manager.EnsureCertificateAsync(CancellationToken.None));
+ Assert.Equal("bug in issuer", ex.Message);
+ Assert.Equal(1, issuer.Calls);
+ }
+
+ ///
+ /// A certificate-store persistence failure is not classified as an ordinary CA renewal outage.
+ ///
+ [Fact]
+ public async Task EnsureCertificate_propagates_storage_exception_during_due_renewal()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)));
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new AcmeStorageException("certificate_persist_failed", "disk full"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var ex = await Assert.ThrowsAsync(
+ () => manager.EnsureCertificateAsync(CancellationToken.None));
+ Assert.Equal("certificate_persist_failed", ex.Category);
+ Assert.Equal(1, issuer.Calls);
+ }
+
+ ///
+ /// Periodic renewal propagates unexpected programming errors instead of preserving.
+ ///
+ [Fact]
+ public async Task RenewIfDue_propagates_unexpected_programming_exception()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)));
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new NullReferenceException("unexpected null"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ await Assert.ThrowsAsync(
+ () => manager.RenewIfDueAsync(CancellationToken.None));
+ Assert.Equal(1, issuer.Calls);
+ }
+
+ ///
+ /// Periodic renewal propagates storage failures instead of treating them as CA outages.
+ ///
+ [Fact]
+ public async Task RenewIfDue_propagates_storage_exception()
+ {
+ using var dir = new TempStateDir();
+ var names = CertificateIdentities.ForFqdn(Fqdn, includeNewsHostname: false);
+ var store = new CertificateStore(
+ dir.Path,
+ Fqdn,
+ TestCertificateFactory.Password,
+ names,
+ TimeSpan.FromDays(30));
+ store.Save(TestCertificateFactory.CreateMaterial(names, DateTimeOffset.UtcNow.AddDays(20)));
+ var issuer = new CountingIssuer
+ {
+ ThrowOnIssue = new AcmeStorageException("certificate_persist_failed", "disk full"),
+ };
+ var manager = CreateManager(dir.Path, store, issuer);
+
+ var ex = await Assert.ThrowsAsync(
+ () => manager.RenewIfDueAsync(CancellationToken.None));
+ Assert.Equal("certificate_persist_failed", ex.Category);
+ }
+
+ private static void AssertSameCertificateInstant(DateTimeOffset expected, DateTimeOffset actual) =>
+ Assert.Equal(TruncateToSeconds(expected), TruncateToSeconds(actual));
+
+ private static DateTimeOffset TruncateToSeconds(DateTimeOffset value) =>
+ new(value.Year, value.Month, value.Day, value.Hour, value.Minute, value.Second, value.Offset);
+
+ private static CertificateManager CreateManager(
+ string stateDir,
+ CertificateStore store,
+ ICertificateIssuer issuer) =>
+ new(
+ Fqdn,
+ stateDir,
+ DirectoryUrl,
+ store,
+ issuer,
+ TestCertificateFactory.Password,
+ TimeSpan.FromDays(30),
+ NullLogger.Instance,
+ includeNewsHostname: false);
+
+ /// Issuer that counts calls and optionally throws or returns a fixed not-after.
+ private sealed class CountingIssuer : ICertificateIssuer
+ {
+ public int Calls { get; private set; }
+
+ public Exception? ThrowOnIssue { get; set; }
+
+ public DateTimeOffset? FixedNotAfter { get; set; }
+
+ /// When set, cancelled after the call is counted so issuance sees a cancelled caller token.
+ public CancellationTokenSource? CancelOnIssue { get; set; }
+
+ public Task IssueAsync(
+ IReadOnlyList domains,
+ CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ Calls++;
+ CancelOnIssue?.Cancel();
+ if (ThrowOnIssue is not null)
+ {
+ throw ThrowOnIssue;
+ }
+
+ return Task.FromResult(
+ TestCertificateFactory.CreateMaterial(
+ domains,
+ FixedNotAfter ?? DateTimeOffset.UtcNow.AddDays(90)));
+ }
+ }
+ }
+}