diff --git a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryFraming.cs b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryFraming.cs
index 477537d0..153707dc 100644
--- a/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryFraming.cs
+++ b/src/VectorNNTPD.BackFillerService/ArticleWork/ArticleWorkDeliveryFraming.cs
@@ -4,149 +4,29 @@
namespace VectorNNTPD.BackFillerService.ArticleWork
{
///
- /// Decides whether retained ArtData can be sent as an RFC 3977 multiline block, and builds the
- /// stuffed header bytes that are written ahead of the retained body.
+ /// BackFiller facade over for retained ArtData delivery.
///
///
- ///
- /// Retained ArtData is canonical headers, the header/body separator, and a body whose lines
- /// are already dot-stuffed. The retained buffer is never rewritten. A header line that begins
- /// with . is stuffed only in the socket image, because a client removes one leading dot
- /// per RFC 3977 §3.1.1. The body is the existing slice and is not copied into that image.
- ///
- ///
- /// Every line in the retained buffer must end in CRLF. A bare CR, a bare LF, or NUL cannot be
- /// conveyed and is not frameable. A dot-leading line with no CRLF CRLF separator is also
- /// not frameable: the header region and the already-stuffed body cannot be told apart, so stuffing
- /// the whole buffer could corrupt the body.
- ///
+ /// StorageService and Common callers use directly.
+ /// This type keeps existing BackFiller call sites and tests stable.
///
internal static class ArticleWorkDeliveryFraming
{
- ///
- /// Reports whether can be sent as a multiline block without changing the retained bytes.
- ///
- /// Retained article. Not modified. Only this span is examined.
- ///
- /// when every line ends in CRLF, the buffer ends in CRLF, NUL is absent,
- /// and any dot-leading line sits in a header region before CRLF CRLF.
- ///
+ ///
internal static bool IsFrameable(ReadOnlySpan artData) =>
- CanonicalArticleFraming.Analyze(artData).IsFrameable;
+ CanonicalArticleDeliveryFraming.IsFrameable(artData);
- ///
- /// Builds the dot-stuffed header image when a header line begins with ..
- ///
- /// Retained article. Not modified.
- ///
- /// The stuffed header, including the header/body separator, when at least one header line needed a dot.
- /// when the caller should send unchanged,
- /// or when is not frameable.
- /// The body is not included.
- ///
- ///
- /// when holds the stuffed header.
- /// when the article is not frameable, has no header/body separator,
- /// or no header line begins with ..
- ///
- ///
- /// runs before the header is copied. The body stays in .
- ///
+ ///
internal static bool TryStuffHeaderLines(
ReadOnlySpan artData,
- [NotNullWhen(true)] out byte[]? stuffedHeader)
- {
- stuffedHeader = null;
- var framing = CanonicalArticleFraming.Analyze(artData);
- if (!framing.IsFrameable)
- {
- return false;
- }
+ [NotNullWhen(true)] out byte[]? stuffedHeader) =>
+ CanonicalArticleDeliveryFraming.TryStuffHeaderLines(artData, out stuffedHeader);
- return TryCreateStuffedHeader(artData, framing.BodyStart, out stuffedHeader);
- }
-
- ///
- /// Copies through with one extra leading dot
- /// on each header line that begins with ..
- ///
- /// Retained article. Not modified. Bytes at and after are not read for stuffing.
- /// Index of the first body byte from an earlier framing analysis of this same span.
- /// The stuffed header when a dot was inserted. Otherwise .
- /// when was allocated.
- ///
- /// Does not walk past and does not decide frameability.
- /// The caller uses a body start taken from the same logical bytes.
- ///
+ ///
internal static bool TryCreateStuffedHeader(
ReadOnlySpan artData,
int bodyStart,
- [NotNullWhen(true)] out byte[]? stuffedHeader)
- {
- stuffedHeader = null;
- if ((uint)bodyStart > (uint)artData.Length)
- {
- return false;
- }
-
- var header = artData[..bodyStart];
- var extra = CountDotLeadingLines(header);
- if (extra == 0)
- {
- return false;
- }
-
- stuffedHeader = GC.AllocateUninitializedArray(header.Length + extra);
- WriteDotStuffedLines(header, stuffedHeader);
- return true;
- }
-
- /// Counts CRLF lines in whose first byte is ..
- /// Header region, including the separator.
- /// The number of stuffing dots a header write inserts.
- private static int CountDotLeadingLines(ReadOnlySpan region)
- {
- var extra = 0;
- var index = 0;
- while (index < region.Length)
- {
- if (region[index] == (byte)'.')
- {
- extra++;
- }
-
- var crlf = region[index..].IndexOf("\r\n"u8);
- if (crlf < 0)
- {
- break;
- }
-
- index += crlf + 2;
- }
-
- return extra;
- }
-
- /// Writes with one extra leading dot on each line that begins with ..
- /// Header region whose lines end in CRLF.
- /// Buffer of length plus the stuffing count.
- private static void WriteDotStuffedLines(ReadOnlySpan region, Span destination)
- {
- var written = 0;
- var index = 0;
- while (index < region.Length)
- {
- if (region[index] == (byte)'.')
- {
- destination[written++] = (byte)'.';
- }
-
- var crlf = region[index..].IndexOf("\r\n"u8);
- var lineLength = crlf + 2;
- region.Slice(index, lineLength).CopyTo(destination[written..]);
- written += lineLength;
- index += lineLength;
- }
- }
+ [NotNullWhen(true)] out byte[]? stuffedHeader) =>
+ CanonicalArticleDeliveryFraming.TryCreateStuffedHeader(artData, bodyStart, out stuffedHeader);
}
}
diff --git a/src/VectorNNTPD.Common/Articles/CanonicalArticleDeliveryFraming.cs b/src/VectorNNTPD.Common/Articles/CanonicalArticleDeliveryFraming.cs
new file mode 100644
index 00000000..7aa0c183
--- /dev/null
+++ b/src/VectorNNTPD.Common/Articles/CanonicalArticleDeliveryFraming.cs
@@ -0,0 +1,151 @@
+using System.Diagnostics.CodeAnalysis;
+
+namespace VectorNNTPD.Common.Articles
+{
+ ///
+ /// Decides whether CanonicalV1 ArtData can be sent as an RFC 3977 multiline block, and builds the
+ /// stuffed header bytes that are written ahead of the retained body.
+ ///
+ ///
+ ///
+ /// CanonicalV1 ArtData is canonical headers, the header/body separator, and a body whose lines
+ /// are already storage-dot-stuffed. The retained buffer is never rewritten. A header line that begins
+ /// with . is stuffed only in the socket image, because a client removes one leading dot
+ /// per RFC 3977 §3.1.1. The body is the existing slice and is not copied into that image.
+ ///
+ ///
+ /// Every line in the retained buffer must end in CRLF. A bare CR, a bare LF, or NUL cannot be
+ /// conveyed and is not frameable. A dot-leading line with no CRLF CRLF separator is also
+ /// not frameable: the header region and the already-stuffed body cannot be told apart, so stuffing
+ /// the whole buffer could corrupt the body.
+ ///
+ ///
+ internal static class CanonicalArticleDeliveryFraming
+ {
+ ///
+ /// Reports whether can be sent as a multiline block without changing the retained bytes.
+ ///
+ /// Retained article. Not modified. Only this span is examined.
+ ///
+ /// when every line ends in CRLF, the buffer ends in CRLF, NUL is absent,
+ /// and any dot-leading line sits in a header region before CRLF CRLF.
+ ///
+ internal static bool IsFrameable(ReadOnlySpan artData) =>
+ CanonicalArticleFraming.Analyze(artData).IsFrameable;
+
+ ///
+ /// Builds the dot-stuffed header image when a header line begins with ..
+ ///
+ /// Retained article. Not modified.
+ ///
+ /// The stuffed header, including the header/body separator, when at least one header line needed a dot.
+ /// when the caller should send unchanged,
+ /// or when is not frameable.
+ /// The body is not included.
+ ///
+ ///
+ /// when holds the stuffed header.
+ /// when the article is not frameable, has no header/body separator,
+ /// or no header line begins with ..
+ ///
+ ///
+ /// runs before the header is copied. The body stays in .
+ ///
+ internal static bool TryStuffHeaderLines(
+ ReadOnlySpan artData,
+ [NotNullWhen(true)] out byte[]? stuffedHeader)
+ {
+ stuffedHeader = null;
+ var framing = CanonicalArticleFraming.Analyze(artData);
+ if (!framing.IsFrameable)
+ {
+ return false;
+ }
+
+ return TryCreateStuffedHeader(artData, framing.BodyStart, out stuffedHeader);
+ }
+
+ ///
+ /// Copies through with one extra leading dot
+ /// on each header line that begins with ..
+ ///
+ /// Retained article. Not modified. Bytes at and after are not read for stuffing.
+ /// Index of the first body byte from an earlier framing analysis of this same span.
+ /// The stuffed header when a dot was inserted. Otherwise .
+ /// when was allocated.
+ ///
+ /// Does not walk past and does not decide frameability.
+ /// The caller uses a body start taken from the same logical bytes.
+ ///
+ internal static bool TryCreateStuffedHeader(
+ ReadOnlySpan artData,
+ int bodyStart,
+ [NotNullWhen(true)] out byte[]? stuffedHeader)
+ {
+ stuffedHeader = null;
+ if ((uint)bodyStart > (uint)artData.Length)
+ {
+ return false;
+ }
+
+ var header = artData[..bodyStart];
+ var extra = CountDotLeadingLines(header);
+ if (extra == 0)
+ {
+ return false;
+ }
+
+ stuffedHeader = GC.AllocateUninitializedArray(header.Length + extra);
+ WriteDotStuffedLines(header, stuffedHeader);
+ return true;
+ }
+
+ /// Counts CRLF lines in whose first byte is ..
+ /// Header region, including the separator.
+ /// The number of stuffing dots a header write inserts.
+ private static int CountDotLeadingLines(ReadOnlySpan region)
+ {
+ var extra = 0;
+ var index = 0;
+ while (index < region.Length)
+ {
+ if (region[index] == (byte)'.')
+ {
+ extra++;
+ }
+
+ var crlf = region[index..].IndexOf("\r\n"u8);
+ if (crlf < 0)
+ {
+ break;
+ }
+
+ index += crlf + 2;
+ }
+
+ return extra;
+ }
+
+ /// Writes with one extra leading dot on each line that begins with ..
+ /// Header region whose lines end in CRLF.
+ /// Buffer of length plus the stuffing count.
+ private static void WriteDotStuffedLines(ReadOnlySpan region, Span destination)
+ {
+ var written = 0;
+ var index = 0;
+ while (index < region.Length)
+ {
+ if (region[index] == (byte)'.')
+ {
+ destination[written++] = (byte)'.';
+ }
+
+ var crlf = region[index..].IndexOf("\r\n"u8);
+ var lineLength = crlf + 2;
+ region.Slice(index, lineLength).CopyTo(destination[written..]);
+ written += lineLength;
+ index += lineLength;
+ }
+ }
+ }
+}
diff --git a/src/VectorNNTPD.StorageService/Configuration/SegmentPoolsOptions.cs b/src/VectorNNTPD.StorageService/Configuration/SegmentPoolsOptions.cs
index 827b028f..c2379f89 100644
--- a/src/VectorNNTPD.StorageService/Configuration/SegmentPoolsOptions.cs
+++ b/src/VectorNNTPD.StorageService/Configuration/SegmentPoolsOptions.cs
@@ -4,25 +4,27 @@ namespace VectorNNTPD.StorageService.Configuration
{
///
- /// Dedicated segment-pool directories and used-space thresholds for a future eviction loop.
- /// , , and do not
- /// admit or reject article writes. Advertisements are not changed by these thresholds.
+ /// Dedicated segment-pool directories and filesystem used-space thresholds for capacity policy.
+ /// Each configured directory has an independent High→Low eviction monitor.
+ /// excludes a spool from placement (and blocks TAKETHIS when no
+ /// eligible spool remains). Advertisements are not changed by these thresholds.
///
public sealed class SegmentPoolsOptions
{
///
- /// Highest used-space percent reserved for a future eviction loop.
- /// It does not make a spool ineligible for a new write and does not change the RabbitMQ advertisement.
+ /// Occupancy percent above which a spool is ineligible for new placement (strict greater-than).
+ /// Equal to this value remains eligible. Global TAKETHIS admission requires at least one
+ /// valid, fresh, eligible spool. Does not change RabbitMQ advertisements.
///
public int CriticalPercent { get; set; } = 85;
///
- /// Occupancy percent at or above which a future eviction loop would start. Not enforced in this phase.
+ /// Occupancy percent at or above which that spool's independent LFU eviction monitor starts.
///
public int HighPercent { get; set; } = 79;
///
- /// Occupancy percent at which a future eviction loop would stop. Not enforced in this phase.
+ /// Occupancy percent at or below which that spool's independent LFU eviction monitor stops.
///
public int LowPercent { get; set; } = 75;
diff --git a/src/VectorNNTPD.StorageService/Configuration/SpoolPersistenceOptions.cs b/src/VectorNNTPD.StorageService/Configuration/SpoolPersistenceOptions.cs
index 60297944..41754ca6 100644
--- a/src/VectorNNTPD.StorageService/Configuration/SpoolPersistenceOptions.cs
+++ b/src/VectorNNTPD.StorageService/Configuration/SpoolPersistenceOptions.cs
@@ -12,10 +12,13 @@ namespace VectorNNTPD.StorageService.Configuration
///
/// At 10 Gbps the payload rate is 1.25 GB/s. An average 750 KiB article
/// (750 × 1024 = 768_000 bytes) is about 1,628 articles/s. The default
- /// 1 GiB queue holds about 1,398 such articles. STORE admission reserves that
- /// article's META ArtSize before WINDOW credit is granted, so a full queue
- /// does not receive the body. TryCommit moves the reservation into the queue
- /// without adding the bytes twice. A dequeued article leaves the budget.
+ /// 1 GiB queue holds about 1,398 such articles. Admission reserves ArtSize
+ /// before 239. TryCommit moves the reservation into the queue without
+ /// adding the bytes twice. A dequeued article stays in the budget as in-flight until
+ /// the worker calls ReleaseProcessed (capacity-blocked holds included), so
+ /// sustained critical storage cannot refill the queue past .
+ /// TAKETHIS may assemble a body before reservation; concurrent assemblies are capped by
+ /// listener MaxActiveConnections and are outside this byte budget until reserved.
/// Path rewrite keeps a rented second buffer until publish returns, then returns it
/// to a pool capped at eight arrays per size. Live memory is still the original plus
/// that buffer. The pool retains at most eight arrays of each size that has been used.
diff --git a/src/VectorNNTPD.StorageService/Configuration/StorageServerOptions.cs b/src/VectorNNTPD.StorageService/Configuration/StorageServerOptions.cs
index 6b21d95b..62c4d1fe 100644
--- a/src/VectorNNTPD.StorageService/Configuration/StorageServerOptions.cs
+++ b/src/VectorNNTPD.StorageService/Configuration/StorageServerOptions.cs
@@ -226,6 +226,13 @@ public sealed class StorageServerListenerOptions
///
public int IoProgressTimeoutSeconds { get; set; } = 60;
+ ///
+ /// Overall deadline in seconds for receiving one TAKETHIS article (headers and body through
+ /// the multiline terminator). Unlike , this does not
+ /// reset when the peer drips another byte. Default is 300 seconds.
+ ///
+ public int ArticleReceiveTimeoutSeconds { get; set; } = 300;
+
/// Maximum concurrently active accepted connections.
public int MaxActiveConnections { get; set; } = 1024;
}
diff --git a/src/VectorNNTPD.StorageService/Configuration/StorageServerOptionsValidator.cs b/src/VectorNNTPD.StorageService/Configuration/StorageServerOptionsValidator.cs
index a1fc2306..e02042d6 100644
--- a/src/VectorNNTPD.StorageService/Configuration/StorageServerOptionsValidator.cs
+++ b/src/VectorNNTPD.StorageService/Configuration/StorageServerOptionsValidator.cs
@@ -437,6 +437,11 @@ private static void ValidateListener(StorageServerOptions options, List
failures.Add("StorageServer:Listener:IoProgressTimeoutSeconds must be between 1 and 600.");
}
+ if (listener.ArticleReceiveTimeoutSeconds is < 30 or > 3600)
+ {
+ failures.Add("StorageServer:Listener:ArticleReceiveTimeoutSeconds must be between 30 and 3600.");
+ }
+
if (listener.MaxActiveConnections < 1)
{
failures.Add("StorageServer:Listener:MaxActiveConnections must be between 1 and 2147483647.");
diff --git a/src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptions.cs b/src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptions.cs
index ef371ce7..de4c170d 100644
--- a/src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptions.cs
+++ b/src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptions.cs
@@ -33,5 +33,6 @@ public sealed record StorageServerListenerRuntimeOptions(
int ParserAccumulationMaxBytes,
TimeSpan TlsHandshakeTimeout,
TimeSpan IoProgressTimeout,
- int MaxActiveConnections);
+ int MaxActiveConnections,
+ TimeSpan ArticleReceiveTimeout);
}
diff --git a/src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptionsFactory.cs b/src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptionsFactory.cs
index 3ffef92f..ac274162 100644
--- a/src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptionsFactory.cs
+++ b/src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptionsFactory.cs
@@ -99,7 +99,8 @@ public static StorageServerRuntimeOptions Create(
listener.ParserAccumulationMaxBytes,
TimeSpan.FromSeconds(listener.TlsHandshakeTimeoutSeconds),
TimeSpan.FromSeconds(listener.IoProgressTimeoutSeconds),
- listener.MaxActiveConnections),
+ listener.MaxActiveConnections,
+ TimeSpan.FromSeconds(listener.ArticleReceiveTimeoutSeconds)),
CertificateDomainNames: CertificateIdentities.ForFqdn(fqdn, includeNewsHostname: false));
}
}
diff --git a/src/VectorNNTPD.StorageService/Hosting/StorageServerServiceCollectionExtensions.cs b/src/VectorNNTPD.StorageService/Hosting/StorageServerServiceCollectionExtensions.cs
index 42086907..17096257 100644
--- a/src/VectorNNTPD.StorageService/Hosting/StorageServerServiceCollectionExtensions.cs
+++ b/src/VectorNNTPD.StorageService/Hosting/StorageServerServiceCollectionExtensions.cs
@@ -128,7 +128,6 @@ public static HostApplicationBuilder AddStorageServerHosting(this HostApplicatio
new AcceptedArticleQueue(
provider.GetRequiredService>().Value.SpoolPersistence.MaxQueuedBytes,
provider.GetRequiredService()));
- builder.Services.TryAddSingleton();
builder.Services.TryAddSingleton(static provider => new SpoolHealth(provider.GetRequiredService()));
builder.Services.TryAddSingleton();
builder.Services.TryAddSingleton();
@@ -155,9 +154,17 @@ public static HostApplicationBuilder AddStorageServerHosting(this HostApplicatio
builder.Services.TryAddSingleton();
builder.Services.TryAddSingleton();
builder.Services.TryAddSingleton();
+ builder.Services.TryAddSingleton();
+ builder.Services.TryAddSingleton();
builder.Services.TryAddSingleton();
builder.Services.TryAddSingleton(static provider => provider.GetRequiredService());
builder.Services.TryAddSingleton();
+ builder.Services.TryAddSingleton();
+ builder.Services.TryAddSingleton(static provider =>
+ new SpoolPressure(provider.GetRequiredService()));
+ builder.Services.TryAddSingleton();
+ builder.Services.TryAddSingleton();
+ builder.Services.TryAddSingleton();
builder.Services.TryAddSingleton();
builder.Services.TryAddSingleton();
@@ -167,7 +174,7 @@ public static HostApplicationBuilder AddStorageServerHosting(this HostApplicatio
// Startup order: NntpDB → nntpsharedconfig → Cloudflare → RabbitMQ → ACME →
// segment-pool validation → spool shard initialization → article index → block-device monitor →
- // accepted-article worker → Listener → AdvertisementPublisher.
+ // per-spool eviction monitors → accepted-article worker → Listener → AdvertisementPublisher.
// Validation and shard initialization throw before the worker, listener, or advertisement publisher start.
// ApplicationServiceManager stops in reverse, so advertisement stops first and the worker stops after the listener completes the queue.
builder.Services.TryAddEnumerable(
@@ -195,6 +202,9 @@ public static HostApplicationBuilder AddStorageServerHosting(this HostApplicatio
builder.Services.TryAddEnumerable(
ServiceDescriptor.Singleton(static provider =>
provider.GetRequiredService()));
+ builder.Services.TryAddEnumerable(
+ ServiceDescriptor.Singleton(static provider =>
+ provider.GetRequiredService()));
builder.Services.TryAddEnumerable(
ServiceDescriptor.Singleton(static provider =>
provider.GetRequiredService()));
@@ -203,6 +213,11 @@ public static HostApplicationBuilder AddStorageServerHosting(this HostApplicatio
var listener = provider.GetRequiredService>().Value.Listener;
var dns = provider.GetRequiredService();
var index = provider.GetService();
+ var readLeases = provider.GetService();
+ Func? acquireArticleReadLease = readLeases is null
+ ? null
+ : readLeases.Acquire;
+ var time = provider.GetService() ?? TimeProvider.System;
Func? articlePresent = index is null
? null
: id =>
@@ -216,15 +231,59 @@ public static HostApplicationBuilder AddStorageServerHosting(this HostApplicatio
return false;
}
};
+ Func? resolveRetrievableArticle = index is null
+ ? null
+ : id =>
+ {
+ try
+ {
+ if (!index.IsReady || !index.TryGetRetrievableArticle(id, out var path))
+ {
+ return null;
+ }
+
+ return path;
+ }
+ catch (InvalidOperationException)
+ {
+ return null;
+ }
+ };
+ Func? afterArticleDelivered = index is null
+ ? null
+ : async (id, cancellationToken) =>
+ {
+ if (!index.TryGetIndexedRecord(id, out var record))
+ {
+ return;
+ }
+
+ await index.TouchAsync(
+ id,
+ record.SpoolDirectory,
+ time.GetUtcNow().ToUnixTimeMilliseconds(),
+ cancellationToken)
+ .ConfigureAwait(false);
+ };
+ var capacityGate = provider.GetRequiredService();
return new StorageNntpListenerService(
provider.GetRequiredService(),
provider.GetRequiredService(),
provider.GetRequiredService(),
provider.GetRequiredService>(),
provider.GetRequiredService(),
- new StoreAssemblyAdmission(listener.MaxActiveConnections),
+ new StoreAssemblyAdmission(
+ StoreAssemblyAdmission.LimitFor(
+ listener.MaxActiveConnections,
+ provider.GetRequiredService>().Value.SpoolPersistence
+ ?.MaxQueuedBytes
+ ?? SpoolPersistenceOptions.DefaultMaxQueuedBytes)),
+ allowsArticleAdmission: capacityGate.AllowsAdmission,
articlePresent: articlePresent,
- publishAfterBind: dns.PublishAfterListenerBoundAsync);
+ publishAfterBind: dns.PublishAfterListenerBoundAsync,
+ resolveRetrievableArticle: resolveRetrievableArticle,
+ afterArticleDelivered: afterArticleDelivered,
+ acquireArticleReadLease: acquireArticleReadLease);
});
builder.Services.TryAddEnumerable(
ServiceDescriptor.Singleton(static provider =>
diff --git a/src/VectorNNTPD.StorageService/Ingress/AcceptedArticleQueue.cs b/src/VectorNNTPD.StorageService/Ingress/AcceptedArticleQueue.cs
index 8646178a..f42b3ccf 100644
--- a/src/VectorNNTPD.StorageService/Ingress/AcceptedArticleQueue.cs
+++ b/src/VectorNNTPD.StorageService/Ingress/AcceptedArticleQueue.cs
@@ -5,7 +5,7 @@ namespace VectorNNTPD.StorageService.Ingress
{
///
- /// Multi-writer, multi-reader queue of accepted articles, bounded by queued bytes.
+ /// Multi-writer, multi-reader queue of accepted articles, bounded by accounted bytes.
///
///
///
@@ -14,9 +14,11 @@ namespace VectorNNTPD.StorageService.Ingress
/// not dropped after the budget check. Enqueue does not copy .
///
///
- /// The default budget is one gibibyte. At 768,000-byte articles that is about 1,398 articles.
- /// TAKETHIS 239 is sent only after a true enqueue. A false enqueue is the overload signal
- /// and is not a durable-storage acknowledgement.
+ /// The budget covers queued payloads, admission reservations, and dequeued in-flight articles
+ /// (including capacity-blocked holds) until . Dequeue alone does
+ /// not free budget. The default budget is one gibibyte. At 768,000-byte articles that is about
+ /// 1,398 articles. TAKETHIS 239 is sent only after a true enqueue. A false enqueue is the
+ /// overload signal and is not a durable-storage acknowledgement.
///
///
public sealed class AcceptedArticleQueue : IAcceptedArticleQueue
@@ -39,6 +41,7 @@ public sealed class AcceptedArticleQueue : IAcceptedArticleQueue
private long _dequeued;
private long _queuedBytes;
private long _reservedBytes;
+ private long _inFlightBytes;
private long _nextReservationId;
private bool _completed;
@@ -121,7 +124,19 @@ public long AccountedBytes
{
lock (_gate)
{
- return _queuedBytes + _reservedBytes;
+ return _queuedBytes + _reservedBytes + _inFlightBytes;
+ }
+ }
+ }
+
+ ///
+ public long InFlightBytes
+ {
+ get
+ {
+ lock (_gate)
+ {
+ return _inFlightBytes;
}
}
}
@@ -191,7 +206,7 @@ public bool TryEnqueue(AcceptedArticle article)
lock (_gate)
{
- var accounted = _queuedBytes + _reservedBytes;
+ var accounted = _queuedBytes + _reservedBytes + _inFlightBytes;
if (_completed || article.ArtSize > _maxQueuedBytes || accounted > _maxQueuedBytes - article.ArtSize)
{
return false;
@@ -387,19 +402,46 @@ public void Release(QueueReservation reservation)
DisposeRegistrations(granted);
}
- /// Accounts a dequeue and grants the oldest waiter that now fits.
- /// Bytes removed from the queue.
- /// Waiters whose registrations must be disposed after the lock is released.
+ ///
+ public void ReleaseProcessed(int bytes)
+ {
+ if (bytes < 1)
+ {
+ return;
+ }
+
+ List? granted;
+ lock (_gate)
+ {
+ if (_inFlightBytes <= 0)
+ {
+ return;
+ }
+
+ _inFlightBytes = _inFlightBytes > bytes ? _inFlightBytes - bytes : 0;
+ granted = PumpWaiters();
+ }
+
+ DisposeRegistrations(granted);
+ }
+
+ ///
+ /// Moves dequeued bytes from the queue into in-flight so capacity-blocked holds still
+ /// consume the admission budget. Does not grant waiters (accounted total is unchanged).
+ ///
+ /// Bytes removed from the queue channel.
+ /// Always null; dequeue does not free budget for new admissions.
private List? NoteDequeued(int bytes)
{
_dequeued++;
_queuedBytes -= bytes;
+ _inFlightBytes += bytes;
if (_stamps.Count > 0)
{
_stamps.Dequeue();
}
- return PumpWaiters();
+ return null;
}
///
@@ -415,7 +457,7 @@ private bool TryTakeReservation(int bytes, out QueueReservation reservation)
return false;
}
- var accounted = _queuedBytes + _reservedBytes;
+ var accounted = _queuedBytes + _reservedBytes + _inFlightBytes;
if (accounted > _maxQueuedBytes - bytes)
{
return false;
@@ -448,7 +490,7 @@ private QueueReservation AddReservation(int bytes)
while (_waiters.Count > 0)
{
var head = _waiters.Peek();
- var accounted = _queuedBytes + _reservedBytes;
+ var accounted = _queuedBytes + _reservedBytes + _inFlightBytes;
if (head.Bytes > _maxQueuedBytes || accounted > _maxQueuedBytes - head.Bytes)
{
return granted;
diff --git a/src/VectorNNTPD.StorageService/Ingress/AcceptedArticleWorker.cs b/src/VectorNNTPD.StorageService/Ingress/AcceptedArticleWorker.cs
index 98bd3246..1ad4a35e 100644
--- a/src/VectorNNTPD.StorageService/Ingress/AcceptedArticleWorker.cs
+++ b/src/VectorNNTPD.StorageService/Ingress/AcceptedArticleWorker.cs
@@ -20,9 +20,15 @@ namespace VectorNNTPD.StorageService.Ingress
///
///
/// Each worker holds at most one dequeued article. A waiting-for-capacity result keeps that
- /// article off the queue and retries after .
- /// Shutdown cancels in-flight publishes. An article already dequeued and not yet published is
- /// logged and dropped. TAKETHIS 239 was sent at enqueue and is not repeated here.
+ /// article off the queue (still bounded by worker count) and retries when
+ /// fires or after
+ /// , whichever comes first.
+ ///
+ ///
+ /// Graceful completes the queue (if the listener has not already), then waits
+ /// for every queued and in-flight article to finish Path rewrite, spool publish, and index update.
+ /// Host cancellation of is only a drain deadline: it does not abandon work
+ /// until that deadline fires. Forced process kill after TAKETHIS 239 may still lose accepted articles.
///
///
internal sealed class AcceptedArticleWorker : IApplicationService
@@ -32,6 +38,7 @@ internal sealed class AcceptedArticleWorker : IApplicationService
private readonly ILogger _logger;
private readonly SpoolPersistenceOptions _options;
private readonly SpoolPressure? _pressure;
+ private readonly SpoolCapacityWaitSignal? _capacityWait;
private readonly SpoolPersistenceCounters? _counters;
private readonly TimeProvider _time;
private readonly CancellationTokenSource _runCts = new();
@@ -45,6 +52,7 @@ internal sealed class AcceptedArticleWorker : IApplicationService
private long _scaleArmedAt;
private int _scaleDirection;
private long _lastSummaryAt;
+ private int _drainFailures;
/// Initializes the host worker from validated options.
/// Accepted-article queue.
@@ -54,6 +62,7 @@ internal sealed class AcceptedArticleWorker : IApplicationService
/// Saturation signal. A null signal never reports saturation.
/// Optional fault counter.
/// Clock for dwell and retry delays.
+ /// Wakes capacity-blocked workers on fresh samples or reclaim.
public AcceptedArticleWorker(
IAcceptedArticleQueue queue,
IAcceptedArticleProcessor processor,
@@ -61,7 +70,8 @@ public AcceptedArticleWorker(
IOptions options,
SpoolPressure pressure,
SpoolPersistenceCounters counters,
- TimeProvider time)
+ TimeProvider time,
+ SpoolCapacityWaitSignal capacityWait)
: this(
queue,
processor,
@@ -69,7 +79,8 @@ public AcceptedArticleWorker(
options.Value.SpoolPersistence ?? new SpoolPersistenceOptions(),
pressure,
counters,
- time)
+ time,
+ capacityWait)
{
ArgumentNullException.ThrowIfNull(options);
}
@@ -82,6 +93,7 @@ public AcceptedArticleWorker(
/// Saturation signal.
/// Fault counter.
/// Clock. Null uses .
+ /// Optional capacity-recovery wake signal.
internal AcceptedArticleWorker(
IAcceptedArticleQueue queue,
IAcceptedArticleProcessor processor,
@@ -89,7 +101,8 @@ internal AcceptedArticleWorker(
SpoolPersistenceOptions? options = null,
SpoolPressure? pressure = null,
SpoolPersistenceCounters? counters = null,
- TimeProvider? time = null)
+ TimeProvider? time = null,
+ SpoolCapacityWaitSignal? capacityWait = null)
{
ArgumentNullException.ThrowIfNull(queue);
ArgumentNullException.ThrowIfNull(processor);
@@ -99,6 +112,7 @@ internal AcceptedArticleWorker(
_logger = logger;
_options = options ?? new SpoolPersistenceOptions();
_pressure = pressure;
+ _capacityWait = capacityWait;
_counters = counters;
_time = time ?? TimeProvider.System;
}
@@ -136,24 +150,88 @@ public Task StartAsync(CancellationToken cancellationToken)
}
///
+ ///
+ /// Completes the accepted-article queue when the listener has not already done so, waits until
+ /// every worker has finished queued and in-flight articles (without cancelling persistence),
+ /// then cancels the run token so the control loop can exit (including when it is parked on a
+ /// delay). Host cancellation of
+ /// is only the drain deadline.
+ ///
public async Task StopAsync(CancellationToken cancellationToken)
{
- await _runCts.CancelAsync().ConfigureAwait(false);
+ if (!_queue.IsCompleted)
+ {
+ _queue.Complete();
+ }
+
Wake();
var execution = _execution;
if (execution is null)
{
+ ThrowIfDrainFailed();
return;
}
+ try
+ {
+ await WaitForWorkersToDrainAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ await _runCts.CancelAsync().ConfigureAwait(false);
+ Wake();
+ try
+ {
+ await execution.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is TimeoutException or OperationCanceledException)
+ {
+ ExpectedException.Ignore(ex);
+ }
+
+ throw new InvalidOperationException(
+ "Accepted-article drain did not finish before the graceful shutdown budget elapsed.");
+ }
+
+ await _runCts.CancelAsync().ConfigureAwait(false);
+ Wake();
try
{
await execution.WaitAsync(cancellationToken).ConfigureAwait(false);
}
- catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested)
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
- ExpectedException.Ignore(ex);
+ throw new InvalidOperationException(
+ "Accepted-article worker control loop did not exit before the graceful shutdown budget elapsed.");
}
+
+ ThrowIfDrainFailed();
+ }
+
+ ///
+ /// Waits until the completed queue is empty and no worker is still processing.
+ /// Uses the host clock so a test cannot stall the drain.
+ ///
+ private async Task WaitForWorkersToDrainAsync(CancellationToken cancellationToken)
+ {
+ while (_queue.QueuedCount > 0 || Volatile.Read(ref _active) > 0)
+ {
+ Wake();
+ await Task.Delay(10, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ /// Fails stop when any accepted article was abandoned or reported .
+ private void ThrowIfDrainFailed()
+ {
+ var failures = Volatile.Read(ref _drainFailures);
+ if (failures == 0)
+ {
+ return;
+ }
+
+ throw new InvalidOperationException(
+ $"Accepted-article drain finished with {failures} failed or abandoned article(s).");
}
///
@@ -353,32 +431,40 @@ private async Task WorkerLoop(CancellationToken cancellationToken)
try
{
+ // Capture before ProcessAsync so a Notify during the attempt completes this wait.
+ var capacityWait = _capacityWait?.CaptureWait() ?? Task.CompletedTask;
var status = await _processor.ProcessAsync(article, cancellationToken).ConfigureAwait(false);
if (status == AcceptedArticleProcessResult.WaitingForCapacity)
{
held = article;
- await Task.Delay(
- TimeSpan.FromMilliseconds(_options.IneligibleRetryMilliseconds),
- _time,
- cancellationToken)
- .ConfigureAwait(false);
+ await WaitForCapacityRetryAsync(capacityWait, cancellationToken).ConfigureAwait(false);
continue;
}
+ if (status == AcceptedArticleProcessResult.Failed)
+ {
+ Interlocked.Increment(ref _drainFailures);
+ }
+
+ _queue.ReleaseProcessed(article.ArtSize);
held = null;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
LogDropped(article);
+ Interlocked.Increment(ref _drainFailures);
+ _queue.ReleaseProcessed(article.ArtSize);
held = null;
break;
}
catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
{
+ Interlocked.Increment(ref _drainFailures);
AcceptedArticleWorkerLogMessages.ProcessingFailed(
_logger,
ex,
article.ArtId.ToLowerHexString());
+ _queue.ReleaseProcessed(article.ArtSize);
held = null;
}
}
@@ -386,13 +472,17 @@ await Task.Delay(
catch (Exception ex) when (ex is not OperationCanceledException && !ExceptionHelpers.IsFatal(ex))
{
_counters?.NoteWorkerFault();
+ Interlocked.Increment(ref _drainFailures);
AcceptedArticleWorkerLogMessages.ProcessingFailed(_logger, ex, "worker");
}
finally
{
- if (held is { } leftover && cancellationToken.IsCancellationRequested)
+ if (held is { } leftover)
{
+ // Queue already completed; abandoning a held article violates the drain contract.
LogDropped(leftover);
+ Interlocked.Increment(ref _drainFailures);
+ _queue.ReleaseProcessed(leftover.ArtSize);
}
if (!retired)
@@ -402,6 +492,26 @@ await Task.Delay(
}
}
+ ///
+ /// Waits for a capacity-recovery notify or the configured poll interval.
+ /// Does not allocate a per-article retry task beyond this worker's stack frame.
+ ///
+ private async Task WaitForCapacityRetryAsync(Task capacityWait, CancellationToken cancellationToken)
+ {
+ if (capacityWait.IsCompleted)
+ {
+ // Notify raced during ProcessAsync; retry immediately without waiting the poll interval.
+ return;
+ }
+
+ var delay = Task.Delay(
+ TimeSpan.FromMilliseconds(_options.IneligibleRetryMilliseconds),
+ _time,
+ cancellationToken);
+ var completed = await Task.WhenAny(capacityWait, delay).ConfigureAwait(false);
+ await completed.ConfigureAwait(false);
+ }
+
/// Waits for an article or for a scale-down wake. A wake does not drop a queued article.
private async ValueTask DequeueAsync(CancellationToken cancellationToken)
{
diff --git a/src/VectorNNTPD.StorageService/Ingress/IAcceptedArticleQueue.cs b/src/VectorNNTPD.StorageService/Ingress/IAcceptedArticleQueue.cs
index 1b5601b2..9aaa41d8 100644
--- a/src/VectorNNTPD.StorageService/Ingress/IAcceptedArticleQueue.cs
+++ b/src/VectorNNTPD.StorageService/Ingress/IAcceptedArticleQueue.cs
@@ -33,12 +33,29 @@ public interface IAcceptedArticleQueue
/// Bytes reserved for admitted transfers that are not yet queued.
long ReservedBytes { get; }
- /// Queued bytes plus reserved bytes. Both count toward .
+ ///
+ /// Queued bytes plus reserved bytes plus in-flight (dequeued, not yet released) bytes.
+ /// All three count toward .
+ ///
long AccountedBytes { get; }
+ ///
+ /// Bytes for articles dequeued but not yet finished by a worker
+ /// (including capacity-blocked holds). Counts toward .
+ ///
+ long InFlightBytes { get; }
+
/// Admission requests waiting for reserved bytes. They do not hold a reservation yet.
int AdmissionWaiters { get; }
+ ///
+ /// Releases bytes retained after dequeue once processing finishes (completed, failed, or abandoned).
+ /// Do not call while the article is still held for WaitingForCapacity.
+ /// A second release of the same bytes is a no-op only when the caller never over-releases the total.
+ ///
+ /// Article size previously moved into in-flight on dequeue.
+ void ReleaseProcessed(int bytes);
+
///
/// Reserves when no admission is waiting and the reservation fits.
/// A false result reserves nothing. Callers that must wait use .
@@ -84,8 +101,10 @@ public interface IAcceptedArticleQueue
///
/// Removes one article when one is already queued.
/// Does not wait and does not allocate a cancellation source.
- /// A false result leaves the queue unchanged. A true result releases that article's bytes
- /// from the queue budget and transfers ownership of the article to the caller.
+ /// A false result leaves the queue unchanged. A true result moves that article's bytes
+ /// into (still counted in ) and
+ /// transfers ownership of the article to the caller, who must call
+ /// when finished.
/// An empty completed queue returns ; it does not signal completion.
/// Call to wait or to observe completion.
///
diff --git a/src/VectorNNTPD.StorageService/Listener/StorageNntpFeedSession.cs b/src/VectorNNTPD.StorageService/Listener/StorageNntpFeedSession.cs
index fb46ef76..ff41ef89 100644
--- a/src/VectorNNTPD.StorageService/Listener/StorageNntpFeedSession.cs
+++ b/src/VectorNNTPD.StorageService/Listener/StorageNntpFeedSession.cs
@@ -9,9 +9,10 @@
namespace VectorNNTPD.StorageService.Listener
{
///
- /// RFC 4644 STREAMING feed session over an established TLS transport.
- /// Accepts CAPABILITIES, MODE STREAM, CHECK, TAKETHIS, and QUIT.
+ /// Storage NNTP session over an established TLS transport.
+ /// Accepts ARTICLE, TAKETHIS, and QUIT only. Other verbs receive 500.
/// A successful TAKETHIS commits into before 239 is written.
+ /// ARTICLE serves indexed CanonicalV1 spool files immediately; no mode negotiation is required.
///
public sealed partial class StorageNntpFeedSession : IAsyncDisposable
{
@@ -24,19 +25,18 @@ public sealed partial class StorageNntpFeedSession : IAsyncDisposable
private const int InitialAssemblyRentBytes = 64 * 1024;
private static readonly byte[] Greeting = "200 VectorNNTP StorageServer ready\r\n"u8.ToArray();
- private static readonly byte[] CapabilitiesHeader = "101 Capability list follows\r\n"u8.ToArray();
- private static readonly byte[] CapabilitiesBody = "VERSION 2\r\nSTREAMING\r\nIMPLEMENTATION VectorNNTP-StorageServer\r\n.\r\n"u8.ToArray();
- private static readonly byte[] ModeStreamOk = "203 Streaming allowed\r\n"u8.ToArray();
private static readonly byte[] Closing = "205 Connection closing\r\n"u8.ToArray();
private static readonly byte[] SyntaxError = "501 Syntax error\r\n"u8.ToArray();
private static readonly byte[] UnknownCommand = "500 Unknown command\r\n"u8.ToArray();
private static readonly byte[] TemporaryFailure = "400 Temporary failure\r\n"u8.ToArray();
+ private static readonly byte[] NoSuchArticle = "430 No Such Article Found\r\n"u8.ToArray();
+ private static readonly byte[] ArticleNotAvailable = "403 Article not available\r\n"u8.ToArray();
+ private static readonly byte[] ArticlePrefix = "220 0 "u8.ToArray();
+ private static readonly byte[] ArticleFollowsSuffix = " article follows\r\n"u8.ToArray();
+ private static readonly byte[] ArticleTerminator = ".\r\n"u8.ToArray();
- private static readonly byte[] CapabilitiesKeyword = "CAPABILITIES"u8.ToArray();
- private static readonly byte[] ModeKeyword = "MODE"u8.ToArray();
- private static readonly byte[] StreamKeyword = "STREAM"u8.ToArray();
- private static readonly byte[] CheckKeyword = "CHECK"u8.ToArray();
private static readonly byte[] TakeThisKeyword = "TAKETHIS"u8.ToArray();
+ private static readonly byte[] ArticleKeyword = "ARTICLE"u8.ToArray();
private static readonly byte[] QuitKeyword = "QUIT"u8.ToArray();
private readonly IStorageListenerTransport _transport;
@@ -44,15 +44,20 @@ public sealed partial class StorageNntpFeedSession : IAsyncDisposable
private readonly ILogger _logger;
private readonly IAcceptedArticleQueue? _acceptedArticles;
private readonly StoreAssemblyAdmission? _assemblyAdmission;
+ private readonly Func? _allowsArticleAdmission;
private readonly Func? _articlePresent;
+ private readonly Func? _resolveRetrievableArticle;
+ private readonly Func? _afterArticleDelivered;
+ private readonly Func? _acquireArticleReadLease;
private readonly NntpArticleParser _parser;
private byte[] _pending;
private readonly byte[] _readScratch;
private readonly byte[] _statusScratch;
+ private readonly byte[] _articleFollows;
private int _pendingCount;
private int _disposed;
- /// Initializes a STREAMING feed session over an established transport.
+ /// Initializes a Storage NNTP session over an established transport.
public StorageNntpFeedSession(
IStorageListenerTransport transport,
StorageServerListenerRuntimeOptions listener,
@@ -60,7 +65,11 @@ public StorageNntpFeedSession(
string pathIdentity,
IAcceptedArticleQueue? acceptedArticles = null,
StoreAssemblyAdmission? assemblyAdmission = null,
- Func? articlePresent = null)
+ Func? allowsArticleAdmission = null,
+ Func? articlePresent = null,
+ Func? resolveRetrievableArticle = null,
+ Func? afterArticleDelivered = null,
+ Func? acquireArticleReadLease = null)
{
ArgumentNullException.ThrowIfNull(transport);
ArgumentNullException.ThrowIfNull(listener);
@@ -71,12 +80,18 @@ public StorageNntpFeedSession(
_logger = logger;
_acceptedArticles = acceptedArticles;
_assemblyAdmission = assemblyAdmission;
+ _allowsArticleAdmission = allowsArticleAdmission;
_articlePresent = articlePresent;
+ _resolveRetrievableArticle = resolveRetrievableArticle;
+ _afterArticleDelivered = afterArticleDelivered;
+ _acquireArticleReadLease = acquireArticleReadLease;
_parser = new NntpArticleParser(pathIdentity.Trim());
var capacity = Math.Max(listener.ParserAccumulationMaxBytes, MaximumCommandOctets);
_pending = new byte[MaximumCommandOctets];
_readScratch = new byte[Math.Min(capacity, 64 * 1024)];
_statusScratch = new byte[4 + 1 + NntpMessageIdValidation.MaxMessageIdLength + 2];
+ _articleFollows = new byte[
+ ArticlePrefix.Length + NntpMessageIdValidation.MaxMessageIdLength + ArticleFollowsSuffix.Length];
}
/// Sends the greeting, then reads commands until QUIT, peer close, or cancellation.
@@ -110,34 +125,6 @@ public async Task RunAsync(CancellationToken cancellationToken)
return;
}
- if (TryMatchKeyword(line, CapabilitiesKeyword, out var capsRest))
- {
- if (!capsRest.IsEmpty)
- {
- await WriteFullyAsync(SyntaxError, cancellationToken).ConfigureAwait(false);
- continue;
- }
-
- await WriteFullyAsync(CapabilitiesHeader, cancellationToken).ConfigureAwait(false);
- await WriteFullyAsync(CapabilitiesBody, cancellationToken).ConfigureAwait(false);
- continue;
- }
-
- if (TryMatchModeStream(line, out var modeStreamSyntaxError))
- {
- await WriteFullyAsync(
- modeStreamSyntaxError ? SyntaxError : ModeStreamOk,
- cancellationToken)
- .ConfigureAwait(false);
- continue;
- }
-
- if (TryMatchKeyword(line, CheckKeyword, out var checkRest))
- {
- await HandleCheckAsync(SkipSpaces(checkRest), cancellationToken).ConfigureAwait(false);
- continue;
- }
-
if (TryMatchKeyword(line, TakeThisKeyword, out var takeRest))
{
// One heap copy of the message-id argument; spans cannot cross the TAKETHIS awaits.
@@ -151,6 +138,14 @@ await WriteFullyAsync(
continue;
}
+ if (TryMatchKeyword(line, ArticleKeyword, out var articleRest))
+ {
+ // Message-ID octets must outlive the awaits used while reading the spool file.
+ var articleArg = SkipSpaces(articleRest).ToArray();
+ await HandleArticleAsync(articleArg, cancellationToken).ConfigureAwait(false);
+ continue;
+ }
+
await WriteFullyAsync(UnknownCommand, cancellationToken).ConfigureAwait(false);
}
}
@@ -166,20 +161,184 @@ public ValueTask DisposeAsync()
return ValueTask.CompletedTask;
}
- private Task HandleCheckAsync(ReadOnlySpan argument, CancellationToken cancellationToken)
+ ///
+ /// Serves ARTICLE <message-id> from the indexed final CanonicalV1 spool file.
+ /// The published file is read and framed fully before any 220 is written, so a read
+ /// failure never injects a second status line into a partial article response. A write failure
+ /// after 220 propagates and ends the session without emitting another NNTP status.
+ ///
+ private async Task HandleArticleAsync(byte[] argument, CancellationToken cancellationToken)
{
- if (!TryParseMessageIdArgument(argument, out var messageId))
+ if (!TryParseMessageIdArgument(argument, out var messageIdSpan))
{
- return WriteFullyAsync(SyntaxError, cancellationToken);
+ await WriteFullyAsync(SyntaxError, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+
+ // Heap copy: ReadOnlySpan cannot live across the spool-read awaits below.
+ var messageId = messageIdSpan.ToArray();
+ var articleId = ArticleId.FromMessageId(messageId);
+ // Lease before path resolve so eviction cannot delete between lookup and open.
+ using var readLease = _acquireArticleReadLease?.Invoke(articleId);
+ string? articlePath;
+ try
+ {
+ articlePath = _resolveRetrievableArticle?.Invoke(articleId);
+ }
+ catch (Exception ex) when (ex is InvalidOperationException or ArgumentException)
+ {
+ await WriteFullyAsync(NoSuchArticle, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+
+ if (string.IsNullOrEmpty(articlePath))
+ {
+ await WriteFullyAsync(NoSuchArticle, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+
+ byte[] artData;
+ try
+ {
+ artData = await ReadPublishedArticleAsync(articlePath, cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ throw;
+ }
+ catch (FileNotFoundException)
+ {
+ await WriteFullyAsync(NoSuchArticle, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+ catch (DirectoryNotFoundException)
+ {
+ await WriteFullyAsync(NoSuchArticle, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or EndOfStreamException)
+ {
+ StorageNntpFeedSessionLogMessages.ArticleRetrievalFailed(
+ _logger,
+ articleId.ToLowerHexString(),
+ ex.GetType().Name);
+ await WriteFullyAsync(TemporaryFailure, cancellationToken).ConfigureAwait(false);
+ return;
}
- var alreadyHave = _articlePresent?.Invoke(ArticleId.FromMessageId(messageId)) == true;
- return WriteStatusAsync(
- alreadyHave ? (byte)'4' : (byte)'2',
- (byte)'3',
- (byte)'8',
- messageId,
- cancellationToken);
+ if (artData.Length == 0 || artData.Length > ArticleResourceLimits.MaxArticleBytes)
+ {
+ await WriteFullyAsync(ArticleNotAvailable, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+
+ var framing = CanonicalArticleFraming.Analyze(artData);
+ if (!framing.IsFrameable)
+ {
+ await WriteFullyAsync(ArticleNotAvailable, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+
+ byte[]? stuffedHeader = null;
+ var splitBody = framing.BodyStart >= 0
+ && CanonicalArticleDeliveryFraming.TryCreateStuffedHeader(
+ artData,
+ framing.BodyStart,
+ out stuffedHeader);
+
+ // Response phase: do not catch write failures — a mid-response I/O error must not emit 4xx into the body.
+ await WriteFullyAsync(BuildArticleFollowsLine(messageId), cancellationToken).ConfigureAwait(false);
+ if (splitBody)
+ {
+ await WriteFullyAsync(stuffedHeader, cancellationToken).ConfigureAwait(false);
+ await WriteFullyAsync(artData.AsMemory(framing.BodyStart), cancellationToken).ConfigureAwait(false);
+ }
+ else
+ {
+ await WriteFullyAsync(artData, cancellationToken).ConfigureAwait(false);
+ }
+
+ await WriteFullyAsync(ArticleTerminator, cancellationToken).ConfigureAwait(false);
+ await NotifyArticleDeliveredBestEffortAsync(articleId, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// Reads only the published final path. Temporary .writing-* names are never opened.
+ ///
+ private static async Task ReadPublishedArticleAsync(
+ string articlePath,
+ CancellationToken cancellationToken)
+ {
+ await using var stream = new FileStream(
+ articlePath,
+ FileMode.Open,
+ FileAccess.Read,
+ FileShare.Read,
+ bufferSize: 64 * 1024,
+ options: FileOptions.Asynchronous | FileOptions.SequentialScan);
+
+ if (stream.Length <= 0 || stream.Length > ArticleResourceLimits.MaxArticleBytes)
+ {
+ return [];
+ }
+
+ var length = (int)stream.Length;
+ var buffer = GC.AllocateUninitializedArray(length);
+ var offset = 0;
+ while (offset < length)
+ {
+ var read = await stream.ReadAsync(buffer.AsMemory(offset, length - offset), cancellationToken)
+ .ConfigureAwait(false);
+ if (read == 0)
+ {
+ throw new EndOfStreamException();
+ }
+
+ offset += read;
+ }
+
+ return buffer;
+ }
+
+ ///
+ /// Best-effort post-delivery hook (typically last-access touch). Failures never change the already-written 220.
+ ///
+ private async Task NotifyArticleDeliveredBestEffortAsync(
+ ArticleId articleId,
+ CancellationToken cancellationToken)
+ {
+ var notify = _afterArticleDelivered;
+ if (notify is null)
+ {
+ return;
+ }
+
+ try
+ {
+ await notify(articleId, cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ StorageNntpFeedSessionLogMessages.ArticleTouchFailed(
+ _logger,
+ articleId.ToLowerHexString(),
+ ex.GetType().Name);
+ }
+ }
+
+ /// Writes 220 0 <message-id> article follows ending in CRLF into .
+ private ReadOnlyMemory BuildArticleFollowsLine(ReadOnlySpan messageId)
+ {
+ ArgumentOutOfRangeException.ThrowIfGreaterThan(messageId.Length, NntpMessageIdValidation.MaxMessageIdLength);
+ var length = ArticlePrefix.Length + messageId.Length + ArticleFollowsSuffix.Length;
+ ArticlePrefix.CopyTo(_articleFollows);
+ messageId.CopyTo(_articleFollows.AsSpan(ArticlePrefix.Length));
+ ArticleFollowsSuffix.CopyTo(_articleFollows.AsSpan(ArticlePrefix.Length + messageId.Length));
+ return _articleFollows.AsMemory(0, length);
}
private async Task HandleTakeThisAsync(byte[] argument, CancellationToken cancellationToken)
@@ -195,12 +354,27 @@ private async Task HandleTakeThisAsync(byte[] argument, CancellationToken
// argument holds the message-id octets for the rest of this method (copied once in RunAsync).
var requestId = ArticleId.FromMessageId(requestMessageId);
+ // Critical / unknown capacity: reject before assembly or queue admission (no 239).
+ if (_allowsArticleAdmission is not null && !_allowsArticleAdmission())
+ {
+ await DrainArticleAsync(cancellationToken).ConfigureAwait(false);
+ await WriteStatusAsync(
+ (byte)'4',
+ (byte)'3',
+ (byte)'9',
+ argument,
+ cancellationToken)
+ .ConfigureAwait(false);
+ return true;
+ }
+
StoreAssemblyAdmission? admission = null;
var gate = _assemblyAdmission;
if (gate is not null)
{
if (!gate.TryEnter())
{
+ // No assembly permit: discard without renting a growing article buffer.
await DrainArticleAsync(cancellationToken).ConfigureAwait(false);
await WriteStatusAsync(
(byte)'4',
@@ -217,6 +391,8 @@ await WriteStatusAsync(
byte[]? rented = null;
QueueReservation? reservation = null;
+ // Canonical buffer owned until queue commit transfers ArtData; release on any pre-commit exit.
+ ArticleRecord? assembled = null;
try
{
var destuffed = await ReadDestuffedArticleAsync(cancellationToken).ConfigureAwait(false);
@@ -234,6 +410,11 @@ await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationTo
_parser,
payload,
ArticlePathMode.Normalize);
+
+ // Destuffed wire buffer is no longer required; free it before any admission wait so
+ // blocked queue reservation cannot retain both wire and canonical copies.
+ ReturnAssemblyBuffer(ref rented);
+
if (!created.IsAccepted)
{
await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationToken)
@@ -241,10 +422,9 @@ await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationTo
return true;
}
- var record = created.Record;
- if (!record.ArtId.Equals(requestId))
+ assembled = created.Record;
+ if (!assembled.Value.ArtId.Equals(requestId))
{
- record.ReleaseUnusedCanonicalBuffer();
await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationToken)
.ConfigureAwait(false);
return true;
@@ -253,14 +433,21 @@ await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationTo
var queue = _acceptedArticles;
if (queue is null || queue.IsCompleted)
{
- record.ReleaseUnusedCanonicalBuffer();
await WriteFullyAsync(TemporaryFailure, cancellationToken).ConfigureAwait(false);
return false;
}
+ // Re-check after assembly so a critical sample published during the read cannot slip a 239.
+ if (_allowsArticleAdmission is not null && !_allowsArticleAdmission())
+ {
+ await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationToken)
+ .ConfigureAwait(false);
+ return true;
+ }
+
+ var record = assembled.Value;
if (record.ArtSize > queue.MaxQueuedBytes)
{
- record.ReleaseUnusedCanonicalBuffer();
await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationToken)
.ConfigureAwait(false);
return true;
@@ -280,7 +467,6 @@ await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationTo
if (reservation is null)
{
- record.ReleaseUnusedCanonicalBuffer();
await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationToken)
.ConfigureAwait(false);
return true;
@@ -301,18 +487,19 @@ await WriteStatusAsync((byte)'4', (byte)'3', (byte)'9', argument, cancellationTo
{
queue.Release(reservation.Value);
reservation = null;
- record.ReleaseUnusedCanonicalBuffer();
throw;
}
reservation = null;
if (!committed)
{
- record.ReleaseUnusedCanonicalBuffer();
await WriteFullyAsync(TemporaryFailure, cancellationToken).ConfigureAwait(false);
return false;
}
+ // Queue owns ArtData; do not release the canonical creator on this path.
+ assembled = null;
+
// ArtData remains reachable via the queued Memory; Exact buffers are not pooled.
if (_logger.IsEnabled(LogLevel.Debug))
{
@@ -334,6 +521,25 @@ await WriteStatusAsync((byte)'2', (byte)'3', (byte)'9', argument, cancellationTo
throw;
}
+ catch (OperationCanceledException)
+ {
+ if (reservation is { } held)
+ {
+ _acceptedArticles?.Release(held);
+ }
+
+ throw;
+ }
+ catch (TimeoutException)
+ {
+ if (reservation is { } held)
+ {
+ _acceptedArticles?.Release(held);
+ }
+
+ await WriteFullyAsync(TemporaryFailure, cancellationToken).ConfigureAwait(false);
+ return false;
+ }
catch (InvalidOperationException)
{
// Oversized article or unrecoverable framing residue: RFC 4644 temporary failure + close.
@@ -347,34 +553,202 @@ await WriteStatusAsync((byte)'2', (byte)'3', (byte)'9', argument, cancellationTo
}
finally
{
- if (rented is not null)
+ if (assembled is { } leftover)
{
- ArrayPool.Shared.Return(rented);
+ leftover.ReleaseUnusedCanonicalBuffer();
}
+ ReturnAssemblyBuffer(ref rented);
admission?.Exit();
}
}
+ ///
+ /// Discards a framed TAKETHIS body without renting a growing assembly buffer.
+ /// Used when no permit is held. Still enforces
+ /// and the overall receive deadline.
+ ///
private async Task DrainArticleAsync(CancellationToken cancellationToken)
{
try
{
- var destuffed = await ReadDestuffedArticleAsync(cancellationToken).ConfigureAwait(false);
- if (destuffed.Array is not null)
+ await DiscardFramedArticleAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is EndOfStreamException or InvalidOperationException or TimeoutException)
+ {
+ // Peer closed, oversized body, or receive deadline — stop draining; caller decides response.
+ ExpectedException.Ignore(ex);
+ }
+ }
+
+ private async Task DiscardFramedArticleAsync(CancellationToken cancellationToken)
+ {
+ using var overall = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ overall.CancelAfter(_listener.ArticleReceiveTimeout);
+ try
+ {
+ await DiscardFramedCoreAsync(ArticleResourceLimits.MaxArticleBytes, overall.Token)
+ .ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested
+ && overall.IsCancellationRequested)
+ {
+ throw new TimeoutException(
+ $"NNTP TAKETHIS article receive exceeded overall timeout of {_listener.ArticleReceiveTimeout}.");
+ }
+ }
+
+ ///
+ /// Reads through the multiline terminator while counting destuffed bytes only.
+ /// Does not retain article payload beyond the fixed session read scratch.
+ ///
+ private async Task DiscardFramedCoreAsync(int maxBytes, CancellationToken cancellationToken)
+ {
+ var written = 0L;
+ var atLineStart = true;
+ byte[]? rentedInput = null;
+ byte[] input;
+ if (_pendingCount > _readScratch.Length)
+ {
+ rentedInput = ArrayPool.Shared.Rent(_pendingCount);
+ input = rentedInput;
+ }
+ else
+ {
+ input = _readScratch;
+ }
+
+ var inputCount = 0;
+ var inputOffset = 0;
+ try
+ {
+ if (_pendingCount > 0)
{
- ArrayPool.Shared.Return(destuffed.Array);
+ _pending.AsSpan(0, _pendingCount).CopyTo(input);
+ inputCount = _pendingCount;
+ _pendingCount = 0;
+ }
+
+ while (true)
+ {
+ if (inputCount - inputOffset == 0)
+ {
+ inputOffset = 0;
+ inputCount = await _transport.ReadAsync(input.AsMemory(), cancellationToken).ConfigureAwait(false);
+ if (inputCount == 0)
+ {
+ throw new EndOfStreamException("NNTP TAKETHIS ended before multiline terminator.");
+ }
+ }
+
+ var span = input.AsSpan(inputOffset, inputCount - inputOffset);
+ var index = 0;
+ var needMore = false;
+ while (index < span.Length)
+ {
+ if (atLineStart && span[index] == (byte)'.')
+ {
+ if (index + 1 >= span.Length
+ || (span[index + 1] == (byte)'\r' && index + 2 >= span.Length))
+ {
+ needMore = true;
+ break;
+ }
+
+ var next = span[index + 1];
+ if (next == (byte)'\n')
+ {
+ inputOffset += index + 2;
+ PreserveLeftover(input, inputOffset, inputCount - inputOffset);
+ return;
+ }
+
+ if (next == (byte)'\r' && span[index + 2] == (byte)'\n')
+ {
+ inputOffset += index + 3;
+ PreserveLeftover(input, inputOffset, inputCount - inputOffset);
+ return;
+ }
+
+ index += 1;
+ atLineStart = false;
+ continue;
+ }
+
+ var start = index;
+ do
+ {
+ var current = span[index++];
+ atLineStart = current is (byte)'\r' or (byte)'\n';
+ }
+ while (index < span.Length && !(atLineStart && span[index] == (byte)'.'));
+
+ var sliceLength = index - start;
+ if (written + sliceLength > maxBytes)
+ {
+ throw new InvalidOperationException("NNTP TAKETHIS article exceeds maximum size.");
+ }
+
+ written += sliceLength;
+ }
+
+ inputOffset += index;
+ if (needMore)
+ {
+ var leftover = inputCount - inputOffset;
+ if (leftover > 0 && inputOffset > 0)
+ {
+ Buffer.BlockCopy(input, inputOffset, input, 0, leftover);
+ }
+
+ inputOffset = 0;
+ inputCount = leftover;
+ var read = await _transport
+ .ReadAsync(input.AsMemory(inputCount, input.Length - inputCount), cancellationToken)
+ .ConfigureAwait(false);
+ if (read == 0)
+ {
+ throw new EndOfStreamException("NNTP TAKETHIS ended before multiline terminator.");
+ }
+
+ inputCount += read;
+ }
}
}
- catch (EndOfStreamException ex)
+ finally
{
- ExpectedException.Ignore(ex);
+ ReturnRentedInput(rentedInput);
}
}
- private Task<(byte[]? Array, ReadOnlyMemory Memory)> ReadDestuffedArticleAsync(
+ private async Task<(byte[]? Array, ReadOnlyMemory Memory)> ReadDestuffedArticleAsync(
CancellationToken cancellationToken)
- => ReadDestuffedCoreAsync(ArticleResourceLimits.MaxArticleBytes, cancellationToken);
+ {
+ using var overall = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ overall.CancelAfter(_listener.ArticleReceiveTimeout);
+ try
+ {
+ return await ReadDestuffedCoreAsync(ArticleResourceLimits.MaxArticleBytes, overall.Token)
+ .ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested
+ && overall.IsCancellationRequested)
+ {
+ throw new TimeoutException(
+ $"NNTP TAKETHIS article receive exceeded overall timeout of {_listener.ArticleReceiveTimeout}.");
+ }
+ }
+
+ private static void ReturnAssemblyBuffer(ref byte[]? rented)
+ {
+ if (rented is null)
+ {
+ return;
+ }
+
+ ArrayPool.Shared.Return(rented);
+ rented = null;
+ }
private async Task<(byte[]? Array, ReadOnlyMemory Memory)> ReadDestuffedCoreAsync(
int maxBytes,
@@ -465,7 +839,7 @@ private async Task DrainArticleAsync(CancellationToken cancellationToken)
while (index < span.Length && !(atLineStart && span[index] == (byte)'.'));
var slice = span[start..index];
- if (written > maxBytes - slice.Length)
+ if ((long)written + slice.Length > maxBytes)
{
throw new InvalidOperationException("NNTP TAKETHIS article exceeds maximum size.");
}
@@ -731,23 +1105,6 @@ private static bool TryParseMessageIdArgument(ReadOnlySpan argument, out R
return NntpMessageIdValidation.IsValidMessageId(messageId);
}
- private static bool TryMatchModeStream(ReadOnlySpan line, out bool syntaxError)
- {
- syntaxError = false;
- if (!TryMatchKeyword(line, ModeKeyword, out var modeRest))
- {
- return false;
- }
-
- if (!TryMatchKeyword(SkipSpaces(modeRest), StreamKeyword, out var streamRest))
- {
- return false;
- }
-
- syntaxError = !streamRest.IsEmpty;
- return true;
- }
-
private static bool TryMatchKeyword(
ReadOnlySpan line,
ReadOnlySpan keyword,
diff --git a/src/VectorNNTPD.StorageService/Listener/StorageNntpListenerLogMessages.cs b/src/VectorNNTPD.StorageService/Listener/StorageNntpListenerLogMessages.cs
index b8dbce0c..f5d77afb 100644
--- a/src/VectorNNTPD.StorageService/Listener/StorageNntpListenerLogMessages.cs
+++ b/src/VectorNNTPD.StorageService/Listener/StorageNntpListenerLogMessages.cs
@@ -154,5 +154,17 @@ internal static partial class StorageNntpFeedSessionLogMessages
Level = LogLevel.Debug,
Message = "Storage NNTP STORE waiting for queue capacity streamId={StreamId} bytes={Bytes}")]
public static partial void StoreAdmissionWaiting(ILogger logger, uint StreamId, int Bytes);
+
+ [LoggerMessage(
+ EventId = 5509,
+ Level = LogLevel.Warning,
+ Message = "Storage NNTP ARTICLE retrieval failed artId={ArtId} failure={Failure}")]
+ public static partial void ArticleRetrievalFailed(ILogger logger, string ArtId, string Failure);
+
+ [LoggerMessage(
+ EventId = 5510,
+ Level = LogLevel.Debug,
+ Message = "Storage NNTP ARTICLE touch failed artId={ArtId} failure={Failure}")]
+ public static partial void ArticleTouchFailed(ILogger logger, string ArtId, string Failure);
}
}
diff --git a/src/VectorNNTPD.StorageService/Listener/StorageNntpListenerService.cs b/src/VectorNNTPD.StorageService/Listener/StorageNntpListenerService.cs
index 1e9d98ba..112533bc 100644
--- a/src/VectorNNTPD.StorageService/Listener/StorageNntpListenerService.cs
+++ b/src/VectorNNTPD.StorageService/Listener/StorageNntpListenerService.cs
@@ -16,7 +16,7 @@ namespace VectorNNTPD.StorageService.Listener
{
///
/// Process-wide NNTP TLS listener: binds configured endpoints, authenticates TLS, and serves
- /// RFC 4644 STREAMING feed sessions (CHECK/TAKETHIS). Does not own article storage.
+ /// Storage sessions (ARTICLE/TAKETHIS/QUIT). Does not own article storage.
///
public sealed class StorageNntpListenerService : IApplicationService, IAsyncDisposable
{
@@ -27,7 +27,11 @@ public sealed class StorageNntpListenerService : IApplicationService, IAsyncDisp
private readonly ILogger _logger;
private readonly IAcceptedArticleQueue? _acceptedArticles;
private readonly StoreAssemblyAdmission? _assemblyAdmission;
+ private readonly Func? _allowsArticleAdmission;
private readonly Func? _articlePresent;
+ private readonly Func? _resolveRetrievableArticle;
+ private readonly Func? _afterArticleDelivered;
+ private readonly Func? _acquireArticleReadLease;
///
/// Publishes DNS after at least one socket is bound. Null skips publication.
@@ -58,8 +62,23 @@ public StorageNntpListenerService(
ILogger logger,
IAcceptedArticleQueue? acceptedArticles = null,
StoreAssemblyAdmission? assemblyAdmission = null,
- Func? articlePresent = null)
- : this(runtime, certificates, CreateReadyGate(), logger, acceptedArticles, assemblyAdmission, articlePresent)
+ Func? allowsArticleAdmission = null,
+ Func? articlePresent = null,
+ Func? resolveRetrievableArticle = null,
+ Func? afterArticleDelivered = null,
+ Func? acquireArticleReadLease = null)
+ : this(
+ runtime,
+ certificates,
+ CreateReadyGate(),
+ logger,
+ acceptedArticles,
+ assemblyAdmission,
+ allowsArticleAdmission,
+ articlePresent,
+ resolveRetrievableArticle: resolveRetrievableArticle,
+ afterArticleDelivered: afterArticleDelivered,
+ acquireArticleReadLease: acquireArticleReadLease)
{
}
@@ -71,8 +90,12 @@ public StorageNntpListenerService(
ILogger logger,
IAcceptedArticleQueue? acceptedArticles = null,
StoreAssemblyAdmission? assemblyAdmission = null,
+ Func? allowsArticleAdmission = null,
Func? articlePresent = null,
- Func? publishAfterBind = null)
+ Func? publishAfterBind = null,
+ Func? resolveRetrievableArticle = null,
+ Func? afterArticleDelivered = null,
+ Func? acquireArticleReadLease = null)
{
ArgumentNullException.ThrowIfNull(runtime);
ArgumentNullException.ThrowIfNull(certificates);
@@ -84,7 +107,11 @@ public StorageNntpListenerService(
_logger = logger;
_acceptedArticles = acceptedArticles;
_assemblyAdmission = assemblyAdmission;
+ _allowsArticleAdmission = allowsArticleAdmission;
_articlePresent = articlePresent;
+ _resolveRetrievableArticle = resolveRetrievableArticle;
+ _afterArticleDelivered = afterArticleDelivered;
+ _acquireArticleReadLease = acquireArticleReadLease;
_publishAfterBind = publishAfterBind;
}
@@ -99,8 +126,12 @@ public StorageNntpListenerService(
ILogger logger,
IAcceptedArticleQueue? acceptedArticles,
StoreAssemblyAdmission? assemblyAdmission,
+ Func? allowsArticleAdmission = null,
Func? articlePresent = null,
- Func? publishAfterBind = null)
+ Func? publishAfterBind = null,
+ Func? resolveRetrievableArticle = null,
+ Func? afterArticleDelivered = null,
+ Func? acquireArticleReadLease = null)
{
ArgumentNullException.ThrowIfNull(runtimeSource);
ArgumentNullException.ThrowIfNull(certificates);
@@ -112,7 +143,11 @@ public StorageNntpListenerService(
_logger = logger;
_acceptedArticles = acceptedArticles;
_assemblyAdmission = assemblyAdmission;
+ _allowsArticleAdmission = allowsArticleAdmission;
_articlePresent = articlePresent;
+ _resolveRetrievableArticle = resolveRetrievableArticle;
+ _afterArticleDelivered = afterArticleDelivered;
+ _acquireArticleReadLease = acquireArticleReadLease;
_publishAfterBind = publishAfterBind;
}
@@ -265,6 +300,9 @@ public async ValueTask DisposeAsync()
}
}
+ // Finish in-flight TAKETHIS (including TryCommit) before completing the queue so a
+ // reservation cannot race Complete. ApplicationServiceManager then stops the worker,
+ // which drains every accepted article before the article index stops.
await DrainConnectionsAsync().ConfigureAwait(false);
_acceptedArticles?.Complete();
await DisposeBoundResourcesAsync().ConfigureAwait(false);
@@ -438,7 +476,11 @@ private async Task ProcessAsync(Socket accepted, string remote, CancellationToke
Runtime.Fqdn,
acceptedArticles: _acceptedArticles,
assemblyAdmission: _assemblyAdmission,
- articlePresent: _articlePresent);
+ allowsArticleAdmission: _allowsArticleAdmission,
+ articlePresent: _articlePresent,
+ resolveRetrievableArticle: _resolveRetrievableArticle,
+ afterArticleDelivered: _afterArticleDelivered,
+ acquireArticleReadLease: _acquireArticleReadLease);
await session.RunAsync(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested)
diff --git a/src/VectorNNTPD.StorageService/Listener/StoreAssemblyAdmission.cs b/src/VectorNNTPD.StorageService/Listener/StoreAssemblyAdmission.cs
index 75fa81da..e7842a0a 100644
--- a/src/VectorNNTPD.StorageService/Listener/StoreAssemblyAdmission.cs
+++ b/src/VectorNNTPD.StorageService/Listener/StoreAssemblyAdmission.cs
@@ -1,3 +1,5 @@
+using VectorNNTPD.Common.Articles;
+
namespace VectorNNTPD.StorageService.Listener
{
@@ -23,6 +25,28 @@ public StoreAssemblyAdmission(int maximum)
/// Gets the current in-flight count.
public int InFlight => Volatile.Read(ref _inFlight);
+ ///
+ /// Caps concurrent assemblies so worst-case retained payload
+ /// (count × MaxArticleBytes) cannot exceed the accepted-queue byte budget
+ /// and cannot exceed .
+ ///
+ /// Listener connection ceiling.
+ /// Accepted-article queue budget.
+ /// Positive assembly concurrency limit.
+ public static int LimitFor(int maxActiveConnections, long maxQueuedBytes)
+ {
+ ArgumentOutOfRangeException.ThrowIfLessThan(maxActiveConnections, 1);
+ if (maxQueuedBytes < 1)
+ {
+ throw new ArgumentOutOfRangeException(nameof(maxQueuedBytes));
+ }
+
+ var byBudget = (int)Math.Min(
+ int.MaxValue,
+ Math.Max(1L, maxQueuedBytes / ArticleResourceLimits.MaxArticleBytes));
+ return Math.Min(maxActiveConnections, byBudget);
+ }
+
/// Reserves one assembly slot.
public bool TryEnter()
{
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleAccessCoalescer.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleAccessCoalescer.cs
index f2286567..f7921f35 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleAccessCoalescer.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleAccessCoalescer.cs
@@ -4,7 +4,7 @@ namespace VectorNNTPD.StorageService.Storage.ArticleIndex
{
///
- /// Collapses repeated touches for one article into the newest timestamp while a token command is queued.
+ /// Collapses repeated touches for one article into the newest timestamp and a summed access increment while a token command is queued.
///
///
/// The map is bounded by keys and
@@ -39,6 +39,7 @@ internal bool RequiresEnqueue(
slot.Timestamp = unixMilliseconds;
}
+ slot.AccessIncrements++;
if (slot.Count < ArticleIndexSettings.MaxCoalescedWaitersPerKey)
{
slot.Waiters[slot.Count++] = completion;
@@ -53,7 +54,7 @@ internal bool RequiresEnqueue(
return true;
}
- _pending.Add(articleId, new Slot { Timestamp = unixMilliseconds });
+ _pending.Add(articleId, new Slot { Timestamp = unixMilliseconds, AccessIncrements = 1 });
return true;
}
}
@@ -80,12 +81,14 @@ internal bool TryPeek(ArticleId articleId, out long timestamp)
/// Removes the pending touch for a token the writer is about to commit.
/// Token article identity.
/// Newest timestamp seen for that token, including joined touches.
+ /// Summed access increments for the token, including joined touches.
/// Joined completions. Null when nobody joined.
/// Number of valid entries in .
/// when no pending slot exists.
internal bool TryTake(
ArticleId articleId,
out long timestamp,
+ out int accessIncrements,
out TaskCompletionSource[]? waiters,
out int count)
{
@@ -94,18 +97,31 @@ internal bool TryTake(
if (!_pending.Remove(articleId, out var slot))
{
timestamp = 0;
+ accessIncrements = 0;
waiters = null;
count = 0;
return false;
}
timestamp = slot.Timestamp;
+ accessIncrements = slot.AccessIncrements;
waiters = slot.Count == 0 ? null : slot.Waiters;
count = slot.Count;
return true;
}
}
+ /// Returns whether a touch is queued for .
+ /// Article identity.
+ /// when a token or joined touch is pending.
+ internal bool HasPending(ArticleId articleId)
+ {
+ lock (_gate)
+ {
+ return _pending.ContainsKey(articleId);
+ }
+ }
+
///
/// Fails waiters joined to a token that never reached the queue.
/// A slot the writer already took is left alone.
@@ -164,6 +180,9 @@ private sealed class Slot
/// Gets or sets the newest access time observed for the token.
internal long Timestamp;
+ /// Gets or sets the number of coalesced access increments for the token.
+ internal int AccessIncrements;
+
/// Gets completions that joined this token instead of taking a queue slot.
internal readonly TaskCompletionSource[] Waiters =
new TaskCompletionSource[ArticleIndexSettings.MaxCoalescedWaitersPerKey];
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexDatabase.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexDatabase.cs
index 1e6abc21..a8f975fe 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexDatabase.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexDatabase.cs
@@ -34,6 +34,8 @@ internal sealed class ArticleIndexDatabase : IDisposable
private SqliteCommand? _touch;
private SqliteCommand? _exists;
private SqliteCommand? _countUpdate;
+ private SqliteCommand? _storedBytesUpdate;
+ private SqliteCommand? _evictDelete;
private SqliteCommand? _load;
private ReadSlot[]? _readers;
private SemaphoreSlim? _readerGate;
@@ -72,6 +74,9 @@ internal ArticleIndexDatabase(
/// Gets the article count stored in ArticleIndexMeta at initialization.
internal long ArticleCount { get; private set; }
+ /// Gets the stored-byte total from ArticleIndexMeta at initialization and after commits on this connection.
+ internal long StoredBytes { get; private set; }
+
/// Gets the WAL file length, or zero when the file is absent.
internal long WalBytes
{
@@ -82,7 +87,7 @@ internal long WalBytes
}
}
- /// Opens the writer, creates or checks schema version 2, and opens readers.
+ /// Opens the writer, creates or migrates schema version 3, and opens readers.
/// Observed before the file is opened. The SQLite calls themselves are synchronous.
internal void Initialize(CancellationToken cancellationToken)
{
@@ -221,14 +226,21 @@ internal void Commit(IReadOnlyList mutations)
var touch = _touch!;
var exists = _exists!;
var countUpdate = _countUpdate!;
+ var storedBytesUpdate = _storedBytesUpdate!;
+ var evictDelete = _evictDelete!;
using var transaction = connection.BeginTransaction();
insert.Transaction = transaction;
touch.Transaction = transaction;
exists.Transaction = transaction;
countUpdate.Transaction = transaction;
+ storedBytesUpdate.Transaction = transaction;
+ evictDelete.Transaction = transaction;
try
{
var inserted = 0;
+ long insertedBytes = 0;
+ var evicted = 0;
+ long evictedBytes = 0;
foreach (var mutation in mutations)
{
if (mutation.HasInsert)
@@ -238,6 +250,7 @@ internal void Commit(IReadOnlyList mutations)
var existed = exists.ExecuteScalar() is not null and not DBNull;
insert.Parameters[0].Value = _writerHash;
insert.Parameters[1].Value = mutation.ReceivedAt;
+ insert.Parameters[2].Value = mutation.ArtSizeBytes;
var changed = insert.ExecuteNonQuery();
if (existed)
{
@@ -246,6 +259,7 @@ internal void Commit(IReadOnlyList mutations)
else if (changed == 1)
{
inserted++;
+ insertedBytes += mutation.ArtSizeBytes;
mutation.InsertDisposition = ArticleIndexWriteDisposition.Inserted;
}
else
@@ -254,32 +268,54 @@ internal void Commit(IReadOnlyList mutations)
}
}
- if (!mutation.HasTouch)
+ if (mutation.HasTouch)
+ {
+ mutation.Id.CopyTo(_writerHash);
+ var delta = mutation.AccessIncrement <= 0 ? 1 : mutation.AccessIncrement;
+ touch.Parameters[0].Value = delta;
+ touch.Parameters[1].Value = mutation.LastAccessedAt;
+ touch.Parameters[2].Value = _writerHash;
+ var updated = touch.ExecuteNonQuery();
+ if (updated == 1)
+ {
+ mutation.TouchDisposition = ArticleIndexWriteDisposition.Updated;
+ }
+ else if (mutation.HasInsert)
+ {
+ mutation.TouchDisposition = ArticleIndexWriteDisposition.Unchanged;
+ }
+ else
+ {
+ exists.Parameters[0].Value = _writerHash;
+ var found = exists.ExecuteScalar();
+ mutation.TouchDisposition = found is null or DBNull
+ ? ArticleIndexWriteDisposition.Missing
+ : ArticleIndexWriteDisposition.Unchanged;
+ }
+ }
+
+ if (!mutation.HasEvictDelete)
{
continue;
}
mutation.Id.CopyTo(_writerHash);
- touch.Parameters[0].Value = mutation.LastAccessedAt;
- touch.Parameters[1].Value = _writerHash;
- var updated = touch.ExecuteNonQuery();
- if (updated == 1)
+ evictDelete.Parameters[0].Value = _writerHash;
+ object? deletedSize;
+ using (var reader = evictDelete.ExecuteReader())
{
- mutation.TouchDisposition = ArticleIndexWriteDisposition.Updated;
- continue;
+ deletedSize = reader.Read() ? reader.GetInt64(0) : null;
}
- if (mutation.HasInsert)
+ if (deletedSize is null)
{
- mutation.TouchDisposition = ArticleIndexWriteDisposition.Unchanged;
+ mutation.EvictDeleteDisposition = ArticleIndexWriteDisposition.Missing;
continue;
}
- exists.Parameters[0].Value = _writerHash;
- var found = exists.ExecuteScalar();
- mutation.TouchDisposition = found is null or DBNull
- ? ArticleIndexWriteDisposition.Missing
- : ArticleIndexWriteDisposition.Unchanged;
+ evicted++;
+ evictedBytes += (long)deletedSize;
+ mutation.EvictDeleteDisposition = ArticleIndexWriteDisposition.Updated;
}
if (inserted > 0)
@@ -291,11 +327,40 @@ internal void Commit(IReadOnlyList mutations)
}
}
+ if (insertedBytes > 0)
+ {
+ storedBytesUpdate.Parameters[0].Value = insertedBytes;
+ if (storedBytesUpdate.ExecuteNonQuery() != 1)
+ {
+ throw new InvalidOperationException("Article index metadata row is missing.");
+ }
+ }
+
+ if (evicted > 0)
+ {
+ countUpdate.Parameters[0].Value = -evicted;
+ if (countUpdate.ExecuteNonQuery() != 1)
+ {
+ throw new InvalidOperationException("Article index metadata row is missing.");
+ }
+ }
+
+ if (evictedBytes > 0)
+ {
+ storedBytesUpdate.Parameters[0].Value = -evictedBytes;
+ if (storedBytesUpdate.ExecuteNonQuery() != 1)
+ {
+ throw new InvalidOperationException("Article index metadata row is missing.");
+ }
+ }
+
if (CommitHookApplies())
{
_settings.BeforeCommit!.Invoke();
}
transaction.Commit();
+ ArticleCount += inserted - evicted;
+ StoredBytes += insertedBytes - evictedBytes;
}
catch
{
@@ -317,6 +382,8 @@ internal void Commit(IReadOnlyList mutations)
touch.Transaction = null;
exists.Transaction = null;
countUpdate.Transaction = null;
+ storedBytesUpdate.Transaction = null;
+ evictDelete.Transaction = null;
}
}
@@ -344,7 +411,9 @@ record = default;
articleId,
reader.GetInt64(0),
reader.GetInt64(1),
- _spoolDirectory);
+ _spoolDirectory,
+ reader.GetInt64(2),
+ reader.GetInt64(3));
return true;
}
finally
@@ -418,7 +487,71 @@ internal int CopyOldest(
articleId,
reader.GetInt64(1),
accessed,
- _spoolDirectory);
+ _spoolDirectory,
+ reader.GetInt64(3),
+ reader.GetInt64(4));
+ }
+
+ return count;
+ }
+ finally
+ {
+ counters?.NoteRead(System.Diagnostics.Stopwatch.GetElapsedTime(started));
+ Return(slot.Index);
+ }
+ }
+
+ /// Copies rows in LFU order with optional inclusion filter.
+ /// Buffer filled in ascending LFU order.
+ /// Optional read timer.
+ /// Returns false to skip a row and keep scanning.
+ /// The number of rows written to .
+ internal int CopyLeastFrequentlyUsed(
+ Span destination,
+ ArticleIndexCounters? counters,
+ Func? include)
+ {
+ if (destination.Length == 0)
+ {
+ return 0;
+ }
+
+ if (destination.Length > 10_000)
+ {
+ throw new ArgumentOutOfRangeException(nameof(destination), "Article index eviction reads are limited to 10000 rows.");
+ }
+
+ var started = System.Diagnostics.Stopwatch.GetTimestamp();
+ var slot = Rent();
+ try
+ {
+ var command = slot.Lfu;
+ var limit = include is null ? destination.Length : 10_000;
+ command.Parameters[0].Value = limit;
+ using var reader = command.ExecuteReader();
+ var count = 0;
+ while (reader.Read() && count < destination.Length)
+ {
+ var length = reader.GetBytes(0, 0, slot.Hash, 0, slot.Hash.Length);
+ if (length != ArticleIndexSettings.HashLength)
+ {
+ throw new InvalidOperationException("Article index returned a hash that is not 32 bytes.");
+ }
+
+ var articleId = ArticleId.FromSpan(slot.Hash);
+ var accessed = reader.GetInt64(2);
+ if (include is not null && !include(articleId, accessed))
+ {
+ continue;
+ }
+
+ destination[count++] = new ArticleIndexRecord(
+ articleId,
+ reader.GetInt64(1),
+ accessed,
+ _spoolDirectory,
+ reader.GetInt64(3),
+ reader.GetInt64(4));
}
return count;
@@ -548,34 +681,117 @@ private void ApplyWriterPragmas(SqliteConnection connection, bool pageSizeFirst)
PageSize = Convert.ToInt32(ExecuteScalar(connection, "PRAGMA page_size;"), System.Globalization.CultureInfo.InvariantCulture);
}
- /// Creates schema version 2 or refuses a different version. An unknown version is not modified.
+ /// Creates schema version 3, migrates version 2, or refuses an unknown version.
/// Open writer connection.
private void Migrate(SqliteConnection connection)
{
- using var transaction = connection.BeginTransaction();
- Execute(connection, ArticleIndexSql.CreateMeta, transaction);
- Execute(connection, ArticleIndexSql.CreateArticles, transaction);
- Execute(connection, ArticleIndexSql.CreateAccessIndex, transaction);
- Execute(connection, ArticleIndexSql.InsertMeta, transaction);
- transaction.Commit();
+ // Schema DDL stays inside one writer transaction so no reader remains open across
+ // CREATE/ALTER/INDEX (avoids SQLite Error 6: database table is locked).
+ using (var transaction = connection.BeginTransaction())
+ {
+ Execute(connection, ArticleIndexSql.CreateMeta, transaction);
+ Execute(connection, ArticleIndexSql.CreateArticles, transaction);
+ EnsureV3Columns(connection, transaction);
+ Execute(connection, ArticleIndexSql.InsertMeta, transaction);
+ var version = ReadMetaVersion(connection, transaction);
+ if (version == 2)
+ {
+ Execute(connection, ArticleIndexSql.SetSchemaVersion3, transaction);
+ }
+ else if (version != ArticleIndexSettings.SchemaVersion)
+ {
+ throw new InvalidOperationException(
+ $"Article index '{DatabasePath}' has schema version {version}. This build supports version {ArticleIndexSettings.SchemaVersion} only and does not modify a newer or older database.");
+ }
+
+ Execute(connection, ArticleIndexSql.CreateLfuIndex, transaction);
+ Execute(connection, ArticleIndexSql.DropLastAccessedIndex, transaction);
+ transaction.Commit();
+ }
+
+ ReadMeta(connection, out var schemaVersion, out var articleCount, out var storedBytes);
+ SchemaVersion = schemaVersion;
+ ArticleCount = articleCount;
+ StoredBytes = storedBytes;
+ }
+
+ /// Reads only the schema version while an optional writer transaction is open.
+ private static int ReadMetaVersion(SqliteConnection connection, SqliteTransaction? transaction)
+ {
using var command = connection.CreateCommand();
+ command.Transaction = transaction;
command.CommandText = ArticleIndexSql.ReadMeta;
using var reader = command.ExecuteReader();
if (!reader.Read())
{
- throw new InvalidOperationException($"Article index '{DatabasePath}' has no schema version row.");
+ throw new InvalidOperationException("Article index metadata row is missing.");
}
- var version = reader.GetInt32(0);
- if (version != ArticleIndexSettings.SchemaVersion)
+ return reader.GetInt32(0);
+ }
+
+ /// Reads metadata and closes the reader before further writer statements run.
+ private static void ReadMeta(
+ SqliteConnection connection,
+ out int version,
+ out long articleCount,
+ out long storedBytes)
+ {
+ using var command = connection.CreateCommand();
+ command.CommandText = ArticleIndexSql.ReadMeta;
+ using var reader = command.ExecuteReader();
+ if (!reader.Read())
+ {
+ throw new InvalidOperationException("Article index metadata row is missing.");
+ }
+
+ version = reader.GetInt32(0);
+ articleCount = reader.GetInt64(1);
+ storedBytes = reader.FieldCount > 2 && !reader.IsDBNull(2) ? reader.GetInt64(2) : 0;
+ }
+
+ /// Adds version-3 columns when opening an older on-disk layout.
+ /// Open writer connection.
+ /// Optional transaction.
+ private static void EnsureV3Columns(SqliteConnection connection, SqliteTransaction? transaction = null)
+ {
+ if (!ColumnExists(connection, "ArticleIndexMeta", "StoredBytes", transaction))
+ {
+ Execute(connection, ArticleIndexSql.AddMetaStoredBytes, transaction);
+ }
+
+ if (!ColumnExists(connection, "ArticleIndex", "AccessCount", transaction))
{
- throw new InvalidOperationException(
- $"Article index '{DatabasePath}' has schema version {version}. This build supports version {ArticleIndexSettings.SchemaVersion} only and does not modify a newer or older database.");
+ Execute(connection, ArticleIndexSql.AddAccessCount, transaction);
}
- SchemaVersion = version;
- ArticleCount = reader.GetInt64(1);
+ if (!ColumnExists(connection, "ArticleIndex", "ArtSizeBytes", transaction))
+ {
+ Execute(connection, ArticleIndexSql.AddArtSizeBytes, transaction);
+ }
+ }
+
+ /// Returns whether a table already has a column.
+ private static bool ColumnExists(
+ SqliteConnection connection,
+ string table,
+ string column,
+ SqliteTransaction? transaction = null)
+ {
+ using var command = connection.CreateCommand();
+ command.Transaction = transaction;
+ command.CommandText = $"PRAGMA table_info({table});";
+ using var reader = command.ExecuteReader();
+ while (reader.Read())
+ {
+ if (string.Equals(reader.GetString(1), column, StringComparison.Ordinal))
+ {
+ return true;
+ }
+ }
+
+ return false;
}
/// Prepares the writer statements used for every batch and for the startup scan.
@@ -585,6 +801,7 @@ private void PrepareWriter(SqliteConnection connection)
_insert = Prepare(connection, ArticleIndexSql.Insert);
AddBlob(_insert, "$hash");
_insert.Parameters.Add("$received", SqliteType.Integer);
+ _insert.Parameters.Add("$artSize", SqliteType.Integer);
_import = Prepare(connection, ArticleIndexSql.Import);
AddBlob(_import, "$hash");
@@ -592,6 +809,7 @@ private void PrepareWriter(SqliteConnection connection)
_import.Parameters.Add("$accessed", SqliteType.Integer);
_touch = Prepare(connection, ArticleIndexSql.Touch);
+ _touch.Parameters.Add("$delta", SqliteType.Integer);
_touch.Parameters.Add("$accessed", SqliteType.Integer);
AddBlob(_touch, "$hash");
@@ -601,6 +819,12 @@ private void PrepareWriter(SqliteConnection connection)
_countUpdate = Prepare(connection, ArticleIndexSql.AddCount);
_countUpdate.Parameters.Add("$delta", SqliteType.Integer);
+ _storedBytesUpdate = Prepare(connection, ArticleIndexSql.AddStoredBytes);
+ _storedBytesUpdate.Parameters.Add("$delta", SqliteType.Integer);
+
+ _evictDelete = Prepare(connection, ArticleIndexSql.EvictDelete);
+ AddBlob(_evictDelete, "$hash");
+
_load = Prepare(connection, ArticleIndexSql.LoadHashes);
}
@@ -634,6 +858,7 @@ private void OpenReaders()
slot.Lookup.Dispose();
slot.Oldest.Dispose();
slot.Before.Dispose();
+ slot.Lfu.Dispose();
slot.Scan.Dispose();
slot.Connection.Dispose();
}
@@ -664,6 +889,7 @@ private void OpenAndPublishReader(int index, SqliteConnectionStringBuilder build
SqliteCommand? lookup = null;
SqliteCommand? oldest = null;
SqliteCommand? before = null;
+ SqliteCommand? lfu = null;
SqliteCommand? scan = null;
try
{
@@ -679,6 +905,8 @@ private void OpenAndPublishReader(int index, SqliteConnectionStringBuilder build
before = Prepare(connection, ArticleIndexSql.AccessedBefore);
before.Parameters.Add("$threshold", SqliteType.Integer);
before.Parameters.Add("$limit", SqliteType.Integer);
+ lfu = Prepare(connection, ArticleIndexSql.LeastFrequentlyUsed);
+ lfu.Parameters.Add("$limit", SqliteType.Integer);
scan = Prepare(connection, ArticleIndexSql.ScanLocations);
_readers![index] = new ReadSlot(
index,
@@ -686,6 +914,7 @@ private void OpenAndPublishReader(int index, SqliteConnectionStringBuilder build
lookup,
oldest,
before,
+ lfu,
scan,
new byte[ArticleIndexSettings.HashLength]);
_freeReaders!.Push(index);
@@ -694,6 +923,7 @@ private void OpenAndPublishReader(int index, SqliteConnectionStringBuilder build
catch
{
scan?.Dispose();
+ lfu?.Dispose();
before?.Dispose();
oldest?.Dispose();
lookup?.Dispose();
@@ -738,6 +968,7 @@ private void DisposeReaders()
slot.Lookup.Dispose();
slot.Oldest.Dispose();
slot.Before.Dispose();
+ slot.Lfu.Dispose();
slot.Scan.Dispose();
slot.Connection.Dispose();
}
@@ -753,12 +984,16 @@ private void DisposeWriterCommands()
_touch?.Dispose();
_exists?.Dispose();
_countUpdate?.Dispose();
+ _storedBytesUpdate?.Dispose();
+ _evictDelete?.Dispose();
_load?.Dispose();
_insert = null;
_import = null;
_touch = null;
_exists = null;
_countUpdate = null;
+ _storedBytesUpdate = null;
+ _evictDelete = null;
_load = null;
}
@@ -818,6 +1053,7 @@ private static void AddBlob(SqliteCommand command, string name)
/// Primary-key lookup.
/// Oldest rows.
/// Rows at or before an access time.
+ /// Least-frequently-used rows.
/// Hash and spool scan.
/// Reusable 32-byte buffer. It is not shared with another slot.
private readonly record struct ReadSlot(
@@ -826,6 +1062,7 @@ private readonly record struct ReadSlot(
SqliteCommand Lookup,
SqliteCommand Oldest,
SqliteCommand Before,
+ SqliteCommand Lfu,
SqliteCommand Scan,
byte[] Hash);
}
@@ -833,74 +1070,109 @@ private readonly record struct ReadSlot(
/// Literal SQL for the article index. Values are bound, not interpolated.
internal static class ArticleIndexSql
{
- /// Version and article-count row.
+ /// Version, article-count, and stored-byte row.
internal const string CreateMeta =
"""
CREATE TABLE IF NOT EXISTS ArticleIndexMeta (
Id INTEGER NOT NULL CONSTRAINT PK_ArticleIndexMeta PRIMARY KEY CHECK (Id = 1),
SchemaVersion INTEGER NOT NULL,
- ArticleCount INTEGER NOT NULL CHECK (ArticleCount >= 0)
+ ArticleCount INTEGER NOT NULL CHECK (ArticleCount >= 0),
+ StoredBytes INTEGER NOT NULL DEFAULT 0 CHECK (StoredBytes >= 0)
) WITHOUT ROWID;
""";
- /// Article rows keyed by the 32-byte BLAKE3 digest.
+ /// Article rows keyed by the 32-byte BLAKE3 digest (schema version 3 shape).
+ ///
+ /// CREATE TABLE IF NOT EXISTS leaves a pre-existing version-2 table unchanged;
+ /// then adds missing columns via ALTER TABLE.
+ ///
internal const string CreateArticles =
"""
CREATE TABLE IF NOT EXISTS ArticleIndex (
MessageIdHash BLOB NOT NULL CONSTRAINT PK_ArticleIndex PRIMARY KEY,
ReceivedAt INTEGER NOT NULL,
LastAccessedAt INTEGER NOT NULL,
+ AccessCount INTEGER NOT NULL DEFAULT 1 CHECK (AccessCount >= 1),
+ ArtSizeBytes INTEGER NOT NULL DEFAULT 0 CHECK (ArtSizeBytes >= 0),
CONSTRAINT CK_ArticleIndex_HashLength CHECK (length(MessageIdHash) = 32),
CONSTRAINT CK_ArticleIndex_Received CHECK (ReceivedAt >= 0),
CONSTRAINT CK_ArticleIndex_Accessed CHECK (LastAccessedAt >= 0)
) WITHOUT ROWID;
""";
- /// Supports oldest-access and access-threshold scans.
- internal const string CreateAccessIndex =
+ /// LFU eviction ordering index.
+ internal const string CreateLfuIndex =
+ """
+ CREATE INDEX IF NOT EXISTS IX_ArticleIndex_Lfu
+ ON ArticleIndex (AccessCount, LastAccessedAt, MessageIdHash);
+ """;
+
+ internal const string DropLastAccessedIndex =
+ """
+ DROP INDEX IF EXISTS IX_ArticleIndex_LastAccessedAt;
+ """;
+
+ internal const string AddMetaStoredBytes =
+ """
+ ALTER TABLE ArticleIndexMeta ADD COLUMN StoredBytes INTEGER NOT NULL DEFAULT 0;
+ """;
+
+ internal const string AddAccessCount =
+ """
+ ALTER TABLE ArticleIndex ADD COLUMN AccessCount INTEGER NOT NULL DEFAULT 1;
+ """;
+
+ internal const string AddArtSizeBytes =
"""
- CREATE INDEX IF NOT EXISTS IX_ArticleIndex_LastAccessedAt
- ON ArticleIndex (LastAccessedAt);
+ ALTER TABLE ArticleIndex ADD COLUMN ArtSizeBytes INTEGER NOT NULL DEFAULT 0;
""";
- /// Inserts schema version 2 when the metadata row is absent.
+ internal const string SetSchemaVersion3 =
+ """
+ UPDATE ArticleIndexMeta SET SchemaVersion = 3 WHERE Id = 1;
+ """;
+
+ /// Inserts schema version 3 when the metadata row is absent.
internal const string InsertMeta =
"""
- INSERT INTO ArticleIndexMeta (Id, SchemaVersion, ArticleCount)
- VALUES (1, 2, 0)
+ INSERT INTO ArticleIndexMeta (Id, SchemaVersion, ArticleCount, StoredBytes)
+ VALUES (1, 3, 0, 0)
ON CONFLICT(Id) DO NOTHING;
""";
- /// Reads the schema version and article count.
+ /// Reads the schema version, article count, and stored-byte total.
internal const string ReadMeta =
"""
- SELECT SchemaVersion, ArticleCount FROM ArticleIndexMeta WHERE Id = 1;
+ SELECT SchemaVersion, ArticleCount, StoredBytes FROM ArticleIndexMeta WHERE Id = 1;
""";
- /// Inserts a new article. A conflict keeps ReceivedAt and LastAccessedAt.
+ /// Inserts a new article. A conflict keeps the original row.
internal const string Insert =
"""
- INSERT INTO ArticleIndex (MessageIdHash, ReceivedAt, LastAccessedAt)
- VALUES ($hash, $received, $received)
+ INSERT INTO ArticleIndex (MessageIdHash, ReceivedAt, LastAccessedAt, AccessCount, ArtSizeBytes)
+ VALUES ($hash, $received, $received, 1, $artSize)
ON CONFLICT(MessageIdHash) DO UPDATE SET
ReceivedAt = ArticleIndex.ReceivedAt,
- LastAccessedAt = ArticleIndex.LastAccessedAt;
+ LastAccessedAt = ArticleIndex.LastAccessedAt,
+ AccessCount = ArticleIndex.AccessCount,
+ ArtSizeBytes = ArticleIndex.ArtSizeBytes;
""";
/// Inserts a migrated row with its original receipt and access times.
internal const string Import =
"""
- INSERT INTO ArticleIndex (MessageIdHash, ReceivedAt, LastAccessedAt)
- VALUES ($hash, $received, $accessed)
+ INSERT INTO ArticleIndex (MessageIdHash, ReceivedAt, LastAccessedAt, AccessCount, ArtSizeBytes)
+ VALUES ($hash, $received, $accessed, 1, 0)
ON CONFLICT(MessageIdHash) DO NOTHING;
""";
- /// Moves the access time forward. ReceivedAt is not assigned.
+ /// Increments access count and moves the access time forward when newer.
internal const string Touch =
"""
UPDATE ArticleIndex
- SET LastAccessedAt = $accessed
- WHERE MessageIdHash = $hash AND LastAccessedAt < $accessed;
+ SET AccessCount = AccessCount + $delta,
+ LastAccessedAt = CASE WHEN LastAccessedAt < $accessed THEN $accessed ELSE LastAccessedAt END
+ WHERE MessageIdHash = $hash;
""";
/// Distinguishes a missing touch from an unchanged row.
@@ -915,6 +1187,20 @@ UPDATE ArticleIndex
UPDATE ArticleIndexMeta SET ArticleCount = ArticleCount + $delta WHERE Id = 1;
""";
+ /// Adjusts the metadata stored-byte total in the same transaction as row changes.
+ internal const string AddStoredBytes =
+ """
+ UPDATE ArticleIndexMeta SET StoredBytes = StoredBytes + $delta WHERE Id = 1;
+ """;
+
+ /// Deletes one evicted row and returns its indexed size.
+ internal const string EvictDelete =
+ """
+ DELETE FROM ArticleIndex
+ WHERE MessageIdHash = $hash
+ RETURNING ArtSizeBytes;
+ """;
+
/// Startup membership scan. No other columns are selected.
internal const string LoadHashes =
"""
@@ -924,7 +1210,7 @@ UPDATE ArticleIndex
/// Primary-key location lookup.
internal const string Lookup =
"""
- SELECT ReceivedAt, LastAccessedAt
+ SELECT ReceivedAt, LastAccessedAt, AccessCount, ArtSizeBytes
FROM ArticleIndex
WHERE MessageIdHash = $hash;
""";
@@ -932,19 +1218,28 @@ FROM ArticleIndex
/// Oldest committed rows.
internal const string Oldest =
"""
- SELECT MessageIdHash, ReceivedAt, LastAccessedAt
+ SELECT MessageIdHash, ReceivedAt, LastAccessedAt, AccessCount, ArtSizeBytes
FROM ArticleIndex
- ORDER BY LastAccessedAt
+ ORDER BY LastAccessedAt, MessageIdHash
LIMIT $limit;
""";
/// Rows whose access time is at or before a threshold, oldest first.
internal const string AccessedBefore =
"""
- SELECT MessageIdHash, ReceivedAt, LastAccessedAt
+ SELECT MessageIdHash, ReceivedAt, LastAccessedAt, AccessCount, ArtSizeBytes
FROM ArticleIndex
WHERE LastAccessedAt <= $threshold
- ORDER BY LastAccessedAt
+ ORDER BY LastAccessedAt, MessageIdHash
+ LIMIT $limit;
+ """;
+
+ /// LFU rows with recency and hash tie-breakers.
+ internal const string LeastFrequentlyUsed =
+ """
+ SELECT MessageIdHash, ReceivedAt, LastAccessedAt, AccessCount, ArtSizeBytes
+ FROM ArticleIndex
+ ORDER BY AccessCount, LastAccessedAt, MessageIdHash
LIMIT $limit;
""";
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexLogMessages.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexLogMessages.cs
index 97168443..79a25a7b 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexLogMessages.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexLogMessages.cs
@@ -137,5 +137,13 @@ public static partial void Summary(
/// Database directory. The legacy file was renamed, not deleted.
[LoggerMessage(EventId = 5812, Level = LogLevel.Warning, Message = "Article index migrated legacy rows={Rows} databases={Databases} directory={Directory}")]
public static partial void LegacyMigrated(ILogger logger, int rows, int databases, string directory);
+
+ /// Logs a spool file delete failure during eviction.
+ [LoggerMessage(EventId = 5813, Level = LogLevel.Warning, Message = "Article index eviction file delete failed articleId={ArticleId} error={Error}")]
+ public static partial void EvictionFileDeleteFailed(ILogger logger, string ArticleId, string Error);
+
+ /// Logs an index delete failure after the spool file was already removed.
+ [LoggerMessage(EventId = 5814, Level = LogLevel.Error, Message = "Article index eviction row delete failed articleId={ArticleId} spool={Spool}")]
+ public static partial void EvictionIndexDeleteFailed(ILogger logger, string articleId, string spool);
}
}
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexModels.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexModels.cs
index afdb74bc..4b9475bb 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexModels.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexModels.cs
@@ -9,8 +9,11 @@ internal enum ArticleIndexCommandKind : byte
/// Insert a spool-published article. Duplicate keys keep the original receipt time.
Insert = 1,
- /// Move LastAccessedAt forward. A missing row is not created.
+ /// Increment access count and move LastAccessedAt forward when newer. A missing row is not created.
Touch = 2,
+
+ /// Remove one evicted row after its spool file was deleted.
+ EvictDelete = 3,
}
/// Outcome of one committed mutation after the transaction commits.
@@ -60,11 +63,15 @@ internal enum ArticleIndexTouchDisposition : byte
/// UTC Unix epoch milliseconds of first index insert.
/// UTC Unix epoch milliseconds of the newest committed access.
/// Validated spool root. Not the article file path.
+ /// Committed access count used for LFU ordering.
+ /// Indexed payload size in bytes.
internal readonly record struct ArticleIndexRecord(
ArticleId ArticleId,
long ReceivedAtUnixMilliseconds,
long LastAccessedAtUnixMilliseconds,
- string SpoolDirectory);
+ string SpoolDirectory,
+ long AccessCount = 1,
+ long ArtSizeBytes = 0);
/// One wal_checkpoint result row.
/// Non-zero when SQLite could not checkpoint every frame.
@@ -110,12 +117,19 @@ internal sealed class ArticleIndexRequest
/// Canonical article identity.
/// UTC Unix epoch milliseconds supplied by the caller.
/// Spool root for an insert. Null for a touch.
- internal ArticleIndexRequest(ArticleIndexCommandKind kind, ArticleId id, long unixMilliseconds, string? spoolDirectory)
+ /// Payload size for an insert.
+ internal ArticleIndexRequest(
+ ArticleIndexCommandKind kind,
+ ArticleId id,
+ long unixMilliseconds,
+ string? spoolDirectory,
+ long artSizeBytes = 0)
{
Kind = kind;
Id = id;
UnixMilliseconds = unixMilliseconds;
SpoolDirectory = spoolDirectory;
+ ArtSizeBytes = artSizeBytes;
Completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
_ = Completion.Task.ContinueWith(
static task => _ = task.Exception,
@@ -136,6 +150,9 @@ internal ArticleIndexRequest(ArticleIndexCommandKind kind, ArticleId id, long un
/// Gets the spool root for an insert.
internal string? SpoolDirectory { get; }
+ /// Gets the payload size supplied for an insert.
+ internal long ArtSizeBytes { get; }
+
/// Gets the commit completion. Set by the writer after the transaction outcome is known.
internal TaskCompletionSource Completion { get; }
}
@@ -169,12 +186,24 @@ internal ArticleIndexMutation(ArticleId id)
/// Gets or sets the newest access timestamp in the batch.
internal long LastAccessedAt { get; set; }
+ /// Gets or sets the summed access increments for touches in this batch.
+ internal int AccessIncrement { get; set; }
+
/// Gets or sets the spool root from the first insert.
internal string? SpoolDirectory { get; set; }
+ /// Gets or sets the indexed payload size for the first insert in this batch.
+ internal long ArtSizeBytes { get; set; }
+
/// Gets a value indicating whether a touch should run after the insert.
internal bool HasTouch { get; set; }
+ /// Gets a value indicating whether this batch deletes an evicted row.
+ internal bool HasEvictDelete { get; set; }
+
+ /// Gets or sets the evict-delete outcome. Meaningful after a successful commit.
+ internal ArticleIndexWriteDisposition EvictDeleteDisposition { get; set; }
+
/// Gets or sets the insert statement outcome. Meaningful after a successful commit.
internal ArticleIndexWriteDisposition InsertDisposition { get; set; }
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexService.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexService.cs
index f46d5cc0..fffc4c56 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexService.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexService.cs
@@ -38,6 +38,7 @@ internal sealed class ArticleIndexService : IArticleIndex, IApplicationService
private readonly ArticleIndexCounters _counters = new();
private readonly ArticlePresenceIndex _presence = new();
private readonly ArticleIngestionReservations _reservations = new();
+ private readonly SpoolLfuEviction? _eviction;
private readonly List _spools = new();
private readonly Dictionary _bySpool = new(PathComparer);
private readonly SemaphoreSlim _checkpointGate = new(1, 1);
@@ -53,14 +54,17 @@ internal sealed class ArticleIndexService : IArticleIndex, IApplicationService
/// Validated spool roots. One database is opened for each root.
/// Startup and writer logger.
/// Clock for reconciliation elapsed time. Each writer uses the same clock.
+ /// Filesystem-threshold LFU eviction helper.
public ArticleIndexService(
IOptions options,
SegmentPoolCatalog catalog,
ILogger logger,
- TimeProvider time)
- : this(options.Value, catalog, logger, time, settings: null)
+ TimeProvider time,
+ SpoolLfuEviction eviction)
+ : this(options.Value, catalog, logger, time, settings: null, eviction)
{
ArgumentNullException.ThrowIfNull(options);
+ ArgumentNullException.ThrowIfNull(eviction);
}
/// Initializes the service with explicit options and test settings.
@@ -69,12 +73,14 @@ public ArticleIndexService(
/// Startup and writer logger.
/// Clock.
/// Null uses defaults.
+ /// Null disables capacity eviction in tests.
internal ArticleIndexService(
StorageServerOptions options,
SegmentPoolCatalog catalog,
ILogger logger,
TimeProvider time,
- ArticleIndexSettings? settings)
+ ArticleIndexSettings? settings,
+ SpoolLfuEviction? eviction = null)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(catalog);
@@ -85,6 +91,7 @@ internal ArticleIndexService(
_logger = logger;
_time = time;
_settings = settings ?? new ArticleIndexSettings();
+ _eviction = eviction;
DatabaseDirectory = string.Empty;
DatabasePath = string.Empty;
}
@@ -348,10 +355,12 @@ public ValueTask InsertStoredAsync(
ArticleId articleId,
string spoolDirectory,
long receivedAtUnixMilliseconds,
- CancellationToken cancellationToken)
+ CancellationToken cancellationToken,
+ long artSizeBytes = 0)
{
ThrowIfNotReady();
- return Require(spoolDirectory).WriterOrThrow().InsertAsync(articleId, spoolDirectory, receivedAtUnixMilliseconds, cancellationToken);
+ return Require(spoolDirectory).WriterOrThrow()
+ .InsertAsync(articleId, spoolDirectory, receivedAtUnixMilliseconds, artSizeBytes, cancellationToken);
}
///
@@ -441,6 +450,73 @@ public int CopyAccessedAtOrBefore(string spoolDirectory, long accessedAtOrBefore
return spool.Database.CopyOldest(accessedAtOrBeforeUnixMilliseconds, destination, spool.Counters, spool.Include);
}
+ ///
+ public int CopyLeastFrequentlyUsed(string spoolDirectory, Span destination)
+ {
+ ThrowIfNotReady();
+ var spool = Require(spoolDirectory);
+ return spool.Database.CopyLeastFrequentlyUsed(
+ destination,
+ spool.Counters,
+ (id, accessed) => IsEvictionCandidate(spoolDirectory, id, accessed));
+ }
+
+ ///
+ public ValueTask EvictUntilBelowHighAsync(string spoolDirectory, CancellationToken cancellationToken)
+ {
+ ThrowIfNotReady();
+ if (_eviction is null)
+ {
+ return ValueTask.CompletedTask;
+ }
+
+ return _eviction.EvictUntilBelowHighAsync(this, spoolDirectory, cancellationToken);
+ }
+
+ /// Gets the stored-byte total recorded in one spool database.
+ internal long StoredBytesFor(string spoolDirectory) => Require(spoolDirectory).Database.StoredBytes;
+
+ /// Returns whether has an active ingestion reservation.
+ internal bool IsIngestionReserved(ArticleId articleId) => _reservations.IsReserved(articleId);
+
+ /// Returns whether an indexed row may be considered for eviction.
+ internal bool IsEvictionCandidate(string spoolDirectory, ArticleId articleId, long accessedAt)
+ {
+ var spool = Require(spoolDirectory);
+ return spool.Include(articleId, accessedAt) && !spool.Coalescer.HasPending(articleId);
+ }
+
+ /// Deletes one evicted index row on the writer thread.
+ internal ValueTask DeleteEvictedRowAsync(
+ string spoolDirectory,
+ ArticleId articleId,
+ CancellationToken cancellationToken) =>
+ Require(spoolDirectory).WriterOrThrow().EvictDeleteAsync(articleId, cancellationToken);
+
+ /// Evicts one LFU-eligible article when possible.
+ internal async ValueTask TryEvictOneAsync(
+ string spoolDirectory,
+ SpoolLfuEviction eviction,
+ CancellationToken cancellationToken)
+ {
+ var spool = Require(spoolDirectory);
+ var buffer = new ArticleIndexRecord[128];
+ var count = spool.Database.CopyLeastFrequentlyUsed(
+ buffer,
+ spool.Counters,
+ (id, accessed) => eviction.IsEligible(this, spoolDirectory, id, accessed));
+ for (var i = 0; i < count; i++)
+ {
+ if (await eviction.EvictCandidateAsync(this, spoolDirectory, buffer[i], cancellationToken)
+ .ConfigureAwait(false))
+ {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
///
public string DatabasePathFor(string spoolDirectory) => Require(spoolDirectory).Binding.DatabasePath;
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexSettings.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexSettings.cs
index 7030fbc3..f664a660 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexSettings.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexSettings.cs
@@ -20,7 +20,10 @@ internal sealed class ArticleIndexSettings
internal const string SpoolRegistryFileName = "article-index-spools.registry";
/// Schema version written to a per-spool ArticleIndexMeta. Version 1 is the retired single-database shape.
- internal const int SchemaVersion = 2;
+ internal const int SchemaVersion = 3;
+
+ /// Maximum eviction attempts per capacity-eviction pass.
+ internal const int MaxEvictionAttempts = 256;
/// Page size applied before the first schema write. Existing databases keep their page size.
internal const int PageSizeBytes = 8192;
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexWriter.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexWriter.cs
index caf3f3fa..530f8e05 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexWriter.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexWriter.cs
@@ -134,12 +134,14 @@ internal ArticleIndexWriter(
/// Canonical article identity.
/// Validated spool root.
/// UTC Unix epoch milliseconds used for both initial timestamps.
+ /// Payload size recorded in the index row and metadata.
/// Cancels the wait. A command already queued can still commit.
/// Whether this call created the row.
internal async ValueTask InsertAsync(
ArticleId articleId,
string spoolDirectory,
long receivedAtUnixMilliseconds,
+ long artSizeBytes,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(spoolDirectory);
@@ -150,12 +152,18 @@ internal async ValueTask InsertAsync(
nameof(spoolDirectory));
}
+ if (artSizeBytes < 0)
+ {
+ throw new ArgumentOutOfRangeException(nameof(artSizeBytes));
+ }
+
EnsureTimestamp(receivedAtUnixMilliseconds);
var request = new ArticleIndexRequest(
ArticleIndexCommandKind.Insert,
articleId,
receivedAtUnixMilliseconds,
- spoolDirectory);
+ spoolDirectory,
+ artSizeBytes);
await EnqueueAsync(request, cancellationToken).ConfigureAwait(false);
var result = await request.Completion.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
return new ArticleIndexInsertResult(true, result.Disposition == ArticleIndexWriteDisposition.Inserted);
@@ -196,6 +204,18 @@ internal async ValueTask TouchAsync(
return MapTouch(result.Disposition);
}
+ /// Queues deletion of one evicted row after its spool file was removed.
+ /// Canonical article identity.
+ /// Cancels the wait.
+ /// when the row was deleted.
+ internal async ValueTask EvictDeleteAsync(ArticleId articleId, CancellationToken cancellationToken)
+ {
+ var request = new ArticleIndexRequest(ArticleIndexCommandKind.EvictDelete, articleId, unixMilliseconds: 0, spoolDirectory: null);
+ await EnqueueAsync(request, cancellationToken).ConfigureAwait(false);
+ var result = await request.Completion.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
+ return result.Disposition == ArticleIndexWriteDisposition.Updated;
+ }
+
///
/// Reads queued commands until the channel is completed or the token is cancelled, then drains.
/// Cancellation of this token still commits commands already in the queue.
@@ -615,6 +635,7 @@ private void Build(List batch)
mutation.HasInsert = true;
mutation.ReceivedAt = request.UnixMilliseconds;
mutation.SpoolDirectory = request.SpoolDirectory;
+ mutation.ArtSizeBytes = request.ArtSizeBytes;
if (!mutation.HasTouch || request.UnixMilliseconds > mutation.LastAccessedAt)
{
mutation.LastAccessedAt = request.UnixMilliseconds;
@@ -625,14 +646,23 @@ private void Build(List batch)
continue;
}
+ if (request.Kind == ArticleIndexCommandKind.EvictDelete)
+ {
+ mutation.HasEvictDelete = true;
+ mutation.InsertWaiters.Add(request);
+ continue;
+ }
+
var stamp = request.UnixMilliseconds;
- if (_coalescer.TryTake(request.Id, out var pending, out var waiters, out var count))
+ var increment = 1;
+ if (_coalescer.TryTake(request.Id, out var pending, out var accessIncrements, out var waiters, out var count))
{
if (pending > stamp)
{
stamp = pending;
}
+ increment = accessIncrements <= 0 ? 1 : accessIncrements;
if (count > 0 && waiters is not null)
{
mutation.CoalescedTouches ??= new List>(count);
@@ -644,6 +674,7 @@ private void Build(List batch)
}
mutation.HasTouch = true;
+ mutation.AccessIncrement += increment;
if (stamp > mutation.LastAccessedAt)
{
mutation.LastAccessedAt = stamp;
@@ -687,6 +718,12 @@ private void Publish()
{
foreach (var mutation in _mutations)
{
+ if (mutation.HasEvictDelete && mutation.EvictDeleteDisposition == ArticleIndexWriteDisposition.Updated)
+ {
+ _presence.Remove(mutation.Id);
+ continue;
+ }
+
var present = mutation.HasInsert
|| mutation.TouchDisposition is ArticleIndexWriteDisposition.Updated or ArticleIndexWriteDisposition.Unchanged;
if (!present || mutation.TouchDisposition == ArticleIndexWriteDisposition.Missing && !mutation.HasInsert)
@@ -721,7 +758,8 @@ private void CompleteWaiters(Exception? exception)
}
Note(mutation);
- var insert = new ArticleIndexWriteResult(mutation.InsertDisposition);
+ var insert = new ArticleIndexWriteResult(
+ mutation.HasEvictDelete ? mutation.EvictDeleteDisposition : mutation.InsertDisposition);
foreach (var waiter in mutation.InsertWaiters)
{
waiter.Completion.TrySetResult(insert);
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIngestionReservations.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIngestionReservations.cs
index f209eae9..24e3773d 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIngestionReservations.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIngestionReservations.cs
@@ -67,6 +67,18 @@ internal async ValueTask ReserveAsync(
/// Gets or sets a test signal invoked under the shard lock when a caller is about to wait.
internal Action? OnWaiting { get; set; }
+ /// Returns whether is reserved for in-flight ingestion.
+ /// Canonical article identity.
+ /// when a caller holds the reservation.
+ internal bool IsReserved(ArticleId articleId)
+ {
+ var shard = _shards[Shard(articleId)];
+ lock (shard.Gate)
+ {
+ return shard.InFlight.ContainsKey(articleId);
+ }
+ }
+
/// Drops a reservation this caller acquired. Published membership is left unchanged.
/// Identity passed to .
internal void Release(ArticleId articleId)
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticlePresenceIndex.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticlePresenceIndex.cs
index daaedc3b..c59fb323 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticlePresenceIndex.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticlePresenceIndex.cs
@@ -20,8 +20,7 @@ namespace VectorNNTPD.StorageService.Storage.ArticleIndex
/// of every shard.
///
///
- /// The set does not delete. Eviction must not clear a hash until a later change removes the
- /// durable row and the file together.
+ /// Eviction removes a hash only after the spool file and the durable index row are deleted together.
///
///
internal sealed class ArticlePresenceIndex
@@ -128,6 +127,22 @@ internal bool Add(ArticleId articleId)
return true;
}
+ /// Removes a published hash after eviction deleted its row and file.
+ /// Canonical article identity.
+ /// when the hash was present.
+ internal bool Remove(ArticleId articleId)
+ {
+ Span hash = stackalloc byte[ArticleIndexSettings.HashLength];
+ articleId.CopyTo(hash);
+ if (!_shards[ShardIndex(hash)].Remove(hash))
+ {
+ return false;
+ }
+
+ Interlocked.Decrement(ref _count);
+ return true;
+ }
+
/// Next capacity for a shard that has reached its load factor.
/// Current slot count. Zero selects the first table.
/// The new slot count. Large tables grow by about one quarter, not by doubling.
@@ -245,6 +260,23 @@ internal bool Add(ReadOnlySpan hash)
return true;
}
}
+
+ /// Removes one hash by rebuilding the shard table without it.
+ /// 32-byte hash.
+ /// when the hash was absent.
+ internal bool Remove(ReadOnlySpan hash)
+ {
+ lock (_gate)
+ {
+ if (!_table.TryRemove(hash, out var rebuilt))
+ {
+ return false;
+ }
+
+ Volatile.Write(ref _table, rebuilt);
+ return true;
+ }
+ }
}
/// Immutable-capacity key array. Occupied slots are published with a release write on the bit word.
@@ -391,10 +423,52 @@ internal bool TryAdd(ReadOnlySpan hash)
return false;
}
+ /// Removes one hash by rebuilding a table that omits it.
+ /// 32-byte hash.
+ /// Replacement table when the method returns .
+ /// when the hash was absent.
+ internal bool TryRemove(ReadOnlySpan hash, out Table rebuilt)
+ {
+ rebuilt = this;
+ if (!Contains(hash))
+ {
+ return false;
+ }
+
+ if (Count <= 1)
+ {
+ rebuilt = Create(Math.Max(16, Capacity));
+ return true;
+ }
+
+ var next = Create(Capacity);
+ for (var index = 0; index < Capacity; index++)
+ {
+ if (!IsOccupied(index))
+ {
+ continue;
+ }
+
+ var candidate = _keys.AsSpan(index * ArticleIndexSettings.HashLength, ArticleIndexSettings.HashLength);
+ if (candidate.SequenceEqual(hash))
+ {
+ continue;
+ }
+
+ if (!next.TryAdd(candidate))
+ {
+ throw new InvalidOperationException("Article membership rebuild failed after eviction.");
+ }
+ }
+
+ rebuilt = next;
+ return true;
+ }
+
/// Reads one occupancy bit with acquire semantics.
/// Slot index.
/// when the slot holds a hash.
- private bool IsOccupied(int index)
+ internal bool IsOccupied(int index)
{
var word = Volatile.Read(ref _occupied[index >> 6]);
return (word & (1UL << (index & 63))) != 0;
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleReadLeaseRegistry.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleReadLeaseRegistry.cs
new file mode 100644
index 00000000..442d464b
--- /dev/null
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleReadLeaseRegistry.cs
@@ -0,0 +1,76 @@
+using VectorNNTPD.Common.Articles;
+
+namespace VectorNNTPD.StorageService.Storage.ArticleIndex
+{
+
+ /// Tracks active ARTICLE read leases so eviction does not delete an article being served.
+ internal sealed class ArticleReadLeaseRegistry
+ {
+ private readonly Dictionary _counts = new();
+
+ /// Gets a lease for . Dispose to release.
+ /// Article being read for NNTP delivery.
+ /// A disposable lease handle.
+ internal IDisposable Acquire(ArticleId articleId)
+ {
+ lock (_counts)
+ {
+ if (_counts.TryGetValue(articleId, out var count))
+ {
+ _counts[articleId] = count + 1;
+ }
+ else
+ {
+ _counts.Add(articleId, 1);
+ }
+ }
+
+ return new Release(this, articleId);
+ }
+
+ /// Returns whether any caller holds a read lease for .
+ /// Canonical article identity.
+ /// when an ARTICLE response is in progress.
+ internal bool IsLeased(ArticleId articleId)
+ {
+ lock (_counts)
+ {
+ return _counts.TryGetValue(articleId, out var count) && count > 0;
+ }
+ }
+
+ private void ReleaseLease(ArticleId articleId)
+ {
+ lock (_counts)
+ {
+ if (!_counts.TryGetValue(articleId, out var count))
+ {
+ return;
+ }
+
+ if (count <= 1)
+ {
+ _counts.Remove(articleId);
+ return;
+ }
+
+ _counts[articleId] = count - 1;
+ }
+ }
+
+ private sealed class Release(ArticleReadLeaseRegistry owner, ArticleId articleId) : IDisposable
+ {
+ private int _disposed;
+
+ public void Dispose()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) != 0)
+ {
+ return;
+ }
+
+ owner.ReleaseLease(articleId);
+ }
+ }
+ }
+}
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/IArticleIndex.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/IArticleIndex.cs
index 875f90a6..e49407b5 100644
--- a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/IArticleIndex.cs
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/IArticleIndex.cs
@@ -49,12 +49,14 @@ internal interface IArticleIndex
/// Validated spool root that holds the final file.
/// One UTC Unix epoch millisecond sample.
/// Cancels the wait. A queued command can still commit.
+ /// Payload size stored for LFU accounting and eviction metadata.
/// The commit result. is false when the original timestamps were kept.
ValueTask InsertStoredAsync(
ArticleId articleId,
string spoolDirectory,
long receivedAtUnixMilliseconds,
- CancellationToken cancellationToken);
+ CancellationToken cancellationToken,
+ long artSizeBytes = 0);
///
/// Waits until this caller owns the in-flight reservation for .
@@ -69,7 +71,7 @@ ValueTask InsertStoredAsync(
/// Identity that this caller reserved.
void ReleaseIngestion(ArticleId articleId);
- /// Moves LastAccessedAt forward in when that row exists and the timestamp is newer.
+ /// Increments AccessCount and moves LastAccessedAt forward when the timestamp is newer.
/// Canonical article identity.
/// Spool whose database owns the article.
/// UTC Unix epoch milliseconds.
@@ -109,6 +111,17 @@ ValueTask TouchAsync(
/// The number of rows written.
int CopyAccessedAtOrBefore(string spoolDirectory, long accessedAtOrBeforeUnixMilliseconds, Span destination);
+ /// Copies rows in LFU order. Pending touches, leases, and ingestion reservations are skipped.
+ /// Spool whose database is queried.
+ /// Buffer. At most 10000 rows are accepted.
+ /// The number of rows written.
+ int CopyLeastFrequentlyUsed(string spoolDirectory, Span destination);
+
+ /// Evicts LFU victims until filesystem used percent is at or below LowPercent.
+ /// Spool whose device occupancy is observed.
+ /// Observed between attempts.
+ ValueTask EvictUntilBelowHighAsync(string spoolDirectory, CancellationToken cancellationToken);
+
/// Gets the SQLite path for a registered spool.
/// Registered spool root.
/// The database file under DbDir.
diff --git a/src/VectorNNTPD.StorageService/Storage/ArticleIndex/SpoolLfuEviction.cs b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/SpoolLfuEviction.cs
new file mode 100644
index 00000000..ff1645e8
--- /dev/null
+++ b/src/VectorNNTPD.StorageService/Storage/ArticleIndex/SpoolLfuEviction.cs
@@ -0,0 +1,168 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using VectorNNTPD.Common.Articles;
+using VectorNNTPD.Common.Core;
+using VectorNNTPD.StorageService.Configuration;
+using VectorNNTPD.StorageService.Storage.SegmentPools;
+using VectorNNTPD.StorageService.Storage.SpoolPersistence;
+using VectorNNTPD.StorageService.Storage.SpoolShards;
+
+namespace VectorNNTPD.StorageService.Storage.ArticleIndex
+{
+
+ /// Filesystem-threshold LFU eviction for one spool.
+ ///
+ /// Eviction starts when that spool's filesystem used percent is at or above
+ /// and continues until usage is at or below
+ /// , no eligible victim remains, measurement is lost,
+ /// or is reached. Falling below High alone
+ /// does not stop an in-progress eviction run.
+ ///
+ internal sealed class SpoolLfuEviction
+ {
+ private readonly StorageServerOptions _options;
+ private readonly BlockDeviceMonitorService _monitor;
+ private readonly ISpoolArticleFiles _files;
+ private readonly ArticleReadLeaseRegistry _readLeases;
+ private readonly ILogger _logger;
+
+ /// Initializes eviction helpers.
+ public SpoolLfuEviction(
+ IOptions options,
+ BlockDeviceMonitorService monitor,
+ ISpoolArticleFiles files,
+ ArticleReadLeaseRegistry readLeases,
+ ILogger logger)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentNullException.ThrowIfNull(monitor);
+ ArgumentNullException.ThrowIfNull(files);
+ ArgumentNullException.ThrowIfNull(readLeases);
+ ArgumentNullException.ThrowIfNull(logger);
+ _options = options.Value;
+ _monitor = monitor;
+ _files = files;
+ _readLeases = readLeases;
+ _logger = logger;
+ }
+
+ ///
+ /// Evicts until filesystem used percent is at or below .
+ ///
+ /// Open article index.
+ /// Spool whose occupancy is measured.
+ /// Observed between attempts.
+ internal async ValueTask EvictUntilBelowHighAsync(
+ ArticleIndexService index,
+ string spoolDirectory,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(index);
+ ArgumentException.ThrowIfNullOrWhiteSpace(spoolDirectory);
+ var pools = _options.SegmentPools ?? new SegmentPoolsOptions();
+ if (!TryGetUsedPercent(spoolDirectory, out var used))
+ {
+ // No sample: do not invent an empty filesystem or start eviction.
+ return;
+ }
+
+ if (used < pools.HighPercent)
+ {
+ return;
+ }
+
+ var attempts = 0;
+ while (attempts < ArticleIndexSettings.MaxEvictionAttempts)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (!TryGetUsedPercent(spoolDirectory, out used))
+ {
+ return;
+ }
+
+ if (used <= pools.LowPercent)
+ {
+ return;
+ }
+
+ if (!await index.TryEvictOneAsync(spoolDirectory, this, cancellationToken).ConfigureAwait(false))
+ {
+ return;
+ }
+
+ attempts++;
+ }
+ }
+
+ /// Deletes one eligible LFU victim when possible.
+ /// Open article index.
+ /// Spool root.
+ /// LFU row chosen by the index.
+ /// Observed before index deletion.
+ /// when the file and index row were removed.
+ internal async ValueTask EvictCandidateAsync(
+ ArticleIndexService index,
+ string spoolDirectory,
+ ArticleIndexRecord candidate,
+ CancellationToken cancellationToken)
+ {
+ var path = SpoolArticlePath.CombineArticle(spoolDirectory, candidate.ArticleId);
+ var fileExisted = _files.Exists(path);
+ try
+ {
+ _files.Delete(path);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ ArticleIndexLogMessages.EvictionFileDeleteFailed(
+ _logger,
+ candidate.ArticleId.ToLowerHexString(),
+ ex.GetType().Name);
+ return false;
+ }
+
+ try
+ {
+ if (!await index.DeleteEvictedRowAsync(spoolDirectory, candidate.ArticleId, cancellationToken)
+ .ConfigureAwait(false))
+ {
+ ArticleIndexLogMessages.EvictionIndexDeleteFailed(
+ _logger,
+ candidate.ArticleId.ToLowerHexString(),
+ spoolDirectory);
+ return false;
+ }
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ ArticleIndexLogMessages.EvictionIndexDeleteFailed(
+ _logger,
+ candidate.ArticleId.ToLowerHexString(),
+ spoolDirectory);
+ ExpectedException.Ignore(ex);
+ return false;
+ }
+
+ if (fileExisted && candidate.ArtSizeBytes > 0)
+ {
+ _monitor.NoteFreedBytes(spoolDirectory, candidate.ArtSizeBytes);
+ }
+
+ return true;
+ }
+
+ /// Returns whether may be evicted from .
+ internal bool IsEligible(ArticleIndexService index, string spoolDirectory, ArticleId articleId, long accessedAt)
+ {
+ if (_readLeases.IsLeased(articleId) || index.IsIngestionReserved(articleId))
+ {
+ return false;
+ }
+
+ return index.IsEvictionCandidate(spoolDirectory, articleId, accessedAt);
+ }
+
+ private bool TryGetUsedPercent(string spoolDirectory, out int usedPercent) =>
+ SegmentPoolCapacityGate.TryGetUsedPercent(_monitor.Observations, spoolDirectory, out usedPercent);
+ }
+}
diff --git a/src/VectorNNTPD.StorageService/Storage/SegmentPools/BlockDeviceMonitorService.cs b/src/VectorNNTPD.StorageService/Storage/SegmentPools/BlockDeviceMonitorService.cs
index b161a4e1..273a12d2 100644
--- a/src/VectorNNTPD.StorageService/Storage/SegmentPools/BlockDeviceMonitorService.cs
+++ b/src/VectorNNTPD.StorageService/Storage/SegmentPools/BlockDeviceMonitorService.cs
@@ -27,6 +27,12 @@ internal sealed class BlockDeviceMonitorService : IApplicationService
private Task? _execution;
private int _started;
+ ///
+ /// Raised after a consistent free-space snapshot is published (periodic sample or test injection).
+ /// Subscribers include the admission gate and per-spool eviction monitors.
+ ///
+ internal event Action? CapacitySamplePublished;
+
/// Initializes a one-second monitor.
/// Devices published by successful validation.
/// Platform counter source.
@@ -87,8 +93,9 @@ internal SpoolObservationSnapshot Observations
///
/// Subtracts from the cached available space for
- /// and records the commit so a probe already in progress cannot add those bytes back.
- /// A missing cache entry is left unchanged. Duplicate and failed publishes do not call this.
+ /// and every sibling path on the same physical device, and records the commit so a probe already
+ /// in progress cannot add those bytes back. Shared-volume directories must not pretend to have
+ /// independent free space.
///
/// Canonical spool path.
/// Bytes renamed into that spool. Negative values are ignored.
@@ -101,20 +108,129 @@ internal void NoteCommittedBytes(string spoolPath, long bytes)
lock (_publishGate)
{
- if (!_observations.Space.TryGetValue(spoolPath, out var current))
+ AdjustCachedAvailable(spoolPath, -bytes);
+ var pending = _commitsAfterProbeOpen.GetValueOrDefault(spoolPath);
+ _commitsAfterProbeOpen[spoolPath] = pending > long.MaxValue - bytes ? long.MaxValue : pending + bytes;
+ }
+
+ CapacitySamplePublished?.Invoke();
+ }
+
+ /// Publishes one filesystem sample for tests and local eviction verification.
+ /// Canonical spool path.
+ /// Filesystem capacity in bytes.
+ /// Available bytes on that spool.
+ internal void SetSpaceObservationForTests(string spoolPath, long totalBytes, long availableBytes)
+ {
+ if (string.IsNullOrEmpty(spoolPath) || totalBytes <= 0)
+ {
+ return;
+ }
+
+ lock (_publishGate)
+ {
+ var space = new Dictionary(_observations.Space, StringComparer.Ordinal);
+ space[spoolPath] = new SpoolSpaceObservation(totalBytes, availableBytes, _time.GetTimestamp());
+ _observations = _observations with { Space = space, Timestamp = _time.GetTimestamp() };
+ }
+
+ CapacitySamplePublished?.Invoke();
+ }
+
+ ///
+ /// Adds back to the cached available space for
+ /// and sibling paths on the same physical device after eviction.
+ ///
+ /// Canonical spool path.
+ /// Bytes reclaimed on that spool. Negative values are ignored.
+ internal void NoteFreedBytes(string spoolPath, long bytes)
+ {
+ if (bytes <= 0 || string.IsNullOrEmpty(spoolPath))
+ {
+ return;
+ }
+
+ lock (_publishGate)
+ {
+ AdjustCachedAvailable(spoolPath, bytes);
+ }
+
+ CapacitySamplePublished?.Invoke();
+ }
+
+ ///
+ /// Adjusts cached available bytes for and every catalog sibling
+ /// on the same device. Caller must hold .
+ ///
+ private void AdjustCachedAvailable(string spoolPath, long deltaBytes)
+ {
+ var paths = PathsSharingDevice(spoolPath);
+ var space = new Dictionary(_observations.Space, StringComparer.Ordinal);
+ var changed = false;
+ foreach (var path in paths)
+ {
+ if (!space.TryGetValue(path, out var current))
{
- return;
+ continue;
}
- var space = new Dictionary(_observations.Space, StringComparer.Ordinal);
- var available = current.AvailableBytes > bytes ? current.AvailableBytes - bytes : 0;
- space[spoolPath] = current with { AvailableBytes = available };
+ long available;
+ if (deltaBytes < 0)
+ {
+ var consume = -deltaBytes;
+ available = current.AvailableBytes > consume ? current.AvailableBytes - consume : 0;
+ }
+ else
+ {
+ available = current.AvailableBytes > long.MaxValue - deltaBytes
+ ? long.MaxValue
+ : current.AvailableBytes + deltaBytes;
+ if (available > current.TotalBytes)
+ {
+ available = current.TotalBytes;
+ }
+ }
+
+ space[path] = current with { AvailableBytes = available };
+ changed = true;
+ }
+
+ if (changed)
+ {
_observations = _observations with { Space = space };
- var pending = _commitsAfterProbeOpen.GetValueOrDefault(spoolPath);
- _commitsAfterProbeOpen[spoolPath] = pending > long.MaxValue - bytes ? long.MaxValue : pending + bytes;
}
}
+ private IEnumerable PathsSharingDevice(string spoolPath)
+ {
+ foreach (var device in _catalog.Devices)
+ {
+ var match = false;
+ foreach (var path in device.PoolPaths)
+ {
+ if (string.Equals(path, spoolPath, StringComparison.Ordinal))
+ {
+ match = true;
+ break;
+ }
+ }
+
+ if (!match)
+ {
+ continue;
+ }
+
+ foreach (var path in device.PoolPaths)
+ {
+ yield return path;
+ }
+
+ yield break;
+ }
+
+ yield return spoolPath;
+ }
+
///
public string Name => "BlockDeviceMonitor";
@@ -312,6 +428,8 @@ private void Sample()
_observations = new SpoolObservationSnapshot(_time.GetTimestamp(), devices, unavailable, space);
}
+
+ CapacitySamplePublished?.Invoke();
}
///
diff --git a/src/VectorNNTPD.StorageService/Storage/SegmentPools/SegmentPoolCapacityGate.cs b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SegmentPoolCapacityGate.cs
new file mode 100644
index 00000000..8b299653
--- /dev/null
+++ b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SegmentPoolCapacityGate.cs
@@ -0,0 +1,150 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using VectorNNTPD.StorageService.Configuration;
+using VectorNNTPD.StorageService.Storage.SpoolPersistence;
+
+namespace VectorNNTPD.StorageService.Storage.SegmentPools
+{
+ ///
+ /// Global TAKETHIS admission gate based on per-spool placement eligibility.
+ ///
+ ///
+ ///
+ /// NNTP admission and persistence placement are distinct: admission requires at least one
+ /// valid, fresh, non-critical destination; placement (in SpoolSelector) picks the
+ /// eligible spool with the greatest available bytes.
+ ///
+ ///
+ /// Admission recovers as soon as any destination becomes eligible. It does not wait for every
+ /// spool to fall to . Per-spool High→Low eviction
+ /// is owned by independent loops.
+ ///
+ ///
+ internal sealed class SegmentPoolCapacityGate
+ {
+ private readonly SpoolCapacityRegistry _registry;
+ private readonly BlockDeviceMonitorService _monitor;
+ private readonly ILogger _logger;
+ private int _admissionBlocked = 1;
+ private int _loggedUnknown;
+ private int _loggedCritical;
+
+ /// Initializes the gate.
+ public SegmentPoolCapacityGate(
+ SpoolCapacityRegistry registry,
+ BlockDeviceMonitorService monitor,
+ IOptions options,
+ ILogger logger)
+ {
+ ArgumentNullException.ThrowIfNull(registry);
+ ArgumentNullException.ThrowIfNull(monitor);
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentNullException.ThrowIfNull(logger);
+ _ = options;
+ _registry = registry;
+ _monitor = monitor;
+ _logger = logger;
+ _monitor.CapacitySamplePublished += OnCapacitySampleUpdated;
+ OnCapacitySampleUpdated();
+ }
+
+ ///
+ /// Returns whether TAKETHIS may admit into the in-memory queue because at least one
+ /// spool is valid, fresh, and at or below CriticalPercent.
+ ///
+ internal bool AllowsAdmission()
+ {
+ _registry.RefreshFromObservations();
+ if (_registry.HasEligibleDestination())
+ {
+ if (Interlocked.Exchange(ref _admissionBlocked, 0) == 1)
+ {
+ SegmentPoolCapacityLogMessages.AdmissionRecovered(_logger);
+ Interlocked.Exchange(ref _loggedUnknown, 0);
+ Interlocked.Exchange(ref _loggedCritical, 0);
+ }
+
+ return true;
+ }
+
+ Interlocked.Exchange(ref _admissionBlocked, 1);
+ var states = _registry.SnapshotAll();
+ var anyValid = false;
+ foreach (var state in states)
+ {
+ if (state.MeasurementValid)
+ {
+ anyValid = true;
+ break;
+ }
+ }
+
+ if (!anyValid)
+ {
+ if (Interlocked.Exchange(ref _loggedUnknown, 1) == 0)
+ {
+ SegmentPoolCapacityLogMessages.AdmissionBlockedUnknown(_logger);
+ }
+ }
+ else if (Interlocked.Exchange(ref _loggedCritical, 1) == 0)
+ {
+ SegmentPoolCapacityLogMessages.AdmissionBlockedCritical(_logger);
+ }
+
+ return false;
+ }
+
+ /// Refreshes registry state after a monitor publish.
+ internal void OnCapacitySampleUpdated()
+ {
+ _registry.RefreshFromObservations();
+ if (_registry.HasEligibleDestination())
+ {
+ Interlocked.Exchange(ref _admissionBlocked, 0);
+ }
+ else
+ {
+ Interlocked.Exchange(ref _admissionBlocked, 1);
+ }
+ }
+
+ /// Resolves used percent for one spool path from the monitor snapshot.
+ internal static bool TryGetUsedPercent(
+ SpoolObservationSnapshot observations,
+ string spoolDirectory,
+ out int usedPercent)
+ {
+ usedPercent = 0;
+ var canonical = Path.TrimEndingDirectorySeparator(Path.GetFullPath(spoolDirectory));
+ if (!observations.Space.TryGetValue(canonical, out var space) || space.TotalBytes <= 0)
+ {
+ return false;
+ }
+
+ usedPercent = SpoolSelector.UsedPercent(space.TotalBytes, space.AvailableBytes);
+ return true;
+ }
+ }
+
+ /// Source-generated capacity-gate logs.
+ internal static partial class SegmentPoolCapacityLogMessages
+ {
+ [LoggerMessage(
+ EventId = 5820,
+ Level = LogLevel.Warning,
+ Message = "Segment-pool TAKETHIS admission blocked: no valid fresh filesystem capacity sample on any spool")]
+ public static partial void AdmissionBlockedUnknown(ILogger logger);
+
+ [LoggerMessage(
+ EventId = 5821,
+ Level = LogLevel.Warning,
+ Message = "Segment-pool TAKETHIS admission blocked: no spool at or below CriticalPercent")]
+ public static partial void AdmissionBlockedCritical(ILogger logger);
+
+ [LoggerMessage(
+ EventId = 5822,
+ Level = LogLevel.Information,
+ Message = "Segment-pool TAKETHIS admission recovered: at least one eligible spool")]
+ public static partial void AdmissionRecovered(ILogger logger);
+ }
+}
diff --git a/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolCapacityRegistry.cs b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolCapacityRegistry.cs
new file mode 100644
index 00000000..708fc85e
--- /dev/null
+++ b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolCapacityRegistry.cs
@@ -0,0 +1,252 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Options;
+using VectorNNTPD.StorageService.Configuration;
+using VectorNNTPD.StorageService.Storage.SpoolPersistence;
+
+namespace VectorNNTPD.StorageService.Storage.SegmentPools
+{
+ ///
+ /// Thread-safe per-spool capacity state derived from samples.
+ /// One entry exists for each catalog path; directories that share a
+ /// share physical free space but keep distinct membership and eviction monitors.
+ ///
+ internal sealed class SpoolCapacityRegistry
+ {
+ private readonly SegmentPoolCatalog _catalog;
+ private readonly BlockDeviceMonitorService _monitor;
+ private readonly StorageServerOptions _options;
+ private readonly TimeProvider _time;
+ private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal);
+
+ /// Initializes the registry.
+ public SpoolCapacityRegistry(
+ SegmentPoolCatalog catalog,
+ BlockDeviceMonitorService monitor,
+ IOptions options,
+ TimeProvider time)
+ {
+ ArgumentNullException.ThrowIfNull(catalog);
+ ArgumentNullException.ThrowIfNull(monitor);
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentNullException.ThrowIfNull(time);
+ _catalog = catalog;
+ _monitor = monitor;
+ _options = options.Value;
+ _time = time;
+ }
+
+ /// Number of configured spool paths currently tracked (catalog size after refresh).
+ internal int MonitorCount
+ {
+ get
+ {
+ EnsureEntries();
+ return _entries.Count;
+ }
+ }
+
+ /// Rebuilds entries from the catalog and refreshes from the latest monitor snapshot.
+ internal void RefreshFromObservations()
+ {
+ EnsureEntries();
+ var pools = _options.SegmentPools ?? new SegmentPoolsOptions();
+ var observations = _monitor.Observations;
+ var now = _time.GetTimestamp();
+ foreach (var entry in _entries.Values)
+ {
+ entry.Apply(observations, pools, now, _time);
+ }
+ }
+
+ /// Returns a snapshot for when tracked.
+ internal bool TryGet(string spoolDirectory, out SpoolCapacityState state)
+ {
+ EnsureEntries();
+ var key = Canonical(spoolDirectory);
+ if (!_entries.TryGetValue(key, out var entry))
+ {
+ state = default;
+ return false;
+ }
+
+ state = entry.Snapshot();
+ return true;
+ }
+
+ /// Copies all current spool states in catalog order.
+ internal SpoolCapacityState[] SnapshotAll()
+ {
+ EnsureEntries();
+ RefreshFromObservations();
+ var list = new List(_entries.Count);
+ foreach (var device in _catalog.Devices)
+ {
+ list.AddRange(
+ device.PoolPaths
+ .Select(Canonical)
+ .Select(path => _entries.TryGetValue(path, out var entry) ? entry : null)
+ .Where(static entry => entry is not null)
+ .Select(static entry => entry!.Snapshot()));
+ }
+
+ return list.ToArray();
+ }
+
+ /// Returns whether any spool is currently eligible for new article placement.
+ internal bool HasEligibleDestination()
+ {
+ EnsureEntries();
+ RefreshFromObservations();
+ foreach (var entry in _entries.Values)
+ {
+ if (entry.Snapshot().PlacementEligible)
+ {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /// Marks whether the named spool's eviction monitor is actively reclaiming.
+ internal void SetEvictionActive(string spoolDirectory, bool active)
+ {
+ EnsureEntries();
+ if (_entries.TryGetValue(Canonical(spoolDirectory), out var entry))
+ {
+ entry.SetEvictionActive(active);
+ }
+ }
+
+ /// Enumerates configured spool paths with their device ids.
+ internal IReadOnlyList<(string SpoolDirectory, string DeviceId)> EnumerateSpools()
+ {
+ EnsureEntries();
+ var list = new List<(string, string)>();
+ foreach (var device in _catalog.Devices)
+ {
+ foreach (var path in device.PoolPaths)
+ {
+ list.Add((Canonical(path), device.DeviceId));
+ }
+ }
+
+ return list;
+ }
+
+ /// Returns sibling spool paths that share .
+ internal IReadOnlyList PathsOnDevice(string deviceId)
+ {
+ EnsureEntries();
+ var list = new List();
+ foreach (var device in _catalog.Devices)
+ {
+ if (!string.Equals(device.DeviceId, deviceId, StringComparison.Ordinal))
+ {
+ continue;
+ }
+
+ foreach (var path in device.PoolPaths)
+ {
+ list.Add(Canonical(path));
+ }
+ }
+
+ return list;
+ }
+
+ private void EnsureEntries()
+ {
+ foreach (var device in _catalog.Devices)
+ {
+ foreach (var canonical in device.PoolPaths.Select(Canonical))
+ {
+ _entries.GetOrAdd(canonical, static (p, id) => new Entry(p, id), device.DeviceId);
+ }
+ }
+ }
+
+ private static string Canonical(string spoolDirectory) =>
+ Path.TrimEndingDirectorySeparator(Path.GetFullPath(spoolDirectory));
+
+ private sealed class Entry
+ {
+ private readonly object _gate = new();
+ private SpoolCapacityState _state;
+
+ internal Entry(string spoolDirectory, string deviceId)
+ {
+ _state = new SpoolCapacityState(
+ spoolDirectory,
+ deviceId,
+ TotalBytes: 0,
+ AvailableBytes: 0,
+ UsedPercent: 0,
+ MeasurementValid: false,
+ ProbeTimestamp: 0,
+ PlacementEligible: false,
+ EvictionRequired: false,
+ EvictionActive: false);
+ }
+
+ internal void Apply(
+ SpoolObservationSnapshot observations,
+ SegmentPoolsOptions pools,
+ long now,
+ TimeProvider time)
+ {
+ lock (_gate)
+ {
+ var path = _state.SpoolDirectory;
+ var valid = false;
+ long total = 0;
+ long available = 0;
+ long probe = 0;
+ var used = 0;
+ if (observations.Space.TryGetValue(path, out var space)
+ && space.TotalBytes > 0
+ && space.AvailableBytes >= 0
+ && space.AvailableBytes <= space.TotalBytes
+ && space.ProbeTimestamp != 0
+ && time.GetElapsedTime(space.ProbeTimestamp, now) <= SpoolSelectionPolicy.SampleFreshness)
+ {
+ valid = true;
+ total = space.TotalBytes;
+ available = space.AvailableBytes;
+ probe = space.ProbeTimestamp;
+ used = SpoolSelector.UsedPercent(total, available);
+ }
+
+ var placementEligible = valid && used <= pools.CriticalPercent;
+ var evictionRequired = valid && used >= pools.HighPercent;
+ _state = _state with
+ {
+ TotalBytes = total,
+ AvailableBytes = available,
+ UsedPercent = used,
+ MeasurementValid = valid,
+ ProbeTimestamp = probe,
+ PlacementEligible = placementEligible,
+ EvictionRequired = evictionRequired,
+ };
+ }
+ }
+
+ internal void SetEvictionActive(bool active)
+ {
+ lock (_gate)
+ {
+ _state = _state with { EvictionActive = active };
+ }
+ }
+
+ internal SpoolCapacityState Snapshot()
+ {
+ lock (_gate)
+ {
+ return _state;
+ }
+ }
+ }
+ }
+}
diff --git a/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolCapacityState.cs b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolCapacityState.cs
new file mode 100644
index 00000000..78f8d76a
--- /dev/null
+++ b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolCapacityState.cs
@@ -0,0 +1,28 @@
+namespace VectorNNTPD.StorageService.Storage.SegmentPools
+{
+ ///
+ /// Immutable view of one configured spool's capacity and eviction posture.
+ /// Percentages are never aggregated across unrelated filesystems.
+ ///
+ /// Canonical spool root.
+ /// Physical device identity from validation (shared when directories share a volume).
+ /// Filesystem capacity when .
+ /// Filesystem available bytes when .
+ /// Derived used percent when .
+ /// Whether the latest sample is present, consistent, and fresh enough for placement.
+ /// Monotonic probe time of the sample, or zero when unknown.
+ /// Valid measurement and used percent at or below Critical (strict greater-than blocks).
+ /// Valid measurement at or above HighPercent.
+ /// Whether this spool's eviction monitor is currently reclaiming.
+ internal readonly record struct SpoolCapacityState(
+ string SpoolDirectory,
+ string DeviceId,
+ long TotalBytes,
+ long AvailableBytes,
+ int UsedPercent,
+ bool MeasurementValid,
+ long ProbeTimestamp,
+ bool PlacementEligible,
+ bool EvictionRequired,
+ bool EvictionActive);
+}
diff --git a/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolEvictionMonitor.cs b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolEvictionMonitor.cs
new file mode 100644
index 00000000..b5a4279c
--- /dev/null
+++ b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolEvictionMonitor.cs
@@ -0,0 +1,258 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using VectorNNTPD.Common.Core;
+using VectorNNTPD.StorageService.Configuration;
+using VectorNNTPD.StorageService.Storage.ArticleIndex;
+
+namespace VectorNNTPD.StorageService.Storage.SegmentPools
+{
+ ///
+ /// Independent High→Low LFU eviction loop for one configured spool directory.
+ /// Never selects victims from another spool. Non-fatal failures are logged and the loop continues
+ /// after a bounded backoff so one monitor cannot terminate the others.
+ ///
+ internal sealed class SpoolEvictionMonitor
+ {
+ private static readonly TimeSpan FailureBackoff = TimeSpan.FromSeconds(5);
+
+ private readonly string _spoolDirectory;
+ private readonly string _deviceId;
+ private readonly SpoolCapacityRegistry _registry;
+ private readonly IServiceProvider _services;
+ private readonly StorageServerOptions _options;
+ private readonly ILogger _logger;
+ private readonly SemaphoreSlim _wake = new(0, 1);
+ private Task? _loop;
+ private int _started;
+ private int _evictionRunning;
+
+ /// Initializes one per-spool monitor.
+ public SpoolEvictionMonitor(
+ string spoolDirectory,
+ string deviceId,
+ SpoolCapacityRegistry registry,
+ BlockDeviceMonitorService monitor,
+ IServiceProvider services,
+ IOptions options,
+ ILogger logger)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(spoolDirectory);
+ ArgumentException.ThrowIfNullOrWhiteSpace(deviceId);
+ ArgumentNullException.ThrowIfNull(registry);
+ ArgumentNullException.ThrowIfNull(monitor);
+ ArgumentNullException.ThrowIfNull(services);
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentNullException.ThrowIfNull(logger);
+ _ = monitor;
+ _spoolDirectory = Path.TrimEndingDirectorySeparator(Path.GetFullPath(spoolDirectory));
+ _deviceId = deviceId;
+ _registry = registry;
+ _services = services;
+ _options = options.Value;
+ _logger = logger;
+ }
+
+ /// Canonical spool root this monitor owns.
+ internal string SpoolDirectory => _spoolDirectory;
+
+ /// Physical device identity for shared-volume awareness.
+ internal string DeviceId => _deviceId;
+
+ /// Background loop task when started.
+ internal Task? Execution => _loop;
+
+ /// Starts the independent monitor loop.
+ internal void Start(CancellationToken cancellationToken)
+ {
+ if (Interlocked.Exchange(ref _started, 1) == 1)
+ {
+ return;
+ }
+
+ SpoolEvictionMonitorLogMessages.MonitorStarted(_logger, _spoolDirectory, _deviceId);
+ _loop = RunAsync(cancellationToken);
+ NotifySample();
+ }
+
+ /// Signals that a new capacity sample may require eviction evaluation.
+ internal void NotifySample()
+ {
+ try
+ {
+ if (_wake.CurrentCount == 0)
+ {
+ _wake.Release();
+ }
+ }
+ catch (SemaphoreFullException)
+ {
+ // Already signaled.
+ }
+ }
+
+ private async Task RunAsync(CancellationToken cancellationToken)
+ {
+ try
+ {
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ await _wake.WaitAsync(cancellationToken).ConfigureAwait(false);
+ await EvaluateAndEvictAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ break;
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ SpoolEvictionMonitorLogMessages.MonitorFailed(_logger, _spoolDirectory, ex.GetType().Name);
+ try
+ {
+ await Task.Delay(FailureBackoff, cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ break;
+ }
+ }
+ }
+ }
+ finally
+ {
+ _registry.SetEvictionActive(_spoolDirectory, false);
+ SpoolEvictionMonitorLogMessages.MonitorStopped(_logger, _spoolDirectory);
+ }
+ }
+
+ private async Task EvaluateAndEvictAsync(CancellationToken cancellationToken)
+ {
+ if (Interlocked.CompareExchange(ref _evictionRunning, 1, 0) != 0)
+ {
+ return;
+ }
+
+ try
+ {
+ _registry.RefreshFromObservations();
+ if (!_registry.TryGet(_spoolDirectory, out var state) || !state.EvictionRequired)
+ {
+ return;
+ }
+
+ var index = _services.GetService();
+ if (index is null || !index.IsReady)
+ {
+ return;
+ }
+
+ var pools = _options.SegmentPools ?? new SegmentPoolsOptions();
+ SpoolEvictionMonitorLogMessages.EvictionStarted(
+ _logger,
+ _spoolDirectory,
+ state.UsedPercent,
+ pools.HighPercent,
+ pools.LowPercent);
+
+ _registry.SetEvictionActive(_spoolDirectory, true);
+ var beforeAvailable = state.AvailableBytes;
+ try
+ {
+ await index.EvictUntilBelowHighAsync(_spoolDirectory, cancellationToken).ConfigureAwait(false);
+ _registry.RefreshFromObservations();
+ if (_registry.TryGet(_spoolDirectory, out var after))
+ {
+ var reclaimed = after.AvailableBytes > beforeAvailable
+ ? after.AvailableBytes - beforeAvailable
+ : 0;
+ if (after.UsedPercent > pools.LowPercent)
+ {
+ SpoolEvictionMonitorLogMessages.EvictionInsufficient(
+ _logger,
+ _spoolDirectory,
+ after.UsedPercent,
+ pools.LowPercent);
+ }
+ else
+ {
+ SpoolEvictionMonitorLogMessages.EvictionCompleted(
+ _logger,
+ _spoolDirectory,
+ after.UsedPercent,
+ reclaimed);
+ }
+ }
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ SpoolEvictionMonitorLogMessages.EvictionFailed(_logger, _spoolDirectory, ex.GetType().Name);
+ }
+ finally
+ {
+ _registry.SetEvictionActive(_spoolDirectory, false);
+ }
+ }
+ finally
+ {
+ Interlocked.Exchange(ref _evictionRunning, 0);
+ }
+ }
+ }
+
+ /// Source-generated per-spool eviction monitor logs.
+ internal static partial class SpoolEvictionMonitorLogMessages
+ {
+ [LoggerMessage(
+ EventId = 5830,
+ Level = LogLevel.Information,
+ Message = "Spool eviction monitor started spool={Spool} device={DeviceId}")]
+ public static partial void MonitorStarted(ILogger logger, string Spool, string DeviceId);
+
+ [LoggerMessage(
+ EventId = 5831,
+ Level = LogLevel.Information,
+ Message = "Spool eviction monitor stopped spool={Spool}")]
+ public static partial void MonitorStopped(ILogger logger, string Spool);
+
+ [LoggerMessage(
+ EventId = 5832,
+ Level = LogLevel.Warning,
+ Message = "Spool eviction monitor failed spool={Spool} failure={Failure}")]
+ public static partial void MonitorFailed(ILogger logger, string Spool, string Failure);
+
+ [LoggerMessage(
+ EventId = 5833,
+ Level = LogLevel.Information,
+ Message = "Spool LFU eviction started spool={Spool} usedPercent={UsedPercent} highPercent={HighPercent} lowPercent={LowPercent}")]
+ public static partial void EvictionStarted(
+ ILogger logger,
+ string Spool,
+ int UsedPercent,
+ int HighPercent,
+ int LowPercent);
+
+ [LoggerMessage(
+ EventId = 5834,
+ Level = LogLevel.Information,
+ Message = "Spool LFU eviction completed spool={Spool} usedPercent={UsedPercent} reclaimedBytes={ReclaimedBytes}")]
+ public static partial void EvictionCompleted(ILogger logger, string Spool, int UsedPercent, long ReclaimedBytes);
+
+ [LoggerMessage(
+ EventId = 5835,
+ Level = LogLevel.Warning,
+ Message = "Spool LFU eviction ended without reaching low-water spool={Spool} usedPercent={UsedPercent} lowPercent={LowPercent}")]
+ public static partial void EvictionInsufficient(ILogger logger, string Spool, int UsedPercent, int LowPercent);
+
+ [LoggerMessage(
+ EventId = 5836,
+ Level = LogLevel.Warning,
+ Message = "Spool LFU eviction failed spool={Spool} failure={Failure}")]
+ public static partial void EvictionFailed(ILogger logger, string Spool, string Failure);
+ }
+}
diff --git a/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolEvictionMonitorService.cs b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolEvictionMonitorService.cs
new file mode 100644
index 00000000..04219848
--- /dev/null
+++ b/src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolEvictionMonitorService.cs
@@ -0,0 +1,141 @@
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using VectorNNTPD.Common.Core;
+using VectorNNTPD.StorageService.Configuration;
+
+namespace VectorNNTPD.StorageService.Storage.SegmentPools
+{
+ ///
+ /// Supervises one independent per configured spool directory.
+ /// Eight configured directories produce eight independently scheduled monitor loops.
+ ///
+ internal sealed class SpoolEvictionMonitorService : IApplicationService
+ {
+ private readonly SegmentPoolCatalog _catalog;
+ private readonly SpoolCapacityRegistry _registry;
+ private readonly BlockDeviceMonitorService _monitor;
+ private readonly IServiceProvider _services;
+ private readonly IOptions _options;
+ private readonly ILoggerFactory _loggerFactory;
+ private readonly ILogger _logger;
+ private readonly CancellationTokenSource _runCts = new();
+ private readonly List _monitors = [];
+ private Task? _execution;
+ private int _started;
+
+ /// Initializes the supervisor.
+ public SpoolEvictionMonitorService(
+ SegmentPoolCatalog catalog,
+ SpoolCapacityRegistry registry,
+ BlockDeviceMonitorService monitor,
+ IServiceProvider services,
+ IOptions options,
+ ILoggerFactory loggerFactory,
+ ILogger logger)
+ {
+ ArgumentNullException.ThrowIfNull(catalog);
+ ArgumentNullException.ThrowIfNull(registry);
+ ArgumentNullException.ThrowIfNull(monitor);
+ ArgumentNullException.ThrowIfNull(services);
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentNullException.ThrowIfNull(loggerFactory);
+ ArgumentNullException.ThrowIfNull(logger);
+ _catalog = catalog;
+ _registry = registry;
+ _monitor = monitor;
+ _services = services;
+ _options = options;
+ _loggerFactory = loggerFactory;
+ _logger = logger;
+ }
+
+ ///
+ public string Name => "SpoolEvictionMonitors";
+
+ ///
+ public Task? Execution => _execution;
+
+ /// Independent monitors created for the current catalog.
+ internal IReadOnlyList Monitors => _monitors;
+
+ ///
+ public Task StartAsync(CancellationToken cancellationToken)
+ {
+ if (Interlocked.Exchange(ref _started, 1) == 1)
+ {
+ return Task.CompletedTask;
+ }
+
+ _monitors.AddRange(
+ _catalog.Devices.SelectMany(device =>
+ device.PoolPaths.Select(path =>
+ new SpoolEvictionMonitor(
+ path,
+ device.DeviceId,
+ _registry,
+ _monitor,
+ _services,
+ _options,
+ _loggerFactory.CreateLogger()))));
+
+ SpoolEvictionMonitorServiceLogMessages.SupervisorStarted(_logger, _monitors.Count);
+ _monitor.CapacitySamplePublished += OnCapacitySample;
+ _registry.RefreshFromObservations();
+ foreach (var monitor in _monitors)
+ {
+ monitor.Start(_runCts.Token);
+ }
+
+ _execution = Task.WhenAll(_monitors.Select(static m => m.Execution ?? Task.CompletedTask));
+ return Task.CompletedTask;
+ }
+
+ ///
+ public async Task StopAsync(CancellationToken cancellationToken)
+ {
+ _monitor.CapacitySamplePublished -= OnCapacitySample;
+ await _runCts.CancelAsync().ConfigureAwait(false);
+ var execution = _execution;
+ if (execution is null)
+ {
+ return;
+ }
+
+ try
+ {
+ await execution.WaitAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException ex) when (cancellationToken.IsCancellationRequested)
+ {
+ ExpectedException.Ignore(ex);
+ }
+
+ SpoolEvictionMonitorServiceLogMessages.SupervisorStopped(_logger, _monitors.Count);
+ }
+
+ private void OnCapacitySample()
+ {
+ _registry.RefreshFromObservations();
+ foreach (var monitor in _monitors)
+ {
+ monitor.NotifySample();
+ }
+ }
+ }
+
+ /// Source-generated eviction-monitor supervisor logs.
+ internal static partial class SpoolEvictionMonitorServiceLogMessages
+ {
+ [LoggerMessage(
+ EventId = 5840,
+ Level = LogLevel.Information,
+ Message = "Spool eviction monitor supervisor started monitorCount={MonitorCount}")]
+ public static partial void SupervisorStarted(ILogger logger, int MonitorCount);
+
+ [LoggerMessage(
+ EventId = 5841,
+ Level = LogLevel.Information,
+ Message = "Spool eviction monitor supervisor stopped monitorCount={MonitorCount}")]
+ public static partial void SupervisorStopped(ILogger logger, int MonitorCount);
+ }
+}
diff --git a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolArticlePersistenceProcessor.cs b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolArticlePersistenceProcessor.cs
index 4ba97c4a..605b6a2f 100644
--- a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolArticlePersistenceProcessor.cs
+++ b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolArticlePersistenceProcessor.cs
@@ -33,6 +33,7 @@ internal sealed class SpoolArticlePersistenceProcessor : IAcceptedArticleProcess
private readonly IArticleIndex? _index;
private IReadOnlyList? _rootSource;
private string[] _roots = [];
+ private long _lastDeferredLogAt;
/// Initializes the host processor.
/// Published server FQDN used as the Path hop.
@@ -176,9 +177,24 @@ public async ValueTask ProcessAsync(
var receivedAt = _time.GetUtcNow().ToUnixTimeMilliseconds();
try
{
- var indexed = await _index.InsertStoredAsync(article.ArtId, outcome.Spool, receivedAt, cancellationToken)
+ var indexed = await _index.InsertStoredAsync(
+ article.ArtId,
+ outcome.Spool,
+ receivedAt,
+ cancellationToken,
+ article.ArtSize)
.ConfigureAwait(false);
- return indexed.Committed ? outcome.Result : AcceptedArticleProcessResult.Failed;
+ if (!indexed.Committed)
+ {
+ return AcceptedArticleProcessResult.Failed;
+ }
+
+ if (indexed.Inserted)
+ {
+ await _index.EvictUntilBelowHighAsync(outcome.Spool, cancellationToken).ConfigureAwait(false);
+ }
+
+ return outcome.Result;
}
catch (OperationCanceledException)
{
@@ -270,18 +286,24 @@ private PersistenceOutcome ProcessCore(AcceptedArticle article, CancellationToke
}
}
- if (!_gate.TryReserve(observed, rewritten.Length, out var chosen))
+ var criticalPercent = (_options.Value.SegmentPools ?? new SegmentPoolsOptions()).CriticalPercent;
+ if (!_gate.TryReserve(observed, rewritten.Length, criticalPercent, out var chosen))
{
_pressure.SetSaturated(true);
_counters.NoteDeferred();
- if (_logger.IsEnabled(LogLevel.Warning))
- {
- SpoolPersistenceLogMessages.Deferred(_logger, article.ArtId.ToLowerHexString());
- }
-
+ LogDeferredThrottled(article.ArtId);
return PersistenceOutcome.Done(AcceptedArticleProcessResult.WaitingForCapacity);
}
+ if (_logger.IsEnabled(LogLevel.Debug))
+ {
+ SpoolPersistenceLogMessages.SpoolSelected(
+ _logger,
+ article.ArtId.ToLowerHexString(),
+ chosen.SpoolRoot,
+ chosen.DeviceId);
+ }
+
_pressure.SetSaturated(false);
try
{
@@ -297,7 +319,7 @@ private PersistenceOutcome ProcessCore(AcceptedArticle article, CancellationToke
}
finally
{
- _gate.Exit(chosen.SpoolRoot);
+ _gate.Exit(chosen);
}
}
}
@@ -413,6 +435,29 @@ private AcceptedArticleProcessResult Apply(
return AcceptedArticleProcessResult.WaitingForCapacity;
}
}
+
+ /// Logs at most one deferred warning every five seconds to avoid retry log floods.
+ private void LogDeferredThrottled(ArticleId articleId)
+ {
+ if (!_logger.IsEnabled(LogLevel.Warning))
+ {
+ return;
+ }
+
+ var now = _time.GetTimestamp();
+ var previous = Volatile.Read(ref _lastDeferredLogAt);
+ if (previous != 0 && _time.GetElapsedTime(previous, now) < TimeSpan.FromSeconds(5))
+ {
+ return;
+ }
+
+ if (Interlocked.CompareExchange(ref _lastDeferredLogAt, now, previous) != previous)
+ {
+ return;
+ }
+
+ SpoolPersistenceLogMessages.Deferred(_logger, articleId.ToLowerHexString());
+ }
}
/// Publish result plus whether the caller should wait for an index commit.
diff --git a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolCapacityWaitSignal.cs b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolCapacityWaitSignal.cs
new file mode 100644
index 00000000..10c6e214
--- /dev/null
+++ b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolCapacityWaitSignal.cs
@@ -0,0 +1,48 @@
+using VectorNNTPD.StorageService.Storage.SegmentPools;
+
+namespace VectorNNTPD.StorageService.Storage.SpoolPersistence
+{
+ ///
+ /// Wakes persistence workers that are holding an article because no spool was eligible.
+ /// Combined with the bounded IneligibleRetryMilliseconds poll so a notify between
+ /// eligibility check and wait subscription cannot be lost when the wait task is captured first.
+ ///
+ internal sealed class SpoolCapacityWaitSignal
+ {
+ private readonly object _gate = new();
+ private TaskCompletionSource _pulse =
+ new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ /// Subscribes to filesystem capacity publishes for the process lifetime.
+ public SpoolCapacityWaitSignal(BlockDeviceMonitorService monitor)
+ {
+ ArgumentNullException.ThrowIfNull(monitor);
+ monitor.CapacitySamplePublished += Notify;
+ }
+
+ ///
+ /// Captures the current wait task. Call before attempting placement so a concurrent
+ /// completes this task even if it races the attempt.
+ ///
+ internal Task CaptureWait()
+ {
+ lock (_gate)
+ {
+ return _pulse.Task;
+ }
+ }
+
+ /// Signals that capacity may have recovered (fresh sample, eviction reclaim, or desaturation).
+ internal void Notify()
+ {
+ TaskCompletionSource previous;
+ lock (_gate)
+ {
+ previous = _pulse;
+ _pulse = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ }
+
+ previous.TrySetResult();
+ }
+ }
+}
diff --git a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPersistenceLogMessages.cs b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPersistenceLogMessages.cs
index 68aaf62e..a7a58f52 100644
--- a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPersistenceLogMessages.cs
+++ b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPersistenceLogMessages.cs
@@ -66,5 +66,12 @@ internal static partial class SpoolPersistenceLogMessages
/// Bytes reserved for admitted bodies that are not queued yet.
[LoggerMessage(EventId = 5707, Level = LogLevel.Information, Message = "Spool persistence stored={Stored} storedBytes={StoredBytes} deferred={Deferred} activeWorkers={Active} queued={Queued} queuedBytes={QueuedBytes} reservedBytes={ReservedBytes}")]
public static partial void Summary(ILogger logger, long stored, long storedBytes, long deferred, int active, int queued, long queuedBytes, long reservedBytes);
+
+ /// Logs the spool chosen for an article by greatest available capacity.
+ [LoggerMessage(
+ EventId = 5708,
+ Level = LogLevel.Debug,
+ Message = "Spool selected for article articleId={ArticleId} spool={Spool} device={DeviceId}")]
+ public static partial void SpoolSelected(ILogger logger, string articleId, string spool, string deviceId);
}
}
diff --git a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPressure.cs b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPressure.cs
index 6bcda829..51fabc7c 100644
--- a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPressure.cs
+++ b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPressure.cs
@@ -9,13 +9,34 @@ namespace VectorNNTPD.StorageService.Storage.SpoolPersistence
///
internal sealed class SpoolPressure
{
+ private readonly SpoolCapacityWaitSignal? _waitSignal;
private int _blocked;
+ /// Initializes pressure without a wait signal.
+ public SpoolPressure()
+ {
+ }
+
+ /// Initializes pressure that wakes capacity-blocked workers when saturation clears.
+ public SpoolPressure(SpoolCapacityWaitSignal waitSignal)
+ {
+ ArgumentNullException.ThrowIfNull(waitSignal);
+ _waitSignal = waitSignal;
+ }
+
/// Whether the control loop must not add workers.
internal bool IsSaturated => Volatile.Read(ref _blocked) != 0;
/// Records whether another publish can be placed.
/// when no spool could accept the article.
- internal void SetSaturated(bool blocked) => Volatile.Write(ref _blocked, blocked ? 1 : 0);
+ internal void SetSaturated(bool blocked)
+ {
+ var next = blocked ? 1 : 0;
+ var previous = Interlocked.Exchange(ref _blocked, next);
+ if (previous != 0 && next == 0)
+ {
+ _waitSignal?.Notify();
+ }
+ }
}
}
diff --git a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelectionPolicy.cs b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelectionPolicy.cs
index ec66bd8f..ab0eb255 100644
--- a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelectionPolicy.cs
+++ b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelectionPolicy.cs
@@ -3,27 +3,25 @@ namespace VectorNNTPD.StorageService.Storage.SpoolPersistence
///
/// Documented ranking and failure-retry constants for .
- /// LowPercent, HighPercent, and CriticalPercent are not read here.
- /// Those remain future eviction thresholds and do not admit or reject a write.
+ /// CriticalPercent is supplied by the caller and excludes spools above that used percent.
+ /// HighPercent/LowPercent drive per-spool eviction monitors, not placement ranking.
///
internal static class SpoolSelectionPolicy
{
///
/// A device sample or free-space probe older than this is not used as a measurement.
- /// A stale sample is not zero, not idle, and not a reason to skip the spool.
+ /// A stale or missing free-space sample makes the spool ineligible for placement.
///
internal static readonly TimeSpan SampleFreshness = TimeSpan.FromSeconds(5);
///
- /// Free-space values within this fraction of the larger value are tied.
- /// One percent ignores megabyte noise on a multi-terabyte spool and still
- /// prefers a spool that is materially emptier.
+ /// Historical free-space tie fraction. Placement now ranks strictly by available bytes;
+ /// this constant remains for tests and documentation of prior tie-band behaviour.
///
internal const double FreeSpaceTieFraction = 0.01;
///
- /// Smallest free-space gap that breaks a tie. One gibibyte.
- /// Below this gap, in-flight writes, fresh measurements, and the spool path decide.
+ /// Historical free-space tie floor (one gibibyte). Placement now ranks strictly by available bytes.
///
internal const long FreeSpaceTieMinimumBytes = 1L << 30;
diff --git a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelector.cs b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelector.cs
index 85187d66..687c7cae 100644
--- a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelector.cs
+++ b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelector.cs
@@ -1,34 +1,38 @@
namespace VectorNNTPD.StorageService.Storage.SpoolPersistence
{
-
- /// The spool chosen for one article.
+ /// The spool chosen for one article, including the pending-byte lease to release on .
/// Canonical directory that will receive the file.
/// Device that backs .
/// Whether the chosen spool's fresh device sample is past a busy ranking line. Busy still accepts the write.
- internal readonly record struct SpoolSelection(string SpoolRoot, string DeviceId, bool Busy);
+ /// Article bytes reserved against pending capacity.
+ internal readonly record struct SpoolSelection(string SpoolRoot, string DeviceId, bool Busy, long ReservedBytes = 0);
///
- /// Picks one spool that can accept a filesystem write.
- /// Free space wins when the gap is material. Load, in-flight work, and path break a capacity tie.
- /// Busy time, stale samples, and used-space percent do not reject a spool.
+ /// Picks the eligible spool with the greatest available capacity in bytes.
+ /// Unknown/stale space and used percent above CriticalPercent are excluded.
+ /// Enumeration order does not change the result; path ordinal breaks exact ties.
///
internal static class SpoolSelector
{
///
/// Selects the best writable candidate. Returns null when every candidate is blocked,
- /// at its in-flight cap, or known not to fit .
- /// Enumeration order does not change the result.
+ /// at its in-flight cap, lacks a valid capacity sample, is above Critical, or cannot fit
+ /// .
///
/// Current spool observations. The method does not mutate them.
/// Rewritten article length. A fresh sample below this cannot fit the file.
+ /// Used percent above which a spool is ineligible (strict greater-than).
/// The selected spool, or null.
- internal static SpoolSelection? Select(ReadOnlySpan candidates, long requiredBytes)
+ internal static SpoolSelection? Select(
+ ReadOnlySpan candidates,
+ long requiredBytes,
+ int criticalPercent)
{
var found = false;
var best = default(SpoolCandidate);
foreach (var candidate in candidates)
{
- if (!IsEligible(candidate, requiredBytes))
+ if (!IsEligible(candidate, requiredBytes, criticalPercent))
{
continue;
}
@@ -49,7 +53,7 @@ internal static class SpoolSelector
}
///
- /// Used-space percent from filesystem totals. Not a write gate.
+ /// Used-space percent from filesystem totals.
/// The caller has already required a positive total.
///
/// Filesystem capacity in bytes.
@@ -67,7 +71,6 @@ internal static int UsedPercent(long totalBytes, long availableBytes)
return 0;
}
- // available * 100 overflows long when the volume is larger than long.MaxValue / 100.
if (totalBytes > long.MaxValue / 100)
{
return (int)Math.Min(100d, Math.Max(0d, 100d - (availableBytes * 100d / totalBytes)));
@@ -78,15 +81,11 @@ internal static int UsedPercent(long totalBytes, long availableBytes)
}
///
- /// A spool may be written when it is initialized, not closed for an operational failure,
- /// under its in-flight cap, and a fresh free-space sample can hold .
- /// Missing or stale free space does not reject it. Used-space percent does not reject it.
- /// Device load does not reject it.
+ /// A spool may be written when it is initialized, accepts writes, under its in-flight cap,
+ /// has a fresh consistent free-space sample that can hold ,
+ /// and used percent is at or below .
///
- /// One spool observation.
- /// Rewritten article length. Zero or negative is treated as one byte.
- /// when the spool may receive this article.
- internal static bool IsEligible(in SpoolCandidate candidate, long requiredBytes)
+ internal static bool IsEligible(in SpoolCandidate candidate, long requiredBytes, int criticalPercent)
{
if (!candidate.Initialized || !candidate.AcceptsWrites)
{
@@ -98,16 +97,18 @@ internal static bool IsEligible(in SpoolCandidate candidate, long requiredBytes)
return false;
}
- if (!candidate.SpaceKnown)
+ if (!candidate.SpaceKnown
+ || candidate.TotalBytes <= 0
+ || candidate.AvailableBytes < 0
+ || candidate.AvailableBytes > candidate.TotalBytes)
{
- return true;
+ return false;
}
- if (candidate.TotalBytes <= 0
- || candidate.AvailableBytes < 0
- || candidate.AvailableBytes > candidate.TotalBytes)
+ var used = UsedPercent(candidate.TotalBytes, candidate.AvailableBytes);
+ if (used > criticalPercent)
{
- return true;
+ return false;
}
var needed = requiredBytes < 1 ? 1 : requiredBytes;
@@ -116,11 +117,8 @@ internal static bool IsEligible(in SpoolCandidate candidate, long requiredBytes)
///
/// Busy means a fresh sample measured active time, write latency, or queue depth past a ranking line.
- /// A null measurement is not busy and is not zero. A stale sample is not busy.
/// Busy does not make the spool ineligible.
///
- /// One spool observation.
- /// when a fresh measurement is past a busy ranking line.
internal static bool IsBusy(in SpoolCandidate candidate)
{
if (!candidate.DeviceSampleFresh)
@@ -148,42 +146,15 @@ internal static bool IsBusy(in SpoolCandidate candidate)
///
/// Compares two writable spools. A negative result means is better.
- /// A material free-space gap wins first. Inside that band, or when free space is unknown,
- /// measured-light beats unknown load, and unknown load beats measured-busy.
- /// Then fewer in-flight writes, then lower write latency when both fresh samples measured it,
- /// then the ordinal path. A spool with fresh free space ranks ahead of one without it.
+ /// Greatest available bytes wins first. Equal available bytes break by fewer in-flight writes,
+ /// then lower write latency when both measured, then ordinal spool path.
///
- /// First writable spool.
- /// Second writable spool.
- /// Negative when should be chosen.
internal static int Compare(in SpoolCandidate left, in SpoolCandidate right)
{
- var leftKnown = HasUsableSpace(left);
- var rightKnown = HasUsableSpace(right);
- if (leftKnown != rightKnown)
- {
- return leftKnown ? -1 : 1;
- }
-
- if (leftKnown)
- {
- var threshold = FreeSpaceTieBytes(left.AvailableBytes, right.AvailableBytes);
- var gap = left.AvailableBytes - right.AvailableBytes;
- if (gap > threshold)
- {
- return -1;
- }
-
- if (gap < -threshold)
- {
- return 1;
- }
- }
-
- var load = LoadRank(left) - LoadRank(right);
- if (load != 0)
+ var availableOrder = right.AvailableBytes.CompareTo(left.AvailableBytes);
+ if (availableOrder != 0)
{
- return load;
+ return availableOrder;
}
var flight = left.InFlightWrites - right.InFlightWrites;
@@ -197,9 +168,6 @@ internal static int Compare(in SpoolCandidate left, in SpoolCandidate right)
&& left.WriteLatencyMilliseconds is { } leftLatency
&& right.WriteLatencyMilliseconds is { } rightLatency)
{
- // Exact ordering via CompareTo (same bit pattern ⇒ 0). No epsilon: approximate
- // equality is not transitive and would break this comparator. Equal samples fall
- // through to the deterministic path ordinal below.
var latencyOrder = leftLatency.CompareTo(rightLatency);
if (latencyOrder != 0)
{
@@ -209,38 +177,5 @@ internal static int Compare(in SpoolCandidate left, in SpoolCandidate right)
return string.CompareOrdinal(left.SpoolRoot, right.SpoolRoot);
}
-
- /// Bytes that must separate two free-space figures before free space decides the rank.
- /// Available bytes on the first spool.
- /// Available bytes on the second spool.
- /// The larger of one gibibyte and one percent of the larger available value.
- internal static long FreeSpaceTieBytes(long leftAvailable, long rightAvailable)
- {
- var larger = Math.Max(0, Math.Max(leftAvailable, rightAvailable));
- var fraction = (long)(larger * SpoolSelectionPolicy.FreeSpaceTieFraction);
- return Math.Max(SpoolSelectionPolicy.FreeSpaceTieMinimumBytes, fraction);
- }
-
- /// Fresh, consistent free space that ranking may compare. Inconsistent numbers are unknown.
- /// One spool observation.
- /// when available bytes are a fresh sample inside the reported total.
- private static bool HasUsableSpace(in SpoolCandidate candidate) =>
- candidate.SpaceKnown
- && candidate.TotalBytes > 0
- && candidate.AvailableBytes >= 0
- && candidate.AvailableBytes <= candidate.TotalBytes;
-
- /// Zero for a fresh light sample, one for unknown load, two for a fresh busy sample.
- /// One spool observation.
- /// The load class. Lower is better. Unknown is not zero.
- private static int LoadRank(in SpoolCandidate candidate)
- {
- if (!candidate.DeviceSampleFresh)
- {
- return 1;
- }
-
- return IsBusy(candidate) ? 2 : 0;
- }
}
}
diff --git a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolWriteGate.cs b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolWriteGate.cs
index 430f8939..27c77497 100644
--- a/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolWriteGate.cs
+++ b/src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolWriteGate.cs
@@ -1,22 +1,31 @@
namespace VectorNNTPD.StorageService.Storage.SpoolPersistence
{
-
- /// Counts in-flight publishes per spool. The count is the selector's in-flight input.
+ ///
+ /// Counts in-flight publishes and pending article bytes per spool/device.
+ /// Pending bytes are keyed by device id so directories that share a volume share projected capacity.
+ /// Each lease is released exactly once via
+ /// so concurrent writers on the same spool cannot pop each other's pending bytes.
+ ///
internal sealed class SpoolWriteGate
{
private readonly Dictionary _inFlight = new(StringComparer.Ordinal);
+ private readonly Dictionary _pendingByDevice = new(StringComparer.Ordinal);
private readonly object _gate = new();
///
- /// Selects the best eligible spool using the in-flight counts under this lock, then reserves it.
- /// Concurrent callers cannot all reserve the same spool from one stale count.
- /// is updated in place with the locked in-flight counts.
+ /// Selects the best eligible spool using in-flight counts and pending bytes under this lock,
+ /// then reserves it. Concurrent callers cannot all herd onto one obsolete free-space reading.
///
- /// Candidates from the latest monitor snapshot. Their in-flight fields are replaced.
- /// Rewritten article length passed to the selector.
+ /// Candidates from the latest monitor snapshot. In-flight and available fields are adjusted in place.
+ /// Rewritten article length passed to the selector and reserved on success.
+ /// Used percent above which a spool is ineligible.
/// The reserved spool when the method returns .
- /// when every spool is blocked, at the cap, or known not to fit.
- internal bool TryReserve(Span observed, long requiredBytes, out SpoolSelection selection)
+ /// when every spool is ineligible or cannot fit.
+ internal bool TryReserve(
+ Span observed,
+ long requiredBytes,
+ int criticalPercent,
+ out SpoolSelection selection)
{
lock (_gate)
{
@@ -26,14 +35,39 @@ internal bool TryReserve(Span observed, long requiredBytes, out
return false;
}
+ var needed = requiredBytes < 1 ? 1 : requiredBytes;
for (var i = 0; i < observed.Length; i++)
{
var candidate = observed[i];
var inFlight = _inFlight.TryGetValue(candidate.SpoolRoot, out var current) ? current : 0;
- observed[i] = candidate with { InFlightWrites = inFlight };
+ var accepts = candidate.AcceptsWrites;
+ if (candidate.SpaceKnown
+ && candidate.TotalBytes > 0
+ && SpoolSelector.UsedPercent(candidate.TotalBytes, candidate.AvailableBytes) > criticalPercent)
+ {
+ // Critical uses the measured sample, not pending-projected available bytes.
+ accepts = false;
+ }
+
+ var pending = _pendingByDevice.TryGetValue(candidate.DeviceId, out var reserved)
+ ? reserved
+ : 0;
+ var available = candidate.AvailableBytes;
+ if (candidate.SpaceKnown && pending > 0)
+ {
+ available = available > pending ? available - pending : 0;
+ }
+
+ observed[i] = candidate with
+ {
+ InFlightWrites = inFlight,
+ AvailableBytes = available,
+ AcceptsWrites = accepts,
+ };
}
- var chosen = SpoolSelector.Select(observed, requiredBytes);
+ // Critical already applied above; pass 100 so projected available cannot re-trigger it.
+ var chosen = SpoolSelector.Select(observed, needed, criticalPercent: 100);
if (chosen is not { } selected)
{
selection = default;
@@ -41,15 +75,14 @@ internal bool TryReserve(Span observed, long requiredBytes, out
}
_inFlight[selected.SpoolRoot] = (_inFlight.TryGetValue(selected.SpoolRoot, out var count) ? count : 0) + 1;
- selection = selected;
+ _pendingByDevice[selected.DeviceId] =
+ (_pendingByDevice.TryGetValue(selected.DeviceId, out var pendingBytes) ? pendingBytes : 0) + needed;
+ selection = selected with { ReservedBytes = needed };
return true;
}
}
/// Reserves one publish slot when the spool is under .
- /// Canonical spool path.
- /// Per-spool cap. Must be at least 1.
- /// when the spool is already at the cap.
internal bool TryEnter(string spoolRoot, int limit)
{
lock (_gate)
@@ -65,30 +98,75 @@ internal bool TryEnter(string spoolRoot, int limit)
}
}
- /// Releases one publish slot. Extra releases are ignored.
+ ///
+ /// Releases the pending-byte lease and publish slot from .
+ /// Safe under concurrent writers: subtracts this selection's bytes only.
+ ///
+ /// Lease returned by .
+ internal void Exit(in SpoolSelection selection)
+ {
+ if (string.IsNullOrEmpty(selection.SpoolRoot))
+ {
+ return;
+ }
+
+ lock (_gate)
+ {
+ ReleasePending(selection.DeviceId, selection.ReservedBytes);
+ DecrementInFlight(selection.SpoolRoot);
+ }
+ }
+
+ /// Releases one publish slot from (no pending-byte lease).
/// Canonical spool path.
internal void Exit(string spoolRoot)
{
lock (_gate)
{
- if (!_inFlight.TryGetValue(spoolRoot, out var current) || current <= 0)
- {
- return;
- }
+ DecrementInFlight(spoolRoot);
+ }
+ }
- if (current == 1)
- {
- _inFlight.Remove(spoolRoot);
- return;
- }
+ private void ReleasePending(string deviceId, long bytes)
+ {
+ if (string.IsNullOrEmpty(deviceId) || bytes <= 0)
+ {
+ return;
+ }
+
+ if (!_pendingByDevice.TryGetValue(deviceId, out var pending))
+ {
+ return;
+ }
- _inFlight[spoolRoot] = current - 1;
+ var next = pending > bytes ? pending - bytes : 0;
+ if (next == 0)
+ {
+ _pendingByDevice.Remove(deviceId);
+ }
+ else
+ {
+ _pendingByDevice[deviceId] = next;
}
}
+ private void DecrementInFlight(string spoolRoot)
+ {
+ if (!_inFlight.TryGetValue(spoolRoot, out var current) || current <= 0)
+ {
+ return;
+ }
+
+ if (current == 1)
+ {
+ _inFlight.Remove(spoolRoot);
+ return;
+ }
+
+ _inFlight[spoolRoot] = current - 1;
+ }
+
/// Current reservations for .
- /// Canonical spool path.
- /// The in-flight count. Zero when the spool has none.
internal int InFlight(string spoolRoot)
{
lock (_gate)
@@ -96,5 +174,14 @@ internal int InFlight(string spoolRoot)
return _inFlight.TryGetValue(spoolRoot, out var current) ? current : 0;
}
}
+
+ /// Pending article bytes projected against .
+ internal long PendingBytes(string deviceId)
+ {
+ lock (_gate)
+ {
+ return _pendingByDevice.TryGetValue(deviceId, out var pending) ? pending : 0;
+ }
+ }
}
}
diff --git a/src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.json b/src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.json
index 3c87f8c0..f2707c92 100644
--- a/src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.json
+++ b/src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.json
@@ -9,6 +9,7 @@
"BindPortTls": 1191,
"GracefulShutdownTimeout": "00:00:30",
"Listener": {
+ "ArticleReceiveTimeoutSeconds": 300,
"IoProgressTimeoutSeconds": 60,
"MaxActiveConnections": 1024,
"ParserAccumulationMaxBytes": 262144,
diff --git a/tests/VectorNNTPD.Common.Tests/Articles/CanonicalArticleDeliveryFramingTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/CanonicalArticleDeliveryFramingTests.cs
new file mode 100644
index 00000000..3b51b505
--- /dev/null
+++ b/tests/VectorNNTPD.Common.Tests/Articles/CanonicalArticleDeliveryFramingTests.cs
@@ -0,0 +1,64 @@
+using VectorNNTPD.Common.Articles;
+
+namespace VectorNNTPD.Common.Tests.Articles
+{
+ ///
+ /// Shared CanonicalV1 → NNTP multiline delivery framing: header-only transport stuffing and rejection.
+ ///
+ public sealed class CanonicalArticleDeliveryFramingTests
+ {
+ /// NUL anywhere in the article makes framing and header stuffing fail without mutating input bytes.
+ [Fact]
+ public void Nul_is_not_frameable()
+ {
+ var article = "Subject: s\r\n\r\na\0b\r\n"u8.ToArray();
+ Assert.False(CanonicalArticleDeliveryFraming.IsFrameable(article));
+ Assert.False(CanonicalArticleDeliveryFraming.TryStuffHeaderLines(article, out var stuffed));
+ Assert.Null(stuffed);
+ Assert.Equal("Subject: s\r\n\r\na\0b\r\n"u8, article);
+ }
+
+ /// Dot-leading content before a header/body separator is not treated as a deliverable NNTP article.
+ [Fact]
+ public void A_dot_leading_line_without_a_header_body_separator_is_not_frameable()
+ {
+ var article = ".X-Dot: kept\r\n"u8.ToArray();
+ Assert.False(CanonicalArticleDeliveryFraming.IsFrameable(article));
+ Assert.False(CanonicalArticleDeliveryFraming.TryStuffHeaderLines(article, out var stuffed));
+ Assert.Null(stuffed);
+ }
+
+ /// Dot-leading headers gain one stuffing dot while the caller's article array is not modified in place.
+ [Fact]
+ public void A_dot_leading_header_is_stuffed_and_the_retained_bytes_stay_unchanged()
+ {
+ var article = ".X-Dot: kept\r\n\r\n.only\r\n"u8.ToArray();
+ var retained = article.ToArray();
+ Assert.True(CanonicalArticleDeliveryFraming.IsFrameable(article));
+ Assert.True(CanonicalArticleDeliveryFraming.TryStuffHeaderLines(article, out var stuffed));
+ Assert.Equal("..X-Dot: kept\r\n\r\n"u8.ToArray(), stuffed);
+ Assert.Equal(".only\r\n"u8.ToArray(), article.AsSpan(".X-Dot: kept\r\n\r\n"u8.Length).ToArray());
+ Assert.Equal(retained, article);
+ }
+
+ /// Body lines that already carry CanonicalV1 stuffing are not part of the stuffed-header buffer.
+ [Fact]
+ public void Stuffing_headers_does_not_copy_or_restuff_the_body()
+ {
+ var article = ".X-One: a\r\n\r\n.only\r\n..\r\n...three\r\n"u8.ToArray();
+ Assert.True(CanonicalArticleDeliveryFraming.TryStuffHeaderLines(article, out var stuffed));
+ Assert.Equal("..X-One: a\r\n\r\n"u8.ToArray(), stuffed);
+ Assert.Equal(".only\r\n..\r\n...three\r\n"u8.ToArray(), article.AsSpan(".X-One: a\r\n\r\n"u8.Length).ToArray());
+ }
+
+ /// No header dot means the caller keeps the retained article and no socket image is allocated.
+ [Fact]
+ public void A_header_without_a_leading_dot_does_not_allocate_a_socket_image()
+ {
+ var article = "Subject: s\r\n\r\n.only\r\n"u8.ToArray();
+ var bodyStart = CanonicalArticleFraming.Analyze(article).BodyStart;
+ Assert.False(CanonicalArticleDeliveryFraming.TryCreateStuffedHeader(article, bodyStart, out var stuffed));
+ Assert.Null(stuffed);
+ }
+ }
+}
diff --git a/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs b/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs
index 4d72aae9..37730f94 100644
--- a/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs
+++ b/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs
@@ -280,6 +280,7 @@ private static void AssertProductionOrder(IApplicationService[] ordered)
var shardIndex = Array.FindIndex(ordered, static service => service is SpoolShardInitializationService);
var indexIndex = Array.FindIndex(ordered, static service => service is ArticleIndexService);
var monitorIndex = Array.FindIndex(ordered, static service => service is BlockDeviceMonitorService);
+ var evictionIndex = Array.FindIndex(ordered, static service => service is SpoolEvictionMonitorService);
var workerIndex = Array.FindIndex(ordered, static service => service is VectorNNTPD.StorageService.Ingress.AcceptedArticleWorker);
Assert.True(databaseIndex >= 0 && databaseIndex < sharedIndex);
Assert.True(sharedIndex < rabbitIndex);
@@ -288,7 +289,8 @@ private static void AssertProductionOrder(IApplicationService[] ordered)
Assert.True(validationIndex < shardIndex);
Assert.True(shardIndex < indexIndex);
Assert.True(indexIndex < monitorIndex);
- Assert.True(monitorIndex < workerIndex);
+ Assert.True(monitorIndex < evictionIndex);
+ Assert.True(evictionIndex < workerIndex);
Assert.True(workerIndex < listenerIndex);
Assert.Equal(1, ordered.Count(static service => service is VectorNNTPD.StorageService.Ingress.AcceptedArticleWorker));
Assert.True(listenerIndex < publisherIndex);
diff --git a/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerHostCompositionTests.cs b/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerHostCompositionTests.cs
index 624b003b..b0dc5329 100644
--- a/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerHostCompositionTests.cs
+++ b/tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerHostCompositionTests.cs
@@ -57,7 +57,7 @@ public void AddStorageServerHosting_registers_expected_service_graph()
host.Services.GetRequiredService());
var application = host.Services.GetServices().ToArray();
- Assert.Equal(12, application.Length);
+ Assert.Equal(13, application.Length);
Assert.IsType(application[0]);
Assert.IsType(application[1]);
Assert.IsType(application[2]);
@@ -67,9 +67,10 @@ public void AddStorageServerHosting_registers_expected_service_graph()
Assert.IsType(application[6]);
Assert.IsType(application[7]);
Assert.IsType(application[8]);
- Assert.Same(host.Services.GetRequiredService(), application[9]);
- Assert.Same(host.Services.GetRequiredService(), application[10]);
- Assert.IsType(application[11]);
+ Assert.IsType(application[9]);
+ Assert.Same(host.Services.GetRequiredService(), application[10]);
+ Assert.Same(host.Services.GetRequiredService(), application[11]);
+ Assert.IsType(application[12]);
Assert.Equal(1, application.Count(static service => service is AcceptedArticleWorker));
Assert.IsType(host.Services.GetRequiredService());
Assert.IsType(host.Services.GetRequiredService());
@@ -193,10 +194,11 @@ public async Task Host_starts_application_services_in_order_when_acme_and_rabbit
Assert.IsType(application[6]);
Assert.IsType(application[7]);
Assert.IsType(application[8]);
- Assert.IsType(application[9]);
+ Assert.IsType(application[9]);
+ Assert.IsType(application[10]);
var listener = host.Services.GetRequiredService();
- Assert.Same(listener, application[10]);
- Assert.IsType(application[11]);
+ Assert.Same(listener, application[11]);
+ Assert.IsType(application[12]);
Assert.Equal(StorageNntpListenerState.Running, listener.State);
Assert.True(host.Services.GetRequiredService().IsReady);
Assert.True(host.Services.GetRequiredService().IsReady);
diff --git a/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleQueueTests.cs b/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleQueueTests.cs
index f64b15ab..4c43477a 100644
--- a/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleQueueTests.cs
+++ b/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleQueueTests.cs
@@ -114,7 +114,7 @@ public async Task Cancelling_a_dequeue_does_not_lose_a_later_article()
}
[Fact]
- public async Task Byte_budget_rejects_an_enqueue_until_a_dequeue_frees_space()
+ public async Task Byte_budget_rejects_an_enqueue_until_processed_bytes_are_released()
{
var queue = new AcceptedArticleQueue(10, TimeProvider.System);
var first = new AcceptedArticle(default, 1, 6, new byte[6]);
@@ -126,12 +126,19 @@ public async Task Byte_budget_rejects_an_enqueue_until_a_dequeue_frees_space()
var delivery = await queue.DequeueAsync(CancellationToken.None);
Assert.True(delivery.HasArticle);
+ Assert.Equal(0, queue.QueuedBytes);
+ Assert.Equal(6, queue.InFlightBytes);
+ Assert.Equal(6, queue.AccountedBytes);
+ // Dequeue alone must not free admission budget (capacity-blocked holds stay accounted).
+ Assert.False(queue.TryEnqueue(second));
+ queue.ReleaseProcessed(delivery.Article.ArtSize);
+ Assert.Equal(0, queue.AccountedBytes);
Assert.True(queue.TryEnqueue(second));
Assert.Equal(6, queue.QueuedBytes);
}
[Fact]
- public void TryDequeue_releases_the_byte_budget_and_does_not_wait()
+ public void TryDequeue_moves_bytes_to_in_flight_until_ReleaseProcessed()
{
var queue = new AcceptedArticleQueue(10, TimeProvider.System);
var first = new AcceptedArticle(default, 1, 6, new byte[6]);
@@ -145,7 +152,10 @@ public void TryDequeue_releases_the_byte_budget_and_does_not_wait()
Assert.True(delivery.HasArticle);
Assert.Equal(6, delivery.Article.ArtSize);
Assert.Equal(0, queue.QueuedBytes);
+ Assert.Equal(6, queue.InFlightBytes);
Assert.Equal(1, queue.DequeuedCount);
+ Assert.False(queue.TryEnqueue(second));
+ queue.ReleaseProcessed(delivery.Article.ArtSize);
Assert.True(queue.TryEnqueue(second));
Assert.Equal(6, queue.QueuedBytes);
@@ -154,6 +164,32 @@ public void TryDequeue_releases_the_byte_budget_and_does_not_wait()
Assert.True(drained.HasArticle);
Assert.False(queue.TryDequeue(out _));
Assert.Equal(0, queue.QueuedBytes);
+ Assert.Equal(6, queue.InFlightBytes);
+ queue.ReleaseProcessed(drained.Article.ArtSize);
+ Assert.Equal(0, queue.AccountedBytes);
+ }
+
+ [Fact]
+ public void Held_in_flight_articles_keep_accounted_budget_under_capacity_blockage()
+ {
+ var queue = new AcceptedArticleQueue(100, TimeProvider.System);
+ Assert.True(queue.TryEnqueue(new AcceptedArticle(default, 1, 40, new byte[40])));
+ Assert.True(queue.TryEnqueue(new AcceptedArticle(default, 2, 40, new byte[40])));
+ Assert.True(queue.TryEnqueue(new AcceptedArticle(default, 3, 20, new byte[20])));
+ Assert.Equal(100, queue.AccountedBytes);
+
+ Assert.True(queue.TryDequeue(out var a));
+ Assert.True(queue.TryDequeue(out var b));
+ Assert.Equal(20, queue.QueuedBytes);
+ Assert.Equal(80, queue.InFlightBytes);
+ Assert.Equal(100, queue.AccountedBytes);
+ Assert.False(queue.TryEnqueue(new AcceptedArticle(default, 4, 1, new byte[1])));
+
+ queue.ReleaseProcessed(a.Article.ArtSize);
+ Assert.Equal(60, queue.AccountedBytes);
+ Assert.True(queue.TryEnqueue(new AcceptedArticle(default, 4, 40, new byte[40])));
+ Assert.Equal(100, queue.AccountedBytes);
+ queue.ReleaseProcessed(b.Article.ArtSize);
}
[Fact]
@@ -170,6 +206,7 @@ public async Task TryDequeue_and_DequeueAsync_do_not_lose_articles_under_concurr
if ((index & 1) == 0 && queue.TryDequeue(out var ready))
{
Assert.True(seen.TryAdd(ready.Article.ArtSize, 0));
+ queue.ReleaseProcessed(ready.Article.ArtSize);
continue;
}
@@ -179,6 +216,7 @@ public async Task TryDequeue_and_DequeueAsync_do_not_lose_articles_under_concurr
if (delivery.HasArticle)
{
Assert.True(seen.TryAdd(delivery.Article.ArtSize, 0));
+ queue.ReleaseProcessed(delivery.Article.ArtSize);
}
}
catch (OperationCanceledException) when (started.IsCancellationRequested)
@@ -197,6 +235,8 @@ public async Task TryDequeue_and_DequeueAsync_do_not_lose_articles_under_concurr
await Task.WhenAll(consumers);
Assert.Equal(count, seen.Count);
Assert.Equal(0, queue.QueuedBytes);
+ Assert.Equal(0, queue.InFlightBytes);
+ Assert.Equal(0, queue.AccountedBytes);
Assert.Equal(count, queue.DequeuedCount);
}
diff --git a/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerDrainTests.cs b/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerDrainTests.cs
new file mode 100644
index 00000000..da38f8f0
--- /dev/null
+++ b/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerDrainTests.cs
@@ -0,0 +1,243 @@
+using System.Text;
+using Microsoft.Extensions.Logging.Abstractions;
+using VectorNNTPD.Common.Articles;
+using VectorNNTPD.Common.Articles.Parsing;
+using VectorNNTPD.StorageService.Configuration;
+using VectorNNTPD.StorageService.Ingress;
+using VectorNNTPD.StorageService.Storage.SpoolPersistence;
+
+namespace VectorNNTPD.StorageService.Tests.Ingress
+{
+ ///
+ /// Graceful-shutdown drain contract: every article accepted before queue completion is awaited,
+ /// including in-flight persistence, and a failed persist fails stop.
+ ///
+ public sealed class AcceptedArticleWorkerDrainTests
+ {
+ private const string CacheFqdn = "cache01.usenet.ninja";
+ private static readonly TimeSpan Safety = TimeSpan.FromSeconds(10);
+
+ [Fact]
+ public async Task Stop_drains_multiple_queued_articles_to_the_processor()
+ {
+ var queue = new AcceptedArticleQueue();
+ var processed = new List();
+ var processor = new RecordingProcessor(article =>
+ {
+ lock (processed)
+ {
+ processed.Add(Encoding.ASCII.GetString(article.ArtData.Span));
+ }
+
+ return AcceptedArticleProcessResult.Completed;
+ });
+ var worker = new AcceptedArticleWorker(queue, processor, NullLogger.Instance);
+ await worker.StartAsync(CancellationToken.None);
+
+ Assert.True(queue.TryEnqueue(ToQueued(Build("", "alpha\r\n"))));
+ Assert.True(queue.TryEnqueue(ToQueued(Build("", "beta\r\n"))));
+ Assert.True(queue.TryEnqueue(ToQueued(Build("", "gamma\r\n"))));
+ queue.Complete();
+
+ await worker.StopAsync(CancellationToken.None);
+
+ Assert.Equal(3, processor.Completed);
+ Assert.Equal(0, queue.QueuedCount);
+ Assert.Equal(0, queue.QueuedBytes);
+ Assert.Equal(0, queue.ReservedBytes);
+ Assert.Equal(0, worker.ActiveWorkers);
+ Assert.True(worker.Execution!.IsCompletedSuccessfully);
+ lock (processed)
+ {
+ Assert.Contains(processed, static text => text.Contains("alpha\r\n", StringComparison.Ordinal));
+ Assert.Contains(processed, static text => text.Contains("beta\r\n", StringComparison.Ordinal));
+ Assert.Contains(processed, static text => text.Contains("gamma\r\n", StringComparison.Ordinal));
+ }
+ }
+
+ [Fact]
+ public async Task Stop_waits_for_an_in_flight_article_before_returning()
+ {
+ var queue = new AcceptedArticleQueue();
+ var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var processor = new RecordingProcessor(async (article, cancellationToken) =>
+ {
+ entered.TrySetResult();
+ await release.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
+ return AcceptedArticleProcessResult.Completed;
+ });
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ new SpoolPersistenceOptions { MinWorkers = 1, MaxWorkers = 1 });
+ await worker.StartAsync(CancellationToken.None);
+ Assert.True(queue.TryEnqueue(ToQueued(Build("", "held-body\r\n"))));
+ await entered.Task.WaitAsync(Safety);
+
+ var stopping = worker.StopAsync(CancellationToken.None);
+ Assert.False(stopping.IsCompleted);
+ await Task.Delay(50);
+ Assert.False(stopping.IsCompleted);
+ Assert.Equal(0, processor.Completed);
+
+ release.TrySetResult();
+ await stopping.WaitAsync(Safety);
+ Assert.Equal(1, processor.Completed);
+ Assert.True(worker.Execution!.IsCompletedSuccessfully);
+ }
+
+ [Fact]
+ public void Complete_rejects_commit_of_a_reservation_taken_before_completion()
+ {
+ var queue = new AcceptedArticleQueue(maxQueuedBytes: 1024, TimeProvider.System);
+ Assert.True(queue.TryReserve(64, out var reservation));
+ queue.Complete();
+ Assert.False(queue.TryCommit(reservation, CreateExact(64)));
+ Assert.Equal(0, queue.QueuedCount);
+ Assert.Equal(0, queue.ReservedBytes);
+ }
+
+ [Fact]
+ public async Task Stop_throws_when_a_processor_reports_failed()
+ {
+ var queue = new AcceptedArticleQueue();
+ var worker = new AcceptedArticleWorker(
+ queue,
+ new RecordingProcessor(_ => AcceptedArticleProcessResult.Failed),
+ NullLogger.Instance);
+ await worker.StartAsync(CancellationToken.None);
+ Assert.True(queue.TryEnqueue(ToQueued(Build("", "fail-body\r\n"))));
+ queue.Complete();
+
+ var ex = await Assert.ThrowsAsync(() => worker.StopAsync(CancellationToken.None));
+ Assert.Contains("failed or abandoned", ex.Message, StringComparison.Ordinal);
+ Assert.True(worker.Execution!.IsCompletedSuccessfully);
+ }
+
+ [Fact]
+ public async Task Host_cancellation_during_in_flight_work_fails_stop_after_abandon()
+ {
+ var queue = new AcceptedArticleQueue();
+ var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var processor = new RecordingProcessor(async (_, cancellationToken) =>
+ {
+ entered.TrySetResult();
+ await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false);
+ return AcceptedArticleProcessResult.Completed;
+ });
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ new SpoolPersistenceOptions { MinWorkers = 1, MaxWorkers = 1 });
+ await worker.StartAsync(CancellationToken.None);
+ Assert.True(queue.TryEnqueue(ToQueued(Build("", "cancel-body\r\n"))));
+ await entered.Task.WaitAsync(Safety);
+
+ using var stop = new CancellationTokenSource(TimeSpan.FromMilliseconds(100));
+ var ex = await Assert.ThrowsAsync(() => worker.StopAsync(stop.Token));
+ Assert.Contains("did not finish", ex.Message, StringComparison.Ordinal);
+ Assert.Equal(0, processor.Completed);
+ }
+
+ [Fact]
+ public async Task Stop_with_empty_queue_completes_promptly()
+ {
+ var queue = new AcceptedArticleQueue();
+ var worker = new AcceptedArticleWorker(
+ queue,
+ new RecordingProcessor(_ => AcceptedArticleProcessResult.Completed),
+ NullLogger.Instance);
+ await worker.StartAsync(CancellationToken.None);
+
+ using var stop = new CancellationTokenSource(Safety);
+ await worker.StopAsync(stop.Token);
+ Assert.True(worker.Execution!.IsCompletedSuccessfully);
+ Assert.Equal(0, worker.ActiveWorkers);
+ Assert.Equal(0, queue.QueuedBytes);
+ Assert.Equal(0, queue.ReservedBytes);
+ }
+
+ [Fact]
+ public async Task Stop_completes_the_queue_when_the_listener_has_not()
+ {
+ var queue = new AcceptedArticleQueue();
+ var worker = new AcceptedArticleWorker(
+ queue,
+ new RecordingProcessor(_ => AcceptedArticleProcessResult.Completed),
+ NullLogger.Instance);
+ await worker.StartAsync(CancellationToken.None);
+ Assert.False(queue.IsCompleted);
+ await worker.StopAsync(CancellationToken.None);
+ Assert.True(queue.IsCompleted);
+ }
+
+ private static AcceptedArticle ToQueued(Built inbound) =>
+ new(inbound.Record.ArtId, inbound.Record.ArtHash, inbound.Record.ArtSize, inbound.Record.ArtData);
+
+ private static AcceptedArticle CreateExact(int size)
+ {
+ var data = new byte[size];
+ Array.Fill(data, (byte)'z');
+ return new AcceptedArticle(ArticleId.FromMessageId(""u8), 1, size, data);
+ }
+
+ private static Built Build(string messageId, string body)
+ {
+ var parser = new NntpArticleParser(CacheFqdn);
+ var text = new StringBuilder()
+ .Append("Path: peer.example\r\n")
+ .Append("Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n")
+ .Append("Message-ID: ").Append(messageId).Append("\r\n")
+ .Append("Newsgroups: alt.test\r\n")
+ .Append("From: user@example.test\r\n")
+ .Append("Subject: drain\r\n")
+ .Append("\r\n")
+ .Append(body);
+ var created = ArticleRecordFactory.TryCreate(
+ parser,
+ Encoding.ASCII.GetBytes(text.ToString()),
+ ArticlePathMode.Normalize);
+ Assert.True(created.IsAccepted, created.MaterializeFailure.ToString());
+ return new Built(created.Record);
+ }
+
+ private readonly record struct Built(ArticleRecord Record);
+
+ private sealed class RecordingProcessor : IAcceptedArticleProcessor
+ {
+ private readonly Func> _handler;
+
+ public RecordingProcessor(Func handler)
+ : this((article, _) => Task.FromResult(handler(article)))
+ {
+ }
+
+ public RecordingProcessor(Func> handler)
+ {
+ _handler = handler;
+ }
+
+ public int Completed { get; private set; }
+
+ public void EnsureReady()
+ {
+ }
+
+ public async ValueTask ProcessAsync(
+ AcceptedArticle article,
+ CancellationToken cancellationToken)
+ {
+ var result = await _handler(article, cancellationToken).ConfigureAwait(false);
+ if (result == AcceptedArticleProcessResult.Completed)
+ {
+ Completed++;
+ }
+
+ return result;
+ }
+ }
+ }
+}
diff --git a/tests/VectorNNTPD.StorageService.Tests/Ingress/CapacityWaitAndShutdownTests.cs b/tests/VectorNNTPD.StorageService.Tests/Ingress/CapacityWaitAndShutdownTests.cs
new file mode 100644
index 00000000..d891b03e
--- /dev/null
+++ b/tests/VectorNNTPD.StorageService.Tests/Ingress/CapacityWaitAndShutdownTests.cs
@@ -0,0 +1,439 @@
+using System.Text;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Time.Testing;
+using VectorNNTPD.Common.Articles;
+using VectorNNTPD.Common.Articles.Parsing;
+using VectorNNTPD.StorageService.Configuration;
+using VectorNNTPD.StorageService.Ingress;
+using VectorNNTPD.StorageService.Storage.SegmentPools;
+using VectorNNTPD.StorageService.Storage.SpoolPersistence;
+
+namespace VectorNNTPD.StorageService.Tests.Ingress
+{
+ ///
+ /// Capacity-blocked persistence: bounded hold, signal wake-up, and shutdown drain semantics.
+ ///
+ public sealed class CapacityWaitAndShutdownTests
+ {
+ private static readonly TimeSpan Safety = TimeSpan.FromSeconds(5);
+
+ [Fact]
+ public async Task Capacity_signal_wakes_blocked_persistence_without_spinning()
+ {
+ var time = new FakeTimeProvider();
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var signal = CreateSignal();
+ var processor = new ControllableCapacityProcessor();
+ var options = new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ };
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ options,
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None);
+ try
+ {
+ Assert.True(queue.TryEnqueue(Article("")));
+ await WaitAsync(() => processor.Calls >= 1);
+ Assert.Equal(0, queue.QueuedCount);
+ Assert.Equal(1, worker.ActiveWorkers);
+
+ var callsBefore = processor.Calls;
+ await Task.Delay(40);
+ Assert.Equal(callsBefore, processor.Calls);
+
+ processor.AllowComplete = true;
+ signal.Notify();
+ await WaitAsync(() => processor.Completed == 1);
+ Assert.True(processor.Calls >= callsBefore + 1);
+ }
+ finally
+ {
+ processor.AllowComplete = true;
+ signal.Notify();
+ await worker.StopAsync(CancellationToken.None);
+ }
+ }
+
+ [Fact]
+ public async Task Notify_during_process_does_not_lose_wakeup()
+ {
+ var time = new FakeTimeProvider();
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var signal = CreateSignal();
+ var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var proceed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var processor = new RaceNotifyProcessor(entered, proceed);
+ var options = new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ };
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ options,
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None);
+ try
+ {
+ Assert.True(queue.TryEnqueue(Article("")));
+ await entered.Task.WaitAsync(Safety);
+ signal.Notify();
+ proceed.TrySetResult();
+ await WaitAsync(() => processor.Completed == 1);
+ }
+ finally
+ {
+ proceed.TrySetResult();
+ await worker.StopAsync(CancellationToken.None);
+ }
+ }
+
+ [Fact]
+ public async Task Multiple_blocked_articles_do_not_create_unbounded_retry_tasks()
+ {
+ var time = new FakeTimeProvider();
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var signal = CreateSignal();
+ var processor = new ControllableCapacityProcessor();
+ var options = new SpoolPersistenceOptions
+ {
+ MinWorkers = 2,
+ MaxWorkers = 2,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ };
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ options,
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None);
+ try
+ {
+ Assert.True(queue.TryEnqueue(Article("")));
+ Assert.True(queue.TryEnqueue(Article("")));
+ Assert.True(queue.TryEnqueue(Article("")));
+ await WaitAsync(() => processor.Calls >= 2 && queue.QueuedCount == 1);
+ Assert.Equal(2, worker.ActiveWorkers);
+ Assert.Equal(1, queue.QueuedCount);
+
+ processor.AllowComplete = true;
+ signal.Notify();
+ await WaitAsync(() => processor.Completed == 3 && queue.QueuedCount == 0);
+ }
+ finally
+ {
+ processor.AllowComplete = true;
+ signal.Notify();
+ await worker.StopAsync(CancellationToken.None);
+ }
+ }
+
+ [Fact]
+ public async Task Stop_waits_while_capacity_blocked_then_completes_after_recovery()
+ {
+ var time = new FakeTimeProvider();
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var signal = CreateSignal();
+ var processor = new ControllableCapacityProcessor();
+ var options = new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ };
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ options,
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None);
+ Assert.True(queue.TryEnqueue(Article("")));
+ await WaitAsync(() => processor.Calls >= 1);
+
+ var stopping = worker.StopAsync(CancellationToken.None);
+ Assert.False(stopping.IsCompleted);
+ await Task.Delay(30);
+ Assert.False(stopping.IsCompleted);
+ Assert.Equal(0, processor.Completed);
+
+ processor.AllowComplete = true;
+ signal.Notify();
+ await stopping.WaitAsync(Safety);
+ Assert.Equal(1, processor.Completed);
+ Assert.Equal(0, queue.QueuedCount);
+ }
+
+ [Fact]
+ public async Task Host_cancellation_during_capacity_wait_fails_drain()
+ {
+ var time = new FakeTimeProvider();
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var signal = CreateSignal();
+ var processor = new ControllableCapacityProcessor();
+ var options = new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ };
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ options,
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None);
+ Assert.True(queue.TryEnqueue(Article("")));
+ await WaitAsync(() => processor.Calls >= 1);
+
+ using var cts = new CancellationTokenSource();
+ var stopping = worker.StopAsync(cts.Token);
+ await Task.Delay(20);
+ Assert.False(stopping.IsCompleted);
+ cts.Cancel();
+ var ex = await Assert.ThrowsAsync(() => stopping.WaitAsync(Safety));
+ Assert.Contains("drain", ex.Message, StringComparison.OrdinalIgnoreCase);
+ }
+
+ [Fact]
+ public async Task Capacity_blocked_hold_keeps_accounted_budget_and_blocks_further_admission()
+ {
+ var time = new FakeTimeProvider();
+ var article = Article("");
+ var queue = new AcceptedArticleQueue(article.ArtSize, time);
+ var signal = CreateSignal();
+ var processor = new ControllableCapacityProcessor();
+ var options = new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ };
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ options,
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None);
+ try
+ {
+ Assert.True(queue.TryEnqueue(article));
+ await WaitAsync(() => processor.Calls >= 1 && queue.QueuedCount == 0);
+ Assert.Equal(article.ArtSize, queue.InFlightBytes);
+ Assert.Equal(article.ArtSize, queue.AccountedBytes);
+ Assert.False(queue.TryEnqueue(Article("")));
+
+ processor.AllowComplete = true;
+ signal.Notify();
+ await WaitAsync(() => processor.Completed == 1 && queue.AccountedBytes == 0);
+ }
+ finally
+ {
+ processor.AllowComplete = true;
+ signal.Notify();
+ await worker.StopAsync(CancellationToken.None);
+ }
+ }
+
+ [Fact]
+ public async Task NoteFreedBytes_publishes_capacity_sample_and_wakes_blocked_worker()
+ {
+ var time = new FakeTimeProvider();
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var spool = Path.TrimEndingDirectorySeparator(Path.GetFullPath(Path.GetTempPath()));
+ var catalog = new SegmentPoolCatalog();
+ catalog.Publish([new SegmentPoolDevice("block:freed", [spool])]);
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new PlatformBlockDeviceCounterSource(),
+ NullLogger.Instance,
+ TimeSpan.FromHours(1),
+ TimeProvider.System,
+ freeSpace: null);
+ monitor.SetSpaceObservationForTests(spool, totalBytes: 1_000_000, availableBytes: 100_000);
+ var signal = new SpoolCapacityWaitSignal(monitor);
+ var processor = new ControllableCapacityProcessor();
+ var options = new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ };
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ options,
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None);
+ try
+ {
+ Assert.True(queue.TryEnqueue(Article("")));
+ await WaitAsync(() => processor.Calls >= 1);
+ var callsBefore = processor.Calls;
+ await Task.Delay(40);
+ Assert.Equal(callsBefore, processor.Calls);
+
+ processor.AllowComplete = true;
+ monitor.NoteFreedBytes(spool, 50_000);
+ await WaitAsync(() => processor.Completed == 1);
+ }
+ finally
+ {
+ processor.AllowComplete = true;
+ signal.Notify();
+ await worker.StopAsync(CancellationToken.None);
+ }
+ }
+
+ private static SpoolCapacityWaitSignal CreateSignal()
+ {
+ var catalog = new SegmentPoolCatalog();
+ catalog.Publish([new SegmentPoolDevice("block:test", [Path.GetTempPath()])]);
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new PlatformBlockDeviceCounterSource(),
+ NullLogger.Instance,
+ TimeSpan.FromHours(1),
+ TimeProvider.System,
+ freeSpace: null);
+ return new SpoolCapacityWaitSignal(monitor);
+ }
+
+ private static AcceptedArticle Article(string messageId)
+ {
+ var parser = new NntpArticleParser("cache01.usenet.ninja");
+ var text = "Path: peer.example\r\nDate: Fri, 23 Aug 2024 07:30:10 +0000\r\nMessage-ID: "
+ + messageId
+ + "\r\nNewsgroups: alt.test\r\nFrom: user@example.test\r\nSubject: capacity\r\n\r\nbody\r\n";
+ var created = ArticleRecordFactory.TryCreate(parser, Encoding.ASCII.GetBytes(text));
+ Assert.True(created.IsAccepted);
+ return new AcceptedArticle(
+ created.Record.ArtId,
+ created.Record.ArtHash,
+ created.Record.ArtSize,
+ created.Record.ArtData);
+ }
+
+ private static async Task WaitAsync(Func condition)
+ {
+ var deadline = DateTime.UtcNow + Safety;
+ while (!condition())
+ {
+ if (DateTime.UtcNow > deadline)
+ {
+ throw new TimeoutException("condition was not met");
+ }
+
+ await Task.Delay(10);
+ }
+ }
+
+ private sealed class ControllableCapacityProcessor : IAcceptedArticleProcessor
+ {
+ private int _calls;
+ private int _completed;
+
+ public int Calls => Volatile.Read(ref _calls);
+
+ public int Completed => Volatile.Read(ref _completed);
+
+ public bool AllowComplete { get; set; }
+
+ public void EnsureReady()
+ {
+ }
+
+ public ValueTask ProcessAsync(
+ AcceptedArticle article,
+ CancellationToken cancellationToken)
+ {
+ Interlocked.Increment(ref _calls);
+ if (!AllowComplete)
+ {
+ return ValueTask.FromResult(AcceptedArticleProcessResult.WaitingForCapacity);
+ }
+
+ Interlocked.Increment(ref _completed);
+ return ValueTask.FromResult(AcceptedArticleProcessResult.Completed);
+ }
+ }
+
+ private sealed class RaceNotifyProcessor : IAcceptedArticleProcessor
+ {
+ private readonly TaskCompletionSource _entered;
+ private readonly TaskCompletionSource _proceed;
+ private int _calls;
+ private int _completed;
+
+ public RaceNotifyProcessor(TaskCompletionSource entered, TaskCompletionSource proceed)
+ {
+ _entered = entered;
+ _proceed = proceed;
+ }
+
+ public int Completed => Volatile.Read(ref _completed);
+
+ public void EnsureReady()
+ {
+ }
+
+ public async ValueTask ProcessAsync(
+ AcceptedArticle article,
+ CancellationToken cancellationToken)
+ {
+ var call = Interlocked.Increment(ref _calls);
+ if (call == 1)
+ {
+ _entered.TrySetResult();
+ await _proceed.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
+ return AcceptedArticleProcessResult.WaitingForCapacity;
+ }
+
+ Interlocked.Increment(ref _completed);
+ return AcceptedArticleProcessResult.Completed;
+ }
+ }
+ }
+}
diff --git a/tests/VectorNNTPD.StorageService.Tests/Ingress/StorageEndToEndResilienceBench.cs b/tests/VectorNNTPD.StorageService.Tests/Ingress/StorageEndToEndResilienceBench.cs
new file mode 100644
index 00000000..c1b86eeb
--- /dev/null
+++ b/tests/VectorNNTPD.StorageService.Tests/Ingress/StorageEndToEndResilienceBench.cs
@@ -0,0 +1,5152 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Net;
+using System.Net.Security;
+using System.Net.Sockets;
+using System.Runtime.InteropServices;
+using System.Security.Authentication;
+using System.Text;
+using Microsoft.Extensions.Logging.Abstractions;
+using VectorNNTPD.Common.Acme;
+using VectorNNTPD.Common.Articles;
+using VectorNNTPD.Common.Articles.Parsing;
+using VectorNNTPD.Common.Core;
+using VectorNNTPD.StorageService;
+using VectorNNTPD.StorageService.Configuration;
+using VectorNNTPD.StorageService.Ingress;
+using VectorNNTPD.StorageService.Listener;
+using VectorNNTPD.StorageService.Storage.ArticleIndex;
+using VectorNNTPD.StorageService.Storage.SegmentPools;
+using VectorNNTPD.StorageService.Storage.SpoolPersistence;
+using VectorNNTPD.StorageService.Storage.SpoolShards;
+using VectorNNTPD.StorageService.Tests.Fixtures;
+using VectorNNTPD.StorageService.Tests.Storage;
+using VectorNNTPD.StorageService.Tests.TestDoubles;
+using Xunit.Abstractions;
+
+namespace VectorNNTPD.StorageService.Tests.Ingress
+{
+ ///
+ /// End-to-end performance and resilience measurements for StorageService.
+ /// Runs only when VECTORNNTPD_STORAGE_E2E_BENCH=1.
+ /// Results are observational; they do not assert throughput targets.
+ ///
+ public sealed class StorageEndToEndResilienceBench
+ {
+ private const int ArticleBytes = 750 * 1024;
+ private const string EnvFlag = "VECTORNNTPD_STORAGE_E2E_BENCH";
+ private const string EnvSeconds = "VECTORNNTPD_STORAGE_E2E_SECONDS";
+ private const string EnvRoot = "VECTORNNTPD_STORAGE_BENCH_ROOT";
+ private const string EnvBottleneck = "VECTORNNTPD_STORAGE_BOTTLENECK_BENCH";
+ private const string EnvBottleneckSeconds = "VECTORNNTPD_STORAGE_BOTTLENECK_SECONDS";
+ /// Dedicated client fan-out matrix; does not enable the full bottleneck matrix.
+ private const string EnvFanout = "VECTORNNTPD_STORAGE_CLIENT_FANOUT_BENCH";
+ private const string EnvFanoutSeconds = "VECTORNNTPD_STORAGE_CLIENT_FANOUT_SECONDS";
+ /// Comma-separated client counts, e.g. 1 or 1,2,4. Required when fan-out bench is enabled.
+ private const string EnvFanoutClients = "VECTORNNTPD_STORAGE_CLIENT_FANOUT_CLIENTS";
+ private const string EnvFanoutWorkers = "VECTORNNTPD_STORAGE_CLIENT_FANOUT_WORKERS";
+ private const string EnvFanoutRepeats = "VECTORNNTPD_STORAGE_CLIENT_FANOUT_REPEATS";
+ /// Optional queue byte budget for fan-out only. Default 64 MiB. Allowed 16 MiB–256 MiB.
+ private const string EnvFanoutMaxQueuedBytes = "VECTORNNTPD_STORAGE_CLIENT_FANOUT_MAX_QUEUED_BYTES";
+ private const long DefaultFanoutMaxQueuedBytes = 64L << 20;
+ ///
+ /// Stage isolation only (sink / filesystem / full indexed) at fixed c=8, w=4.
+ /// Does not enable bottleneck, fan-out, or E2E matrices.
+ ///
+ private const string EnvStage = "VECTORNNTPD_STORAGE_STAGE_ISOLATION_BENCH";
+ private const string EnvStageSeconds = "VECTORNNTPD_STORAGE_STAGE_ISOLATION_SECONDS";
+ /// Repeats for the FullIndexed cell only (default 2). Sink and FS run once unless raised.
+ private const string EnvStageIndexedRepeats = "VECTORNNTPD_STORAGE_STAGE_ISOLATION_INDEXED_REPEATS";
+ ///
+ /// Filesystem publication substage attribution at fixed c=8, w=4, FilesystemNoIndex.
+ /// Does not enable stage-isolation, bottleneck, fan-out, or E2E matrices.
+ ///
+ private const string EnvFsPub = "VECTORNNTPD_STORAGE_FS_PUB_ATTRIBUTION_BENCH";
+ private const string EnvFsPubSeconds = "VECTORNNTPD_STORAGE_FS_PUB_ATTRIBUTION_SECONDS";
+ private const string EnvFsPubRepeats = "VECTORNNTPD_STORAGE_FS_PUB_ATTRIBUTION_REPEATS";
+ ///
+ /// Sink vs FilesystemNoIndex at c=8/w=4 with test-only publish/process concurrency samples.
+ /// Does not enable stage-isolation, FS-pub, bottleneck, fan-out, or layout matrices.
+ ///
+ private const string EnvE2eFsBottleneck = "VECTORNNTPD_STORAGE_E2E_FS_BOTTLENECK_BENCH";
+ private const string EnvE2eFsBottleneckSeconds = "VECTORNNTPD_STORAGE_E2E_FS_BOTTLENECK_SECONDS";
+ ///
+ /// CompletingSink vs CompletingSink+calibrated busy delay vs FilesystemNoIndex at c=8/w=4.
+ /// Does not enable stage-isolation, FS-pub, bottleneck, fan-out, or layout matrices.
+ ///
+ private const string EnvCalibratedDelay = "VECTORNNTPD_STORAGE_CALIBRATED_DELAY_BENCH";
+ private const string EnvCalibratedDelaySeconds = "VECTORNNTPD_STORAGE_CALIBRATED_DELAY_SECONDS";
+ private const string EnvCalibratedDelayMs = "VECTORNNTPD_STORAGE_CALIBRATED_DELAY_MS";
+ private const double DefaultCalibratedDelayMs = 1.8;
+ ///
+ /// CompletingSink vs FilesystemNoIndex with bounded per-article FS-stage correlation for tail attribution.
+ ///
+ private const string EnvFsTail = "VECTORNNTPD_STORAGE_FS_TAIL_BENCH";
+ private const string EnvFsTailSeconds = "VECTORNNTPD_STORAGE_FS_TAIL_SECONDS";
+ private const string EnvFsTailRepeats = "VECTORNNTPD_STORAGE_FS_TAIL_REPEATS";
+ ///
+ /// FilesystemNoIndex w=4 then w=1 with correlated stage samples; absolute slow thresholds
+ /// from the w=4 cell are applied to both. Does not enable .
+ ///
+ private const string EnvFsTailWorkerIsolation = "VECTORNNTPD_STORAGE_FS_TAIL_WORKER_ISOLATION_BENCH";
+ private const string EnvFsTailWorkerIsolationSeconds = "VECTORNNTPD_STORAGE_FS_TAIL_WORKER_ISOLATION_SECONDS";
+ private const string EnvFsTailWorkerIsolationRepeats = "VECTORNNTPD_STORAGE_FS_TAIL_WORKER_ISOLATION_REPEATS";
+ ///
+ /// FilesystemNoIndex c=8/w=4 with client-side aggregate admission pacing to vary queue headroom.
+ /// Does not enable or worker-isolation benches.
+ ///
+ private const string EnvAdmissionHeadroom = "VECTORNNTPD_STORAGE_FS_TAIL_ADMISSION_HEADROOM_BENCH";
+ private const string EnvAdmissionHeadroomSeconds = "VECTORNNTPD_STORAGE_FS_TAIL_ADMISSION_HEADROOM_SECONDS";
+ private const string EnvAdmissionHeadroomRepeats = "VECTORNNTPD_STORAGE_FS_TAIL_ADMISSION_HEADROOM_REPEATS";
+ ///
+ /// FilesystemNoIndex c=8/w=4 baseline vs lightweight host telemetry (process/GC/thread-pool).
+ /// Does not enable or other FS-tail variants.
+ ///
+ private const string EnvHostTelemetry = "VECTORNNTPD_STORAGE_FS_TAIL_HOST_TELEMETRY_BENCH";
+ private const string EnvHostTelemetrySeconds = "VECTORNNTPD_STORAGE_FS_TAIL_HOST_TELEMETRY_SECONDS";
+ private const string EnvHostTelemetryRepeats = "VECTORNNTPD_STORAGE_FS_TAIL_HOST_TELEMETRY_REPEATS";
+ private const string EnvHostTelemetryIntervalMs = "VECTORNNTPD_STORAGE_FS_TAIL_HOST_TELEMETRY_INTERVAL_MS";
+ ///
+ /// FilesystemNoIndex c=8/w=4 with optional same-volume low-rate CreateNew sidecar
+ /// (ETW DiskIO/FileIO fallback when elevation is unavailable).
+ ///
+ private const string EnvFsTailSidecar = "VECTORNNTPD_STORAGE_FS_TAIL_SIDECAR_BENCH";
+ private const string EnvFsTailSidecarSeconds = "VECTORNNTPD_STORAGE_FS_TAIL_SIDECAR_SECONDS";
+ private const string EnvFsTailSidecarRepeats = "VECTORNNTPD_STORAGE_FS_TAIL_SIDECAR_REPEATS";
+ private const string EnvFsTailSidecarRate = "VECTORNNTPD_STORAGE_FS_TAIL_SIDECAR_OPS_PER_SEC";
+
+ private readonly ITestOutputHelper _output;
+
+ public StorageEndToEndResilienceBench(ITestOutputHelper output) => _output = output;
+
+ [Fact]
+ public async Task Optional_e2e_750kib_tls_persist_index_sustained()
+ {
+ if (!Enabled())
+ {
+ Skip();
+ return;
+ }
+
+ var duration = ResolveDuration(defaultSeconds: 60);
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+
+ await RunE2EAsync(root, duration, workers: 4, outstanding: 2, label: "e2e-w4-o2");
+ var shortDuration = TimeSpan.FromSeconds(Math.Min(15, Math.Max(5, duration.TotalSeconds / 4)));
+ await RunE2EAsync(root, shortDuration, workers: 1, outstanding: 1, label: "e2e-w1-o1");
+ }
+
+
+ [Fact]
+ public async Task Optional_timed_capacity_recovery()
+ {
+ if (!Enabled())
+ {
+ Skip();
+ return;
+ }
+
+ var notifySamples = new List(32);
+ var resumeSamples = new List(32);
+ var completeSamples = new List(32);
+ var waitSamples = new List(32);
+
+ for (var i = 0; i < 24; i++)
+ {
+ var sample = await MeasureNotifyWakeAsync();
+ notifySamples.Add(sample.NotifyToResumeMs);
+ resumeSamples.Add(sample.ResumeToCompleteMs);
+ completeSamples.Add(sample.NotifyToCompleteMs);
+ waitSamples.Add(sample.CapacityWaitMs);
+ }
+
+ notifySamples.Sort();
+ resumeSamples.Sort();
+ completeSamples.Sort();
+ waitSamples.Sort();
+
+ var line =
+ "capacity-recovery-notify samples=24 "
+ + $"notifyToResumeMs min={notifySamples[0]:0.000} med={Median(notifySamples):0.000} p95={Percentile(notifySamples, 0.95):0.000} max={notifySamples[^1]:0.000} "
+ + $"resumeToCompleteMs min={resumeSamples[0]:0.000} med={Median(resumeSamples):0.000} p95={Percentile(resumeSamples, 0.95):0.000} max={resumeSamples[^1]:0.000} "
+ + $"notifyToCompleteMs min={completeSamples[0]:0.000} med={Median(completeSamples):0.000} p95={Percentile(completeSamples, 0.95):0.000} max={completeSamples[^1]:0.000} "
+ + $"capacityWaitMs min={waitSamples[0]:0.000} med={Median(waitSamples):0.000} p95={Percentile(waitSamples, 0.95):0.000} max={waitSamples[^1]:0.000} "
+ + "wakePath=SpoolCapacityWaitSignal.Notify pollIntervalMs=60000 (not used when notify wins)";
+ Emit(line);
+
+ var sampleWake = await MeasureSamplePublishedWakeAsync();
+ Emit(
+ "capacity-recovery-sample "
+ + $"eligibleObservedToNotifyMs={sampleWake.EligibleToNotifyMs:0.000} "
+ + $"notifyToResumeMs={sampleWake.NotifyToResumeMs:0.000} "
+ + $"notifyToCompleteMs={sampleWake.NotifyToCompleteMs:0.000} "
+ + "wakePath=BlockDeviceMonitorService.CapacitySamplePublished→SpoolCapacityWaitSignal");
+
+ var eviction = await MeasureEvictionFreedWakeAsync();
+ Emit(
+ "capacity-recovery-eviction-signal "
+ + $"noteFreedToPublishedMs={eviction.FreedToNotifyMs:0.000} "
+ + $"publishedToResumeMs={eviction.NotifyToResumeMs:0.000} "
+ + $"publishedToCompleteMs={eviction.NotifyToCompleteMs:0.000} "
+ + "wakePath=BlockDeviceMonitorService.NoteFreedBytes→CapacitySamplePublished→SpoolCapacityWaitSignal");
+
+ var critical = await MeasureCriticalSpoolExcludedAsync();
+ Emit(
+ "capacity-critical-excluded "
+ + $"processResult={critical.Result} deferred={critical.Deferred} "
+ + $"elapsedMs={critical.ElapsedMs:0.000} "
+ + "contract=spool_at_or_above_CriticalPercent_not_selected");
+
+ var multi = await MeasureMultiSpoolIndependentRecoveryAsync();
+ Emit(
+ "capacity-multi-spool "
+ + $"blockedSpoolDeferred={multi.BlockedDeferred} healthyStored={multi.HealthyStored} "
+ + $"recoveryNotifyToCompleteMs={multi.NotifyToCompleteMs:0.000} "
+ + "contract=one_spool_recovery_does_not_require_unrelated_spool");
+
+ Assert.True(Median(notifySamples) >= 0);
+ Assert.True(sampleWake.NotifyToCompleteMs >= 0);
+ Assert.Equal(AcceptedArticleProcessResult.WaitingForCapacity, critical.Result);
+ Assert.True(multi.HealthyStored >= 1);
+ }
+
+ [Fact]
+ public async Task Optional_shutdown_under_backlog_wall_clock()
+ {
+ if (!Enabled())
+ {
+ Skip();
+ return;
+ }
+
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, TimeSpan.Zero);
+
+ await MeasureShutdownDrainAsync(root, articleCount: 32, label: "shutdown-normal");
+ await MeasureShutdownDrainAsync(root, articleCount: 128, label: "shutdown-backlog");
+ await MeasureShutdownCancelFailsAsync(label: "shutdown-host-cancel");
+ await MeasureDisconnectAfterAdmitAsync(root, label: "shutdown-disconnect-after-admit");
+ }
+
+ ///
+ /// Worker scaling, client scaling, and stage-isolation attribution.
+ /// Runs only when VECTORNNTPD_STORAGE_BOTTLENECK_BENCH=1.
+ /// Does not run when only VECTORNNTPD_STORAGE_CLIENT_FANOUT_BENCH is set.
+ ///
+ [Fact]
+ public async Task Optional_bottleneck_attribution_scaling_and_stages()
+ {
+ if (!string.Equals(Environment.GetEnvironmentVariable(EnvBottleneck), "1", StringComparison.Ordinal))
+ {
+ _output.WriteLine($"Set {EnvBottleneck}=1 to run bottleneck attribution benches.");
+ return;
+ }
+
+ var duration = ResolveBottleneckDuration(defaultSeconds: 30);
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"bottleneck-storage deviceHint=NVMe_or_HDD_see_report filesystem=NTFS dedicatedDisk={root.IsDedicated} "
+ + "note=C_drive_physical_disk_recorded_in_BOTTLENECK_ATTRIBUTION_REPORT");
+
+ // Stage isolation (fixed w=4, c=2): completing sink vs FS-only vs full indexed.
+ await RunE2EAsync(root, duration, workers: 4, outstanding: 2, label: "stage-completing-sink", HostMode.CompletingSink);
+ await RunE2EAsync(root, duration, workers: 4, outstanding: 2, label: "stage-fs-no-index", HostMode.FilesystemNoIndex);
+ await RunE2EAsync(root, duration, workers: 4, outstanding: 2, label: "stage-full-indexed", HostMode.FullIndexed);
+
+ // Worker scaling: hold client connections=2.
+ foreach (var workers in new[] { 1, 2, 4, 8 })
+ {
+ await RunE2EAsync(root, duration, workers, outstanding: 2, label: $"scale-w{workers}-c2-r1", HostMode.FullIndexed);
+ await RunE2EAsync(root, duration, workers, outstanding: 2, label: $"scale-w{workers}-c2-r2", HostMode.FullIndexed);
+ }
+
+ // Client scaling: hold workers=4.
+ foreach (var clients in new[] { 1, 2, 4 })
+ {
+ await RunE2EAsync(root, duration, workers: 4, outstanding: clients, label: $"scale-w4-c{clients}-r1", HostMode.FullIndexed);
+ }
+ }
+
+ ///
+ /// Client fan-out only (workers fixed). Activate with
+ /// VECTORNNTPD_STORAGE_CLIENT_FANOUT_BENCH=1 and
+ /// VECTORNNTPD_STORAGE_CLIENT_FANOUT_CLIENTS=1 (or 1,2,4).
+ /// Does not enable .
+ ///
+ [Fact]
+ public async Task Optional_client_fanout_scaling()
+ {
+ if (!string.Equals(Environment.GetEnvironmentVariable(EnvFanout), "1", StringComparison.Ordinal))
+ {
+ _output.WriteLine(
+ $"Set {EnvFanout}=1 and {EnvFanoutClients}=N (comma-separated) to run client fan-out benches.");
+ return;
+ }
+
+ var clientsList = ParsePositiveIntList(Environment.GetEnvironmentVariable(EnvFanoutClients));
+ if (clientsList.Count == 0)
+ {
+ _output.WriteLine(
+ $"{EnvFanout}=1 but {EnvFanoutClients} is missing/empty; skipping (refuse unbounded matrix).");
+ return;
+ }
+
+ var duration = ResolveFanoutDuration(defaultSeconds: 60);
+ var workers = 4;
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvFanoutWorkers), out var configuredWorkers)
+ && configuredWorkers is >= 1 and <= 64)
+ {
+ workers = configuredWorkers;
+ }
+
+ var repeats = 1;
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvFanoutRepeats), out var configuredRepeats)
+ && configuredRepeats is >= 1 and <= 5)
+ {
+ repeats = configuredRepeats;
+ }
+
+ var maxQueuedBytes = DefaultFanoutMaxQueuedBytes;
+ if (long.TryParse(Environment.GetEnvironmentVariable(EnvFanoutMaxQueuedBytes), out var configuredQueued)
+ && configuredQueued is >= (16L << 20) and <= (256L << 20))
+ {
+ maxQueuedBytes = configuredQueued;
+ }
+
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"fanout-config workers={workers} clients=[{string.Join(',', clientsList)}] repeats={repeats} "
+ + $"maxQueuedBytes={maxQueuedBytes} durationSec={duration.TotalSeconds:0} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvFanout}=1 {EnvFanoutClients}={Environment.GetEnvironmentVariable(EnvFanoutClients)}");
+
+ foreach (var clients in clientsList)
+ {
+ if (clients is < 1 or > 64)
+ {
+ Emit($"fanout-skip clients={clients} reason=out_of_range");
+ continue;
+ }
+
+ for (var r = 1; r <= repeats; r++)
+ {
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fanout-w{workers}-c{clients}-r{r}",
+ HostMode.FullIndexed,
+ maxQueuedBytes);
+ }
+ }
+ }
+
+ ///
+ /// Stage isolation at saturated client offer: CompletingSink vs FilesystemNoIndex vs FullIndexed.
+ /// Activate with VECTORNNTPD_STORAGE_STAGE_ISOLATION_BENCH=1.
+ /// Fixed workers=4, clients=8, MaxQueuedBytes=64 MiB; does not enable other optional benches.
+ ///
+ [Fact]
+ public async Task Optional_stage_isolation_c8_w4()
+ {
+ if (!string.Equals(Environment.GetEnvironmentVariable(EnvStage), "1", StringComparison.Ordinal))
+ {
+ _output.WriteLine($"Set {EnvStage}=1 to run stage-isolation benches (c=8, w=4).");
+ return;
+ }
+
+ var duration = ResolveStageDuration(defaultSeconds: 60);
+ var indexedRepeats = 2;
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvStageIndexedRepeats), out var configuredRepeats)
+ && configuredRepeats is >= 1 and <= 5)
+ {
+ indexedRepeats = configuredRepeats;
+ }
+
+ const int workers = 4;
+ const int clients = 8;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"stage-config workers={workers} clients={clients} maxQueuedBytes={DefaultFanoutMaxQueuedBytes} "
+ + $"indexedRepeats={indexedRepeats} durationSec={duration.TotalSeconds:0} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvStage}=1 "
+ + "modes=CompletingSink,FilesystemNoIndex,FullIndexed "
+ + "note=same_TLS_TAKETHIS_ingress_and_queue_for_all_modes");
+
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: "stage-c8-w4-completing-sink-r1",
+ HostMode.CompletingSink,
+ DefaultFanoutMaxQueuedBytes);
+
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: "stage-c8-w4-fs-no-index-r1",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes);
+
+ for (var r = 1; r <= indexedRepeats; r++)
+ {
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"stage-c8-w4-full-indexed-r{r}",
+ HostMode.FullIndexed,
+ DefaultFanoutMaxQueuedBytes);
+ }
+ }
+
+ ///
+ /// FilesystemNoIndex publication substage timing at saturated offer (c=8, w=4).
+ /// Activate with VECTORNNTPD_STORAGE_FS_PUB_ATTRIBUTION_BENCH=1.
+ /// Also runs a labeled Path-rewrite CPU microbench (no disk) for residual context.
+ ///
+ [Fact]
+ public async Task Optional_filesystem_publication_attribution()
+ {
+ if (!string.Equals(Environment.GetEnvironmentVariable(EnvFsPub), "1", StringComparison.Ordinal))
+ {
+ _output.WriteLine($"Set {EnvFsPub}=1 to run filesystem publication attribution (c=8, w=4, FilesystemNoIndex).");
+ return;
+ }
+
+ var duration = ResolveFsPubDuration(defaultSeconds: 60);
+ var repeats = 2;
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvFsPubRepeats), out var configuredRepeats)
+ && configuredRepeats is >= 1 and <= 5)
+ {
+ repeats = configuredRepeats;
+ }
+
+ const int workers = 4;
+ const int clients = 8;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"fspub-config workers={workers} clients={clients} maxQueuedBytes={DefaultFanoutMaxQueuedBytes} "
+ + $"repeats={repeats} durationSec={duration.TotalSeconds:0} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvFsPub}=1 mode=FilesystemNoIndex "
+ + "instrumentation=test_only_TimingSpoolFiles_open_write_flush_move_exists_dir "
+ + "note=same_TLS_TAKETHIS_ingress_as_stage_isolation");
+
+ MeasurePathRewriteCpuMicrobench(iterations: 2000);
+
+ for (var r = 1; r <= repeats; r++)
+ {
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fspub-c8-w4-fs-no-index-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes);
+ }
+ }
+
+ ///
+ /// Discriminating E2E cell: CompletingSink then FilesystemNoIndex at saturated c=8/w=4,
+ /// reporting max concurrent ProcessAsync / WriteAll / MoveNew (test-only).
+ /// Hypothesis: FS publish I/O overlaps across workers (not a single global serialize lock);
+ /// sink→FS gap is worker busy-time + admit coupling. Activate with
+ /// VECTORNNTPD_STORAGE_E2E_FS_BOTTLENECK_BENCH=1.
+ ///
+ [Fact]
+ public async Task Optional_e2e_fs_bottleneck_concurrency()
+ {
+ if (!string.Equals(Environment.GetEnvironmentVariable(EnvE2eFsBottleneck), "1", StringComparison.Ordinal))
+ {
+ _output.WriteLine(
+ $"Set {EnvE2eFsBottleneck}=1 to run sink vs FilesystemNoIndex concurrency attribution "
+ + "(c=8, w=4; not an optimization claim).");
+ return;
+ }
+
+ var duration = ResolveE2eFsBottleneckDuration(defaultSeconds: 60);
+ const int workers = 4;
+ const int clients = 8;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"e2efs-config workers={workers} clients={clients} maxQueuedBytes={DefaultFanoutMaxQueuedBytes} "
+ + $"durationSec={duration.TotalSeconds:0} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvE2eFsBottleneck}=1 "
+ + "modes=CompletingSink,FilesystemNoIndex "
+ + "instrumentation=test_only_max_concurrent_Process_WriteAll_Move "
+ + "hypothesis=fs_publish_overlaps_across_workers_gap_is_busy_time_plus_admit_coupling "
+ + "competing=global_serialize_lock_on_publish_io "
+ + "note=same_TLS_TAKETHIS_ingress_as_stage_isolation_no_FullIndexed_cell");
+
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: "e2efs-c8-w4-completing-sink",
+ HostMode.CompletingSink,
+ DefaultFanoutMaxQueuedBytes);
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: "e2efs-c8-w4-fs-no-index",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes);
+ }
+
+ ///
+ /// CompletingSink vs CompletingSink with calibrated worker busy-wait (~1.8 ms, no FS I/O)
+ /// vs FilesystemNoIndex at c=8/w=4. Tests whether generic process busy-time + admit/queue
+ /// coupling can reproduce the sink→FS throughput gap. Activate with
+ /// VECTORNNTPD_STORAGE_CALIBRATED_DELAY_BENCH=1.
+ /// Delay holds the worker synchronously (Stopwatch spin) because production publish Write/Move
+ /// are synchronous on the worker; would release the worker and
+ /// mis-model concurrency. Not equivalent to disk I/O.
+ ///
+ [Fact]
+ public async Task Optional_calibrated_delay_vs_filesystem()
+ {
+ if (!string.Equals(Environment.GetEnvironmentVariable(EnvCalibratedDelay), "1", StringComparison.Ordinal))
+ {
+ _output.WriteLine(
+ $"Set {EnvCalibratedDelay}=1 to run CompletingSink vs calibrated-delay sink vs FilesystemNoIndex "
+ + $"(c=8, w=4; default delayMs={DefaultCalibratedDelayMs}).");
+ return;
+ }
+
+ var duration = ResolveCalibratedDelayDuration(defaultSeconds: 60);
+ var delayMs = DefaultCalibratedDelayMs;
+ if (double.TryParse(
+ Environment.GetEnvironmentVariable(EnvCalibratedDelayMs),
+ System.Globalization.NumberStyles.Float,
+ System.Globalization.CultureInfo.InvariantCulture,
+ out var configuredDelay)
+ && configuredDelay is > 0 and <= 50)
+ {
+ delayMs = configuredDelay;
+ }
+
+ const int workers = 4;
+ const int clients = 8;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"caldelay-config workers={workers} clients={clients} maxQueuedBytes={DefaultFanoutMaxQueuedBytes} "
+ + $"durationSec={duration.TotalSeconds:0} delayMs={delayMs:0.###} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvCalibratedDelay}=1 "
+ + "modes=CompletingSink,CompletingSinkCalibratedDelay,FilesystemNoIndex "
+ + "delayMechanism=Stopwatch_busy_wait_sync_on_worker_no_TaskDelay_no_FS_IO "
+ + "delayPlacement=CompletingSinkProcessor_ProcessAsync_before_Completed "
+ + "hypothesis=generic_busy_time_plus_admit_queue_coupling_reproduces_FS_throughput_band "
+ + "note=delay_NOT_equivalent_to_disk_IO_even_if_rates_match");
+
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: "caldelay-c8-w4-completing-sink",
+ HostMode.CompletingSink,
+ DefaultFanoutMaxQueuedBytes);
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: "caldelay-c8-w4-sink-delay",
+ HostMode.CompletingSinkCalibratedDelay,
+ DefaultFanoutMaxQueuedBytes,
+ sinkDelay: TimeSpan.FromMilliseconds(delayMs));
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: "caldelay-c8-w4-fs-no-index",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes);
+ }
+
+ ///
+ /// CompletingSink vs FilesystemNoIndex at c=8/w=4 with bounded correlated open/write/move/process
+ /// samples and before/after AccountedBytes snapshots for queue-pressure event ordering.
+ /// Activate with VECTORNNTPD_STORAGE_FS_TAIL_BENCH=1.
+ ///
+ [Fact]
+ public async Task Optional_filesystem_tail_attribution()
+ {
+ if (!string.Equals(Environment.GetEnvironmentVariable(EnvFsTail), "1", StringComparison.Ordinal))
+ {
+ _output.WriteLine(
+ $"Set {EnvFsTail}=1 to run CompletingSink vs FilesystemNoIndex tail attribution "
+ + "(c=8, w=4; bounded correlated stage samples).");
+ return;
+ }
+
+ var duration = ResolveFsTailDuration(defaultSeconds: 60);
+ var repeats = 1;
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvFsTailRepeats), out var configuredRepeats)
+ && configuredRepeats is >= 1 and <= 3)
+ {
+ repeats = configuredRepeats;
+ }
+
+ const int workers = 4;
+ const int clients = 8;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"fstail-config workers={workers} clients={clients} maxQueuedBytes={DefaultFanoutMaxQueuedBytes} "
+ + $"durationSec={duration.TotalSeconds:0} repeats={repeats} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvFsTail}=1 "
+ + "modes=CompletingSink,FilesystemNoIndex "
+ + "instrumentation=test_only_AsyncLocal_correlated_stages_plus_AccountedBytes_before_after_open_write_move "
+ + "sampleCap=8192 "
+ + "hypothesis=queue_pressure_precedes_vs_follows_slow_FS_ops "
+ + "note=AccountedBytes_is_payload_accounting_not_disk_queue_depth");
+
+ for (var r = 1; r <= repeats; r++)
+ {
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-c8-w4-completing-sink-r{r}",
+ HostMode.CompletingSink,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: false);
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-c8-w4-fs-no-index-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true);
+ }
+ }
+
+ ///
+ /// FilesystemNoIndex c=8 with w=4 then w=1 to isolate multi-worker Acc feedback from
+ /// persistent Open/Move spikes. Absolute slow thresholds from w=4 are applied to both cells.
+ /// Activate with VECTORNNTPD_STORAGE_FS_TAIL_WORKER_ISOLATION_BENCH=1.
+ ///
+ [Fact]
+ public async Task Optional_filesystem_tail_worker_isolation()
+ {
+ if (!string.Equals(
+ Environment.GetEnvironmentVariable(EnvFsTailWorkerIsolation),
+ "1",
+ StringComparison.Ordinal))
+ {
+ _output.WriteLine(
+ $"Set {EnvFsTailWorkerIsolation}=1 to run FilesystemNoIndex w=4 vs w=1 "
+ + "FS-tail isolation (c=8; absolute slow thresholds from w=4).");
+ return;
+ }
+
+ var duration = ResolveFsTailWorkerIsolationDuration(defaultSeconds: 60);
+ var repeats = 1;
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvFsTailWorkerIsolationRepeats),
+ out var configuredRepeats)
+ && configuredRepeats is >= 1 and <= 3)
+ {
+ repeats = configuredRepeats;
+ }
+
+ const int clients = 8;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"fstail-worker-isolation-config clients={clients} maxQueuedBytes={DefaultFanoutMaxQueuedBytes} "
+ + $"durationSec={duration.TotalSeconds:0} repeats={repeats} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvFsTailWorkerIsolation}=1 "
+ + "mode=FilesystemNoIndex workerCells=4,1 "
+ + "instrumentation=test_only_AsyncLocal_correlated_stages_plus_AccountedBytes_before_after "
+ + "sampleCap=8192 "
+ + "hypothesis=Open_Move_spikes_persist_at_w1_vs_multi_worker_Acc_feedback "
+ + "note=only_worker_count_changes_clients_article_queue_budget_constant");
+
+ for (var r = 1; r <= repeats; r++)
+ {
+ AbsoluteSlowThresholds? baseline = null;
+ await RunE2EAsync(
+ root,
+ duration,
+ workers: 4,
+ outstanding: clients,
+ label: $"fstail-isol-c8-w4-fs-no-index-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ // Self-baseline: absolute analysis uses this cell's own p95 (identity check).
+ absoluteSlowThresholds: null,
+ onLocalSlowThresholds: t => baseline = t,
+ applyLocalThresholdsAsAbsolute: true);
+
+ Assert.True(baseline.HasValue, "expected w=4 correlator percentile thresholds");
+ Emit(
+ $"fstail-isol-absolute-baseline-r{r} "
+ + $"processP95Ms={baseline.Value.ProcessP95Ms:0.000} "
+ + $"openP95Ms={baseline.Value.OpenP95Ms:0.000} "
+ + $"writeP95Ms={baseline.Value.WriteP95Ms:0.000} "
+ + $"moveP95Ms={baseline.Value.MoveP95Ms:0.000} "
+ + "appliedTo=w4_self_and_w1");
+
+ await RunE2EAsync(
+ root,
+ duration,
+ workers: 1,
+ outstanding: clients,
+ label: $"fstail-isol-c8-w1-fs-no-index-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ absoluteSlowThresholds: baseline,
+ onLocalSlowThresholds: null);
+ }
+ }
+
+ ///
+ /// FilesystemNoIndex c=8/w=4 with optional client-side aggregate admission pacing to keep
+ /// AccountedBytes below the queue cap (queue-headroom isolation). Activate with
+ /// VECTORNNTPD_STORAGE_FS_TAIL_ADMISSION_HEADROOM_BENCH=1.
+ ///
+ [Fact]
+ public async Task Optional_filesystem_tail_admission_headroom()
+ {
+ if (!string.Equals(
+ Environment.GetEnvironmentVariable(EnvAdmissionHeadroom),
+ "1",
+ StringComparison.Ordinal))
+ {
+ _output.WriteLine(
+ $"Set {EnvAdmissionHeadroom}=1 to run FilesystemNoIndex admission-rate / queue-headroom "
+ + "isolation (c=8, w=4; client-side aggregate pacing).");
+ return;
+ }
+
+ var duration = ResolveAdmissionHeadroomDuration(defaultSeconds: 60);
+ var repeats = 2;
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvAdmissionHeadroomRepeats),
+ out var configuredRepeats)
+ && configuredRepeats is >= 1 and <= 3)
+ {
+ repeats = configuredRepeats;
+ }
+
+ const int workers = 4;
+ const int clients = 8;
+ // Tentative aggregate admit rates (art/s). Fixed for measured cells after short calibration.
+ // Chosen below prior unrestricted ~930–1008 art/s so the queue can drain between admits.
+ const double lowAdmitPerSec = 400;
+ const double midAdmitPerSec = 650;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"fstail-admission-headroom-config workers={workers} clients={clients} "
+ + $"maxQueuedBytes={DefaultFanoutMaxQueuedBytes} durationSec={duration.TotalSeconds:0} "
+ + $"repeats={repeats} dedicatedDisk={root.IsDedicated} activation={EnvAdmissionHeadroom}=1 "
+ + "mode=FilesystemNoIndex "
+ + $"cells=unrestricted,mid_{midAdmitPerSec:0}/s,low_{lowAdmitPerSec:0}/s "
+ + "pacing=test_only_shared_token_bucket_before_TakeThis "
+ + "instrumentation=AsyncLocal_correlated_stages_plus_full_bag_open_stall_counts "
+ + "sampleCap=8192 "
+ + "hypothesis=Open_stalls_persist_when_AccountedBytes_stays_below_cap");
+
+ // Short calibration (not used as primary evidence): confirm occupancy separation.
+ var calibDuration = TimeSpan.FromSeconds(15);
+ Emit($"fstail-admission-calib-start durationSec={calibDuration.TotalSeconds:0}");
+ await RunE2EAsync(
+ root,
+ calibDuration,
+ workers,
+ outstanding: clients,
+ label: "fstail-adm-calib-unrestricted",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ aggregateAdmitPerSec: null);
+ await RunE2EAsync(
+ root,
+ calibDuration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-adm-calib-mid-{midAdmitPerSec:0}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ aggregateAdmitPerSec: midAdmitPerSec);
+ await RunE2EAsync(
+ root,
+ calibDuration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-adm-calib-low-{lowAdmitPerSec:0}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ aggregateAdmitPerSec: lowAdmitPerSec);
+ Emit(
+ $"fstail-admission-calib-done fixedRatesForMeasured=unrestricted,{midAdmitPerSec:0},{lowAdmitPerSec:0} "
+ + "note=rates_not_adjusted_during_measured_cells");
+
+ for (var r = 1; r <= repeats; r++)
+ {
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-adm-unrestricted-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ aggregateAdmitPerSec: null);
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-adm-mid-{midAdmitPerSec:0}-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ aggregateAdmitPerSec: midAdmitPerSec);
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-adm-low-{lowAdmitPerSec:0}-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ aggregateAdmitPerSec: lowAdmitPerSec);
+ }
+ }
+
+ ///
+ /// FilesystemNoIndex c=8/w=4: baseline FS-tail then lightweight host telemetry, to correlate
+ /// ≥50 ms Open stalls with process/GC/thread-pool samples. Activate with
+ /// VECTORNNTPD_STORAGE_FS_TAIL_HOST_TELEMETRY_BENCH=1.
+ ///
+ [Fact]
+ public async Task Optional_filesystem_tail_host_telemetry()
+ {
+ if (!string.Equals(
+ Environment.GetEnvironmentVariable(EnvHostTelemetry),
+ "1",
+ StringComparison.Ordinal))
+ {
+ _output.WriteLine(
+ $"Set {EnvHostTelemetry}=1 to run FilesystemNoIndex host-telemetry isolation "
+ + "(c=8, w=4; baseline then lightweight process/GC/thread-pool sampling).");
+ return;
+ }
+
+ var duration = ResolveHostTelemetryDuration(defaultSeconds: 60);
+ var repeats = 2;
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvHostTelemetryRepeats),
+ out var configuredRepeats)
+ && configuredRepeats is >= 1 and <= 3)
+ {
+ repeats = configuredRepeats;
+ }
+
+ var intervalMs = 100;
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvHostTelemetryIntervalMs),
+ out var configuredInterval)
+ && configuredInterval is >= 50 and <= 1000)
+ {
+ intervalMs = configuredInterval;
+ }
+
+ const int workers = 4;
+ const int clients = 8;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"fstail-host-telemetry-config workers={workers} clients={clients} "
+ + $"maxQueuedBytes={DefaultFanoutMaxQueuedBytes} durationSec={duration.TotalSeconds:0} "
+ + $"repeats={repeats} sampleIntervalMs={intervalMs} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvHostTelemetry}=1 mode=FilesystemNoIndex "
+ + "cells=baseline_fs_tail,lightweight_host_telemetry "
+ + "diskCounters=not_instrumented_no_PerformanceCounter_package "
+ + "gcPause=GC.GetTotalPauseDuration_delta_per_sample "
+ + "hypothesis=correlate_Open_ge50ms_with_process_GC_threadpool_samples "
+ + "note=telemetry_exceptions_recorded_do_not_alter_article_processing");
+
+ for (var r = 1; r <= repeats; r++)
+ {
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-hosttel-baseline-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ enableHostTelemetry: false,
+ hostTelemetryIntervalMs: intervalMs);
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-hosttel-telem-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ enableHostTelemetry: true,
+ hostTelemetryIntervalMs: intervalMs);
+ }
+ }
+
+ ///
+ /// FilesystemNoIndex c=8/w=4 baseline vs same-volume low-rate CreateNew sidecar to test
+ /// whether unrelated creates stall in the same windows as E2E Open bursts. Activate with
+ /// VECTORNNTPD_STORAGE_FS_TAIL_SIDECAR_BENCH=1. Used when elevated ETW is unavailable.
+ ///
+ [Fact]
+ public async Task Optional_filesystem_tail_same_volume_sidecar()
+ {
+ if (!string.Equals(
+ Environment.GetEnvironmentVariable(EnvFsTailSidecar),
+ "1",
+ StringComparison.Ordinal))
+ {
+ _output.WriteLine(
+ $"Set {EnvFsTailSidecar}=1 to run FilesystemNoIndex same-volume CreateNew sidecar "
+ + "(c=8, w=4; E2E-only then E2E+sidecar).");
+ return;
+ }
+
+ var duration = ResolveFsTailSidecarDuration(defaultSeconds: 60);
+ var repeats = 2;
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvFsTailSidecarRepeats),
+ out var configuredRepeats)
+ && configuredRepeats is >= 1 and <= 3)
+ {
+ repeats = configuredRepeats;
+ }
+
+ var sidecarRate = 20d;
+ if (double.TryParse(
+ Environment.GetEnvironmentVariable(EnvFsTailSidecarRate),
+ out var configuredRate)
+ && configuredRate is >= 1 and <= 200)
+ {
+ sidecarRate = configuredRate;
+ }
+
+ const int workers = 4;
+ const int clients = 8;
+ var root = ResolveBenchRoot();
+ WriteEnvPreamble(root, duration);
+ Emit(
+ $"fstail-sidecar-config workers={workers} clients={clients} "
+ + $"maxQueuedBytes={DefaultFanoutMaxQueuedBytes} durationSec={duration.TotalSeconds:0} "
+ + $"repeats={repeats} sidecarOpsPerSec={sidecarRate:0.###} dedicatedDisk={root.IsDedicated} "
+ + $"activation={EnvFsTailSidecar}=1 mode=FilesystemNoIndex "
+ + "cells=e2e_only,e2e_plus_same_volume_CreateNew_sidecar "
+ + "etwStatus=WPR_DiskIO_FileIO_AccessDenied_0x80070005_not_elevated "
+ + "hypothesis=unrelated_CreateNew_on_same_volume_stalls_with_E2E_Open_bursts "
+ + "note=sidecar_cannot_alone_establish_root_cause");
+
+ for (var r = 1; r <= repeats; r++)
+ {
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-sidecar-e2eonly-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ enableSameVolumeSidecar: false,
+ sidecarOpsPerSec: sidecarRate);
+ await RunE2EAsync(
+ root,
+ duration,
+ workers,
+ outstanding: clients,
+ label: $"fstail-sidecar-withside-r{r}",
+ HostMode.FilesystemNoIndex,
+ DefaultFanoutMaxQueuedBytes,
+ enableTailCorrelation: true,
+ enableSameVolumeSidecar: true,
+ sidecarOpsPerSec: sidecarRate);
+ }
+ }
+
+ private async Task RunE2EAsync(
+ BenchRoot benchRoot,
+ TimeSpan duration,
+ int workers,
+ int outstanding,
+ string label,
+ HostMode mode = HostMode.FullIndexed,
+ long maxQueuedBytes = DefaultFanoutMaxQueuedBytes,
+ TimeSpan? sinkDelay = null,
+ bool enableTailCorrelation = false,
+ AbsoluteSlowThresholds? absoluteSlowThresholds = null,
+ Action? onLocalSlowThresholds = null,
+ bool applyLocalThresholdsAsAbsolute = false,
+ double? aggregateAdmitPerSec = null,
+ bool enableHostTelemetry = false,
+ int hostTelemetryIntervalMs = 100,
+ bool enableSameVolumeSidecar = false,
+ double sidecarOpsPerSec = 20)
+ {
+ var runRoot = Path.Join(benchRoot.Path, label + "-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(runRoot);
+ var spool = Path.Join(runRoot, "spool");
+ var db = Path.Join(runRoot, "db");
+ Directory.CreateDirectory(spool);
+ Directory.CreateDirectory(db);
+
+ await using var host = await EndToEndHost.StartAsync(
+ spool,
+ db,
+ workers,
+ mode,
+ maxQueuedBytes,
+ sinkDelay,
+ enableTailCorrelation);
+ // Sequential TLS TAKETHIS (write then read on one SslStream). ArticlePlacementClient's
+ // pipelined reader+writer deadlocks SslStream on large articles in this environment.
+ var clients = new SequentialTlsClient[Math.Max(1, outstanding)];
+ for (var i = 0; i < clients.Length; i++)
+ {
+ clients[i] = await SequentialTlsClient.ConnectAsync(host.Port).ConfigureAwait(false);
+ }
+
+ var admitStarts = new ConcurrentDictionary(concurrencyLevel: Environment.ProcessorCount, capacity: 4096);
+ var latenciesMs = new ConcurrentBag();
+ var admitLatencyMs = new ConcurrentBag();
+ var clientWriteMs = new ConcurrentBag();
+ var clientWait239Ms = new ConcurrentBag();
+ host.AttachCompletionObserver((id, completedTicks) =>
+ {
+ if (admitStarts.TryRemove(id, out var started))
+ {
+ latenciesMs.Add(Stopwatch.GetElapsedTime(started).TotalMilliseconds);
+ }
+ });
+ if (enableTailCorrelation)
+ {
+ host.AttachTailQueueSnapshot(() =>
+ (host.Queue.AccountedBytes, host.Queue.QueuedCount, host.Queue.InFlightBytes));
+ host.AttachTailAdmitStartLookup(id => admitStarts.TryGetValue(id, out var started) ? started : null);
+ }
+
+ var process = Process.GetCurrentProcess();
+ process.Refresh();
+ var wsStart = process.WorkingSet64;
+ var cpuStart = process.TotalProcessorTime;
+ var admitted = 0;
+ var admitRejected = 0;
+ var transportFailed = 0;
+ var nextId = 0;
+ var peakAccounted = 0L;
+ var peakQueued = 0;
+ var peakInFlight = 0L;
+ var wireBytes = 0L;
+ var samples = new List<(double t, long accounted, int queued, long inflight, long stored)>(512);
+ var gate = new object();
+ var stop = false;
+ AggregateAdmissionPacer? admitPacer = aggregateAdmitPerSec is double rate && rate > 0
+ ? new AggregateAdmissionPacer(rate)
+ : null;
+
+ var clock = Stopwatch.StartNew();
+ var benchOriginTicks = Stopwatch.GetTimestamp();
+ var benchOriginUtc = DateTime.UtcNow;
+ host.SetBenchOrigin(benchOriginTicks, benchOriginUtc);
+ HostTelemetrySampler? telemetry = null;
+ if (enableHostTelemetry && !IsCompletingSinkMode(mode))
+ {
+ telemetry = new HostTelemetrySampler(
+ TimeSpan.FromMilliseconds(hostTelemetryIntervalMs),
+ benchOriginTicks,
+ capacity: 4096);
+ telemetry.Start();
+ }
+
+ SameVolumeCreateNewSidecar? sidecar = null;
+ string? sidecarRoot = null;
+ if (enableSameVolumeSidecar && !IsCompletingSinkMode(mode))
+ {
+ sidecarRoot = Path.Join(
+ benchRoot.Path,
+ "sidecar-" + label + "-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(sidecarRoot);
+ sidecar = new SameVolumeCreateNewSidecar(
+ sidecarRoot,
+ sidecarOpsPerSec,
+ benchOriginTicks,
+ runId: label);
+ sidecar.Start();
+ }
+
+ try
+ {
+ var producers = clients.Select(client => Task.Run(async () =>
+ {
+ while (true)
+ {
+ lock (gate)
+ {
+ if (stop || clock.Elapsed >= duration)
+ {
+ return;
+ }
+ }
+
+ if (admitPacer is not null)
+ {
+ await admitPacer.WaitTurnAsync().ConfigureAwait(false);
+ lock (gate)
+ {
+ if (stop || clock.Elapsed >= duration)
+ {
+ return;
+ }
+ }
+ }
+
+ var index = Interlocked.Increment(ref nextId);
+ var built = BuildWireArticle(index, ArticleBytes);
+ var placeStarted = Stopwatch.GetTimestamp();
+ TakeThisTiming timing;
+ try
+ {
+ timing = await client.TakeThisTimedAsync(built).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (
+ ex is not OperationCanceledException && !ExceptionHelpers.IsFatal(ex))
+ {
+ Interlocked.Increment(ref transportFailed);
+ ExpectedException.Ignore(ex);
+ continue;
+ }
+
+ clientWriteMs.Add(timing.WriteMs);
+ clientWait239Ms.Add(timing.Wait239Ms);
+ if (timing.Code == 239)
+ {
+ Interlocked.Increment(ref admitted);
+ Interlocked.Add(ref wireBytes, built.WireBytes);
+ admitStarts[built.ArtId] = placeStarted;
+ admitLatencyMs.Add(timing.WriteMs + timing.Wait239Ms);
+ }
+ else if (timing.Code is 439 or 400)
+ {
+ Interlocked.Increment(ref admitRejected);
+ }
+ else
+ {
+ Interlocked.Increment(ref transportFailed);
+ }
+
+ var accounted = host.Queue.AccountedBytes;
+ var queued = host.Queue.QueuedCount;
+ var inflight = host.Queue.InFlightBytes;
+ lock (gate)
+ {
+ peakAccounted = Math.Max(peakAccounted, accounted);
+ peakQueued = Math.Max(peakQueued, queued);
+ peakInFlight = Math.Max(peakInFlight, inflight);
+ if (samples.Count == 0 || clock.Elapsed.TotalSeconds - samples[^1].t >= 1.0)
+ {
+ samples.Add((clock.Elapsed.TotalSeconds, accounted, queued, inflight, host.Counters.Stored));
+ }
+ }
+ }
+ })).ToArray();
+
+ await Task.Delay(duration).ConfigureAwait(false);
+ lock (gate)
+ {
+ stop = true;
+ }
+
+ await Task.WhenAll(producers).ConfigureAwait(false);
+
+ // Stop sidecar at sustain end (before drain) so correlation covers publish load only.
+ if (sidecar is not null)
+ {
+ await sidecar.StopAsync().ConfigureAwait(false);
+ }
+
+ var durableDone = () => IsCompletingSinkMode(mode)
+ ? host.ObservingCompleted
+ : host.Counters.Stored + host.Counters.Duplicates;
+ var remainingAtStop = Math.Max(0, admitted - (int)durableDone());
+ var drainDeadline = TimeSpan.FromSeconds(Math.Clamp(120 + remainingAtStop * 2, 120, 900));
+ var drainClock = Stopwatch.StartNew();
+ while ((host.Queue.QueuedCount > 0 || host.Queue.InFlightBytes > 0 || host.Queue.AccountedBytes > 0
+ || durableDone() < admitted)
+ && drainClock.Elapsed < drainDeadline)
+ {
+ await Task.Delay(50).ConfigureAwait(false);
+ }
+
+ var completed = durableDone();
+ var drained = host.Queue.QueuedCount == 0
+ && host.Queue.InFlightBytes == 0
+ && host.Queue.AccountedBytes == 0
+ && completed >= admitted;
+
+ process.Refresh();
+ var wsPeak = Math.Max(wsStart, process.WorkingSet64);
+ var cpu = process.TotalProcessorTime - cpuStart;
+ var elapsed = clock.Elapsed;
+ var sustainElapsed = duration;
+ var payloadBytes = host.Counters.StoredBytes;
+ if (payloadBytes == 0 && completed > 0)
+ {
+ payloadBytes = completed * (long)ArticleBytes;
+ }
+
+ var payloadGbit = payloadBytes * 8d / Math.Max(sustainElapsed.TotalSeconds, 0.001) / 1_000_000_000d;
+ var payloadGbitCompleted = completed > 0
+ ? ((double)completed * ArticleBytes * 8d)
+ / Math.Max(sustainElapsed.TotalSeconds, 0.001)
+ / 1_000_000_000d
+ : 0d;
+ // Application framing bytes (TAKETHIS + article + terminator). TLS record overhead not measured.
+ var wireGbitEst = wireBytes * 8d / Math.Max(sustainElapsed.TotalSeconds, 0.001) / 1_000_000_000d;
+
+ var latency = latenciesMs.ToArray();
+ Array.Sort(latency);
+ var admitLat = admitLatencyMs.ToArray();
+ Array.Sort(admitLat);
+ var writeLat = clientWriteMs.ToArray();
+ Array.Sort(writeLat);
+ var wait239 = clientWait239Ms.ToArray();
+ Array.Sort(wait239);
+ var processLat = host.ProcessLatencyMs.ToArray();
+ Array.Sort(processLat);
+
+ var indexedOk = 0;
+ var spoolOk = 0;
+ var checkedCount = 0;
+ if (!IsCompletingSinkMode(mode) && host.Index is not null)
+ {
+ foreach (var id in host.CompletedIds.Take(Math.Min(admitted, 256)))
+ {
+ checkedCount++;
+ if (host.Index.Contains(id))
+ {
+ indexedOk++;
+ }
+
+ if (host.Index.TryGetRetrievableArticle(id, out var path) && File.Exists(path))
+ {
+ spoolOk++;
+ }
+ }
+ }
+ else
+ {
+ checkedCount = Math.Min(admitted, 256);
+ indexedOk = checkedCount;
+ spoolOk = checkedCount;
+ }
+
+ var cpuPct = elapsed.TotalMilliseconds > 0
+ ? 100d * cpu.TotalMilliseconds / (elapsed.TotalMilliseconds * Environment.ProcessorCount)
+ : 0d;
+ if (telemetry is not null)
+ {
+ await telemetry.StopAsync().ConfigureAwait(false);
+ }
+
+ var occupancy = FormatOccupancyStats(samples, maxQueuedBytes);
+ var line =
+ $"{label} mode={mode} sustainMs={sustainElapsed.TotalMilliseconds:0} wallMs={elapsed.TotalMilliseconds:0} drainMs={drainClock.Elapsed.TotalMilliseconds:0} "
+ + $"articleBytes={ArticleBytes} workers={workers} outstanding={outstanding} maxQueuedBytes={maxQueuedBytes} "
+ + $"aggregateAdmitPerSecCfg={(aggregateAdmitPerSec is double cfg ? cfg.ToString("0.###") : "unrestricted")} "
+ + $"hostTelemetry={(enableHostTelemetry ? $"on_intervalMs={hostTelemetryIntervalMs}" : "off")} "
+ + $"sameVolumeSidecar={(enableSameVolumeSidecar ? $"on_opsPerSec={sidecarOpsPerSec:0.###}" : "off")} "
+ + $"dedicatedDisk={benchRoot.IsDedicated} storageRoot={runRoot} filesystem=NTFS "
+ + $"admitted={admitted} admitRejected={admitRejected} transportFailed={transportFailed} "
+ + $"persistedStored={host.Counters.Stored} duplicates={host.Counters.Duplicates} deferred={host.Counters.Deferred} "
+ + $"ioFailures={host.Counters.IoFailures} diskFull={host.Counters.DiskFull} conflicts={host.Counters.Conflicts} "
+ + $"admittedPerSec={admitted / sustainElapsed.TotalSeconds:0.00} completedPerSec={completed / sustainElapsed.TotalSeconds:0.00} "
+ + $"payloadEquivalentGbitPerSec_completedArtSize={payloadGbitCompleted:0.000} "
+ + $"payloadGbitPerSec_storedBytesCounter={payloadGbit:0.000} "
+ + $"wireGbitPerSec_appPayloadFraming={wireGbitEst:0.000} wireNote=takethis_plus_article_plus_terminator_not_tls_record_overhead "
+ + $"tlsClient=sequential_SslStream connections={clients.Length} "
+ + $"e2eLatencyMs_p50={Percentile(latency, 0.50):0.00} p95={Percentile(latency, 0.95):0.00} p99={Percentile(latency, 0.99):0.00} samples={latency.Length} "
+ + $"admitLatencyMs_p50={Percentile(admitLat, 0.50):0.00} p95={Percentile(admitLat, 0.95):0.00} "
+ + $"(clientWriteMs_p50={Percentile(writeLat, 0.50):0.00} clientWait239Ms_p50={Percentile(wait239, 0.50):0.00}) "
+ + $"processMs_p50={Percentile(processLat, 0.50):0.00} p95={Percentile(processLat, 0.95):0.00} p99={Percentile(processLat, 0.99):0.00} "
+ + $"{host.PublishTimingReport} "
+ + $"{host.PublishExtremeStallReport(sustainElapsed.TotalSeconds)} "
+ + $"{host.PublishResidualReport(Percentile(processLat, 0.50))} "
+ + $"{host.ConcurrencyReport} "
+ + $"{host.TailCorrelationReport} "
+ + $"{occupancy} "
+ + $"peakAccounted={peakAccounted} peakQueued={peakQueued} peakInFlight={peakInFlight} "
+ + $"finalAccounted={host.Queue.AccountedBytes} finalQueued={host.Queue.QueuedCount} finalInFlight={host.Queue.InFlightBytes} "
+ + $"drained={drained} remainingAtStop={remainingAtStop} consistencyChecked={checkedCount} indexedOk={indexedOk} spoolOk={spoolOk} "
+ + $"wsStart={wsStart} wsEnd={process.WorkingSet64} wsPeakObserved={wsPeak} "
+ + $"cpuMs={cpu.TotalMilliseconds:0} cpuPctOfMachine={cpuPct:0.0} managedBytes={GC.GetTotalMemory(false)} "
+ + $"{host.SlowOpEventReport(benchOriginTicks)} "
+ + $"{(telemetry?.Report(benchOriginTicks) ?? "hostTelemetrySamples=n/a")} "
+ + $"{(sidecar?.Report(sustainElapsed.TotalSeconds) ?? "sidecar=n/a")}";
+ Emit(line);
+
+ if (enableTailCorrelation && host.TryGetLocalSlowThresholds(out var localThresholds))
+ {
+ onLocalSlowThresholds?.Invoke(localThresholds);
+ var absolute = absoluteSlowThresholds
+ ?? (applyLocalThresholdsAsAbsolute ? localThresholds : (AbsoluteSlowThresholds?)null);
+ if (absolute is AbsoluteSlowThresholds abs)
+ {
+ Emit(
+ $"{label} absoluteSlowAnalysis "
+ + host.TailAbsoluteEventOrderReport(abs));
+ }
+ }
+
+ var e2eSlow = host.EnumerateSlowOpEvents(benchOriginTicks);
+ foreach (var slow in e2eSlow)
+ {
+ Emit($" {slow}");
+ if (telemetry is not null)
+ {
+ Emit($" {telemetry.FormatAlignment(slow.OpenStartTicks, slow.OpenEndTicks, benchOriginTicks)}");
+ }
+ }
+
+ if (sidecar is not null)
+ {
+ Emit($"{label} {sidecar.CorrelationReport(e2eSlow, benchOriginTicks)}");
+ foreach (var lineSidecar in sidecar.EnumerateSlowEvents(benchOriginTicks))
+ {
+ Emit($" {lineSidecar}");
+ }
+ }
+
+ foreach (var s in samples)
+ {
+ Emit($" sample t={s.t:0.0}s accounted={s.accounted} queued={s.queued} inflight={s.inflight} stored={s.stored}");
+ }
+
+ if (telemetry is not null && telemetry.FaultCount > 0)
+ {
+ Assert.Fail($"host telemetry recorded {telemetry.FaultCount} sampler fault(s); see hostTelemetry report");
+ }
+
+ if (sidecar is not null && sidecar.FaultCount > 0)
+ {
+ Assert.Fail($"same-volume sidecar recorded {sidecar.FaultCount} fault(s); see sidecar report");
+ }
+
+ Assert.True(admitted > 0, "expected at least one admitted article");
+ Assert.True(drained, $"accepted work did not drain within {drainDeadline.TotalSeconds:0}s (completed={completed} admitted={admitted})");
+ if (IsCompletingSinkMode(mode))
+ {
+ Assert.True(host.ObservingCompleted >= admitted);
+ // Sink modes (including calibrated delay) never attach TimingSpoolFiles.
+ Assert.Equal("publishTiming=n/a", host.PublishTimingReport);
+ }
+ else
+ {
+ Assert.Equal(admitted, completed);
+ Assert.Equal(checkedCount, indexedOk);
+ Assert.Equal(checkedCount, spoolOk);
+ }
+ }
+ finally
+ {
+ if (sidecar is not null)
+ {
+ try
+ {
+ await sidecar.StopAsync().ConfigureAwait(false);
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ Emit($"{label} sidecar-stop-warning {ex.GetType().Name}:{ex.Message}");
+ }
+ }
+
+ if (telemetry is not null)
+ {
+ try
+ {
+ await telemetry.StopAsync().ConfigureAwait(false);
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ Emit($"{label} hostTelemetry-stop-warning {ex.GetType().Name}:{ex.Message}");
+ }
+ }
+
+ foreach (var client in clients)
+ {
+ await client.DisposeAsync().ConfigureAwait(false);
+ }
+
+ if (sidecarRoot is not null)
+ {
+ try
+ {
+ Directory.Delete(sidecarRoot, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ Emit($"{label} sidecar-cleanup-warning {ex.GetType().Name}:{ex.Message}");
+ }
+ }
+
+ try
+ {
+ Directory.Delete(runRoot, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ // Best-effort cleanup of the isolated run directory.
+ ExpectedException.Ignore(ex);
+ }
+ }
+ }
+
+ private async Task MeasureShutdownDrainAsync(BenchRoot benchRoot, int articleCount, string label)
+ {
+ var runRoot = Path.Join(benchRoot.Path, label + "-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(runRoot);
+ var spool = Path.Join(runRoot, "spool");
+ var db = Path.Join(runRoot, "db");
+ Directory.CreateDirectory(spool);
+ Directory.CreateDirectory(db);
+
+ await using var host = await EndToEndHost.StartAsync(spool, db, workers: 4);
+ for (var i = 0; i < articleCount; i++)
+ {
+ Assert.True(host.Queue.TryEnqueue(BuildAccepted(i, ArticleBytes)));
+ }
+
+ var acceptedAtStop = host.Queue.EnqueuedCount;
+ var queuedAtStop = host.Queue.QueuedCount;
+ var inFlightAtStop = host.Queue.InFlightBytes;
+ var accountedAtStop = host.Queue.AccountedBytes;
+
+ using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5));
+ var sw = Stopwatch.StartNew();
+ Exception? stopError = null;
+ try
+ {
+ await host.StopWorkerAsync(timeout.Token).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ stopError = ex;
+ }
+ catch (OperationCanceledException oce)
+ {
+ stopError = oce;
+ }
+
+ sw.Stop();
+ var completed = host.Counters.Stored + host.Counters.Duplicates;
+ var line =
+ $"{label} articleBytes={ArticleBytes} acceptedAtStop={acceptedAtStop} queuedAtStop={queuedAtStop} "
+ + $"inFlightAtStop={inFlightAtStop} accountedAtStop={accountedAtStop} "
+ + $"shutdownMs={sw.Elapsed.TotalMilliseconds:0} completed={completed} "
+ + $"finalQueued={host.Queue.QueuedCount} finalAccounted={host.Queue.AccountedBytes} finalInFlight={host.Queue.InFlightBytes} "
+ + $"stopError={(stopError is null ? "none" : stopError.GetType().Name + ":" + stopError.Message)} "
+ + $"persistedOk={(stopError is null && completed == acceptedAtStop && host.Queue.AccountedBytes == 0)}";
+ Emit(line);
+
+ Assert.Null(stopError);
+ Assert.Equal(acceptedAtStop, completed);
+ Assert.Equal(0, host.Queue.AccountedBytes);
+
+ try
+ {
+ Directory.Delete(runRoot, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+ }
+
+ private async Task MeasureShutdownCancelFailsAsync(string label)
+ {
+ var time = TimeProvider.System;
+ var queue = new AcceptedArticleQueue(1 << 30, time);
+ var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+ var processor = new HoldingProcessor(entered);
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ new SpoolPersistenceOptions { MinWorkers = 1, MaxWorkers = 1 },
+ counters: new SpoolPersistenceCounters(),
+ time: time);
+ await worker.StartAsync(CancellationToken.None).ConfigureAwait(false);
+ Assert.True(queue.TryEnqueue(BuildAccepted(0, ArticleBytes)));
+ await entered.Task.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
+
+ var acceptedAtStop = queue.EnqueuedCount;
+ using var timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(150));
+ var sw = Stopwatch.StartNew();
+ var ex = await Assert.ThrowsAsync(() => worker.StopAsync(timeout.Token));
+ sw.Stop();
+ Emit(
+ $"{label} acceptedAtStop={acceptedAtStop} shutdownMs={sw.Elapsed.TotalMilliseconds:0} "
+ + $"completed={processor.Completed} outstandingAccounted={queue.AccountedBytes} "
+ + $"failureSurfaced=true message={ex.Message}");
+ Assert.Equal(0, processor.Completed);
+ Assert.Contains("did not finish", ex.Message, StringComparison.Ordinal);
+ }
+
+ private async Task MeasureDisconnectAfterAdmitAsync(BenchRoot benchRoot, string label)
+ {
+ var runRoot = Path.Join(benchRoot.Path, label + "-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(runRoot);
+ var spool = Path.Join(runRoot, "spool");
+ var db = Path.Join(runRoot, "db");
+ Directory.CreateDirectory(spool);
+ Directory.CreateDirectory(db);
+
+ await using var host = await EndToEndHost.StartAsync(spool, db, workers: 2);
+ WireArticle built;
+ await using (var client = await SequentialTlsClient.ConnectAsync(host.Port).ConfigureAwait(false))
+ {
+ built = BuildWireArticle(1, ArticleBytes);
+ var code = await client.TakeThisAsync(built).ConfigureAwait(false);
+ Assert.Equal(239, code);
+ }
+
+ var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60);
+ while (host.Counters.Stored + host.Counters.Duplicates < 1 && DateTime.UtcNow < deadline)
+ {
+ await Task.Delay(20).ConfigureAwait(false);
+ }
+
+ var completed = host.Counters.Stored + host.Counters.Duplicates;
+ var indexed = host.Index is not null && host.Index.Contains(built.ArtId);
+ var retrievable = host.Index is not null
+ && host.Index.TryGetRetrievableArticle(built.ArtId, out var path)
+ && File.Exists(path);
+ Emit(
+ $"{label} admitted=1 completed={completed} indexed={indexed} retrievable={retrievable} "
+ + $"finalAccounted={host.Queue.AccountedBytes} silentlyDropped={completed == 0}");
+ Assert.Equal(1, completed);
+ Assert.True(indexed);
+ Assert.True(retrievable);
+
+ try
+ {
+ Directory.Delete(runRoot, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+ }
+
+ private async Task MeasureNotifyWakeAsync()
+ {
+ var time = TimeProvider.System;
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var signal = CreateSignal();
+ var processor = new TimedCapacityProcessor();
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ },
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None).ConfigureAwait(false);
+ try
+ {
+ Assert.True(queue.TryEnqueue(BuildAccepted(Random.Shared.Next(), 4096)));
+ await WaitAsync(() => processor.WaitingCalls >= 1, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
+ var waitStarted = processor.LastWaitTicks;
+ processor.AllowComplete = true;
+ var notifyTicks = Stopwatch.GetTimestamp();
+ signal.Notify();
+ await WaitAsync(() => processor.Completed >= 1, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
+ var resumeTicks = processor.FirstResumeTicks;
+ var completeTicks = processor.CompleteTicks;
+ return new WakeSample(
+ Stopwatch.GetElapsedTime(notifyTicks, resumeTicks).TotalMilliseconds,
+ Stopwatch.GetElapsedTime(resumeTicks, completeTicks).TotalMilliseconds,
+ Stopwatch.GetElapsedTime(notifyTicks, completeTicks).TotalMilliseconds,
+ Stopwatch.GetElapsedTime(waitStarted, notifyTicks).TotalMilliseconds);
+ }
+ finally
+ {
+ processor.AllowComplete = true;
+ signal.Notify();
+ await worker.StopAsync(CancellationToken.None).ConfigureAwait(false);
+ }
+ }
+
+ private async Task MeasureSamplePublishedWakeAsync()
+ {
+ var root = Path.Join(ResolveBenchRoot().Path, "cap-sample-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(root);
+ try
+ {
+ var probe = new ControllableProbe();
+ probe.Set(root, total: 100L << 30, available: 1L << 20);
+ var catalog = new SegmentPoolCatalog();
+ catalog.Publish([new SegmentPoolDevice("block:cap", [Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))])]);
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new FlatCounters(),
+ NullLogger.Instance,
+ TimeSpan.FromHours(1),
+ TimeProvider.System,
+ probe);
+ InvokeSample(monitor);
+ var signal = new SpoolCapacityWaitSignal(monitor);
+ var time = TimeProvider.System;
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var counters = new SpoolPersistenceCounters();
+ var processor = new SpoolArticlePersistenceProcessor(
+ "cache01.usenet.ninja",
+ catalog,
+ monitor,
+ new SpoolWriteGate(),
+ new SpoolArticleIdentityGate(),
+ new SpoolHealth(time),
+ new SpoolPressure(signal),
+ new SpoolArticlePublisher(new FileSpoolArticleFiles()),
+ counters,
+ new StorageServerOptions(),
+ NullLogger.Instance,
+ time);
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ },
+ new SpoolPressure(signal),
+ counters,
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None).ConfigureAwait(false);
+ try
+ {
+ Assert.True(queue.TryEnqueue(BuildAccepted(1, 8192)));
+ await WaitAsync(() => counters.Deferred >= 1 || queue.InFlightBytes > 0, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
+ await Task.Delay(50).ConfigureAwait(false);
+ Assert.Equal(0, counters.Stored);
+
+ probe.Set(root, total: 100L << 30, available: 80L << 30);
+ var eligibleTicks = Stopwatch.GetTimestamp();
+ long notifyTicks = 0;
+ void OnSample() => notifyTicks = Stopwatch.GetTimestamp();
+ monitor.CapacitySamplePublished += OnSample;
+ try
+ {
+ InvokeSample(monitor);
+ Assert.True(notifyTicks > 0, "CapacitySamplePublished did not fire");
+ var eligibleToNotify = Stopwatch.GetElapsedTime(eligibleTicks, notifyTicks).TotalMilliseconds;
+ await WaitAsync(() => counters.Stored >= 1, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
+ var completeTicks = Stopwatch.GetTimestamp();
+ return new SampleWake(
+ eligibleToNotify,
+ Stopwatch.GetElapsedTime(notifyTicks, completeTicks).TotalMilliseconds,
+ Stopwatch.GetElapsedTime(notifyTicks, completeTicks).TotalMilliseconds);
+ }
+ finally
+ {
+ monitor.CapacitySamplePublished -= OnSample;
+ }
+ }
+ finally
+ {
+ await worker.StopAsync(CancellationToken.None).ConfigureAwait(false);
+ }
+ }
+ finally
+ {
+ try
+ {
+ Directory.Delete(root, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+ }
+ }
+
+ private async Task MeasureEvictionFreedWakeAsync()
+ {
+ var root = Path.Join(ResolveBenchRoot().Path, "cap-evict-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(root);
+ try
+ {
+ var probe = new ControllableProbe();
+ probe.Set(root, total: 100L << 30, available: 1L << 20);
+ var catalog = new SegmentPoolCatalog();
+ var spool = Path.TrimEndingDirectorySeparator(Path.GetFullPath(root));
+ catalog.Publish([new SegmentPoolDevice("block:evict", [spool])]);
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new FlatCounters(),
+ NullLogger.Instance,
+ TimeSpan.FromHours(1),
+ TimeProvider.System,
+ probe);
+ InvokeSample(monitor);
+ var signal = new SpoolCapacityWaitSignal(monitor);
+ var time = TimeProvider.System;
+ var queue = new AcceptedArticleQueue(1 << 20, time);
+ var processor = new TimedCapacityProcessor();
+ var worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ new SpoolPersistenceOptions
+ {
+ MinWorkers = 1,
+ MaxWorkers = 1,
+ IneligibleRetryMilliseconds = 60_000,
+ ControlIntervalMilliseconds = 60_000,
+ },
+ new SpoolPressure(signal),
+ new SpoolPersistenceCounters(),
+ time,
+ signal);
+ await worker.StartAsync(CancellationToken.None).ConfigureAwait(false);
+ try
+ {
+ Assert.True(queue.TryEnqueue(BuildAccepted(2, 4096)));
+ await WaitAsync(() => processor.WaitingCalls >= 1, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
+ processor.AllowComplete = true;
+ long publishedTicks = 0;
+ void OnPublished() => publishedTicks = Stopwatch.GetTimestamp();
+ monitor.CapacitySamplePublished += OnPublished;
+ try
+ {
+ var freedTicks = Stopwatch.GetTimestamp();
+ monitor.NoteFreedBytes(spool, 1L << 30);
+ Assert.True(publishedTicks > 0);
+ await WaitAsync(() => processor.Completed >= 1, TimeSpan.FromSeconds(5)).ConfigureAwait(false);
+ return new EvictionWake(
+ Stopwatch.GetElapsedTime(freedTicks, publishedTicks).TotalMilliseconds,
+ Stopwatch.GetElapsedTime(publishedTicks, processor.FirstResumeTicks).TotalMilliseconds,
+ Stopwatch.GetElapsedTime(publishedTicks, processor.CompleteTicks).TotalMilliseconds);
+ }
+ finally
+ {
+ monitor.CapacitySamplePublished -= OnPublished;
+ }
+ }
+ finally
+ {
+ processor.AllowComplete = true;
+ signal.Notify();
+ await worker.StopAsync(CancellationToken.None).ConfigureAwait(false);
+ }
+ }
+ finally
+ {
+ try
+ {
+ Directory.Delete(root, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+ }
+ }
+
+ private async Task MeasureCriticalSpoolExcludedAsync()
+ {
+ var root = Path.Join(ResolveBenchRoot().Path, "cap-crit-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(root);
+ try
+ {
+ var probe = new ControllableProbe();
+ // ~99% used → above default CriticalPercent 85.
+ probe.Set(root, total: 100L << 30, available: 1L << 20);
+ var catalog = new SegmentPoolCatalog();
+ catalog.Publish([new SegmentPoolDevice("block:crit", [Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))])]);
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new FlatCounters(),
+ NullLogger.Instance,
+ TimeSpan.FromHours(1),
+ TimeProvider.System,
+ probe);
+ InvokeSample(monitor);
+ var counters = new SpoolPersistenceCounters();
+ var processor = new SpoolArticlePersistenceProcessor(
+ "cache01.usenet.ninja",
+ catalog,
+ monitor,
+ new SpoolWriteGate(),
+ new SpoolArticleIdentityGate(),
+ new SpoolHealth(TimeProvider.System),
+ new SpoolPressure(),
+ new SpoolArticlePublisher(new FileSpoolArticleFiles()),
+ counters,
+ new StorageServerOptions(),
+ NullLogger.Instance,
+ TimeProvider.System);
+ var started = Stopwatch.GetTimestamp();
+ var result = await processor.ProcessAsync(BuildAccepted(3, 8192), CancellationToken.None).ConfigureAwait(false);
+ return new CriticalSample(result, counters.Deferred, Stopwatch.GetElapsedTime(started).TotalMilliseconds);
+ }
+ finally
+ {
+ try
+ {
+ Directory.Delete(root, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+ }
+ }
+
+ private async Task MeasureMultiSpoolIndependentRecoveryAsync()
+ {
+ var rootA = Path.Join(ResolveBenchRoot().Path, "cap-ms-a-" + Guid.NewGuid().ToString("N"));
+ var rootB = Path.Join(ResolveBenchRoot().Path, "cap-ms-b-" + Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(rootA);
+ Directory.CreateDirectory(rootB);
+ try
+ {
+ var probe = new ControllableProbe();
+ probe.Set(rootA, total: 100L << 30, available: 1L << 20);
+ probe.Set(rootB, total: 100L << 30, available: 80L << 30);
+ var catalog = new SegmentPoolCatalog();
+ var pathA = Path.TrimEndingDirectorySeparator(Path.GetFullPath(rootA));
+ var pathB = Path.TrimEndingDirectorySeparator(Path.GetFullPath(rootB));
+ catalog.Publish(
+ [
+ new SegmentPoolDevice("block:a", [pathA]),
+ new SegmentPoolDevice("block:b", [pathB]),
+ ]);
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new FlatCounters(),
+ NullLogger.Instance,
+ TimeSpan.FromHours(1),
+ TimeProvider.System,
+ probe);
+ InvokeSample(monitor);
+ var counters = new SpoolPersistenceCounters();
+ var processor = new SpoolArticlePersistenceProcessor(
+ "cache01.usenet.ninja",
+ catalog,
+ monitor,
+ new SpoolWriteGate(),
+ new SpoolArticleIdentityGate(),
+ new SpoolHealth(TimeProvider.System),
+ new SpoolPressure(),
+ new SpoolArticlePublisher(new FileSpoolArticleFiles()),
+ counters,
+ new StorageServerOptions(),
+ NullLogger.Instance,
+ TimeProvider.System);
+
+ var healthy = await processor.ProcessAsync(BuildAccepted(4, 8192), CancellationToken.None).ConfigureAwait(false);
+ Assert.Equal(AcceptedArticleProcessResult.Completed, healthy);
+ var healthyStored = counters.Stored;
+
+ probe.Set(rootB, total: 100L << 30, available: 1L << 20);
+ InvokeSample(monitor);
+ var blockedDeferredBefore = counters.Deferred;
+ var blocked = await processor.ProcessAsync(BuildAccepted(5, 8192), CancellationToken.None).ConfigureAwait(false);
+ Assert.Equal(AcceptedArticleProcessResult.WaitingForCapacity, blocked);
+
+ probe.Set(rootB, total: 100L << 30, available: 80L << 30);
+ var notifyTicks = Stopwatch.GetTimestamp();
+ InvokeSample(monitor);
+ var recovered = await processor.ProcessAsync(BuildAccepted(6, 8192), CancellationToken.None).ConfigureAwait(false);
+ Assert.Equal(AcceptedArticleProcessResult.Completed, recovered);
+ return new MultiSpoolSample(
+ counters.Deferred - blockedDeferredBefore,
+ healthyStored,
+ Stopwatch.GetElapsedTime(notifyTicks).TotalMilliseconds);
+ }
+ finally
+ {
+ try
+ {
+ Directory.Delete(rootA, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+
+ try
+ {
+ Directory.Delete(rootB, recursive: true);
+ }
+ catch (IOException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+ }
+ }
+
+ private bool Enabled() =>
+ string.Equals(Environment.GetEnvironmentVariable(EnvFlag), "1", StringComparison.Ordinal);
+
+ private void Skip() => _output.WriteLine($"Set {EnvFlag}=1 to run end-to-end resilience benches.");
+
+ private void Emit(string line)
+ {
+ Console.WriteLine(line);
+ _output.WriteLine(line);
+ }
+
+ private void WriteEnvPreamble(BenchRoot root, TimeSpan duration)
+ {
+ Emit(
+ $"e2e-env os={RuntimeInformation.OSDescription} arch={RuntimeInformation.OSArchitecture} "
+ + $"framework={RuntimeInformation.FrameworkDescription} cpus={Environment.ProcessorCount} "
+ + $"storageRoot={root.Path} dedicatedDisk={root.IsDedicated} filesystem=NTFS "
+ + $"articleBytes={ArticleBytes} durationSec={duration.TotalSeconds:0} "
+ + $"benchRootEnv={(Environment.GetEnvironmentVariable(EnvRoot) ?? "unset")}");
+ }
+
+ private static TimeSpan ResolveDuration(int defaultSeconds)
+ {
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvSeconds), out var seconds) && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(seconds);
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveBottleneckDuration(int defaultSeconds)
+ {
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvBottleneckSeconds), out var seconds) && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(seconds);
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveFanoutDuration(int defaultSeconds)
+ {
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvFanoutSeconds), out var seconds) && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(seconds);
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveStageDuration(int defaultSeconds)
+ {
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvStageSeconds), out var seconds) && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(seconds);
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveFsPubDuration(int defaultSeconds)
+ {
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvFsPubSeconds), out var seconds) && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(seconds);
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveE2eFsBottleneckDuration(int defaultSeconds)
+ {
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvE2eFsBottleneckSeconds), out var seconds) && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(Math.Clamp(seconds, 10, 180));
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveCalibratedDelayDuration(int defaultSeconds)
+ {
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvCalibratedDelaySeconds), out var seconds) && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(Math.Clamp(seconds, 10, 180));
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveFsTailDuration(int defaultSeconds)
+ {
+ if (int.TryParse(Environment.GetEnvironmentVariable(EnvFsTailSeconds), out var seconds) && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(Math.Clamp(seconds, 10, 180));
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveFsTailWorkerIsolationDuration(int defaultSeconds)
+ {
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvFsTailWorkerIsolationSeconds),
+ out var seconds)
+ && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(Math.Clamp(seconds, 10, 180));
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveAdmissionHeadroomDuration(int defaultSeconds)
+ {
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvAdmissionHeadroomSeconds),
+ out var seconds)
+ && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(Math.Clamp(seconds, 10, 180));
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveHostTelemetryDuration(int defaultSeconds)
+ {
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvHostTelemetrySeconds),
+ out var seconds)
+ && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(Math.Clamp(seconds, 10, 180));
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static TimeSpan ResolveFsTailSidecarDuration(int defaultSeconds)
+ {
+ if (int.TryParse(
+ Environment.GetEnvironmentVariable(EnvFsTailSidecarSeconds),
+ out var seconds)
+ && seconds > 0)
+ {
+ return TimeSpan.FromSeconds(Math.Clamp(seconds, 10, 180));
+ }
+
+ return TimeSpan.FromSeconds(defaultSeconds);
+ }
+
+ private static string FormatOccupancyStats(
+ List<(double t, long accounted, int queued, long inflight, long stored)> samples,
+ long maxQueuedBytes)
+ {
+ if (samples.Count == 0)
+ {
+ return "occupancySamples=0";
+ }
+
+ var accounted = new double[samples.Count];
+ var nearCap = 0;
+ var nearCapThreshold = (long)(maxQueuedBytes * 0.90);
+ for (var i = 0; i < samples.Count; i++)
+ {
+ accounted[i] = samples[i].accounted;
+ if (samples[i].accounted >= nearCapThreshold)
+ {
+ nearCap++;
+ }
+ }
+
+ Array.Sort(accounted);
+ return
+ $"occupancySamples={samples.Count} "
+ + $"accountedSample_p50={Percentile(accounted, 0.50):0} "
+ + $"p95={Percentile(accounted, 0.95):0} "
+ + $"max={accounted[^1]:0} "
+ + $"fracNearCap_ge90pctMaxQueued={nearCap / (double)samples.Count:0.000} "
+ + $"nearCapThresholdBytes={nearCapThreshold}";
+ }
+
+ private static bool IsCompletingSinkMode(HostMode mode) =>
+ mode is HostMode.CompletingSink or HostMode.CompletingSinkCalibratedDelay;
+
+ ///
+ /// Lightweight background sampler of process/GC/thread-pool counters (test-only).
+ /// Disk PDH counters are not collected (no PerformanceCounter package). Samples use the same
+ /// Stopwatch timeline as slow-op events; wall UTC is recorded for external alignment notes.
+ ///
+ private sealed class HostTelemetrySampler
+ {
+ private readonly TimeSpan _interval;
+ private readonly long _originTicks;
+ private readonly TelemetrySample[] _samples;
+ private readonly ConcurrentBag _faults = [];
+ private int _next;
+ private int _overwrites;
+ private int _count;
+ private CancellationTokenSource? _cts;
+ private Task? _loop;
+ private readonly Process _process = Process.GetCurrentProcess();
+
+ public HostTelemetrySampler(TimeSpan interval, long originTicks, int capacity)
+ {
+ ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 64);
+ _interval = interval;
+ _originTicks = originTicks;
+ _samples = new TelemetrySample[capacity];
+ }
+
+ public int FaultCount => _faults.Count;
+
+ public void Start()
+ {
+ _cts = new CancellationTokenSource();
+ _loop = Task.Run(() => LoopAsync(_cts.Token));
+ }
+
+ public async Task StopAsync()
+ {
+ if (_cts is null)
+ {
+ return;
+ }
+
+ _cts.Cancel();
+ if (_loop is not null)
+ {
+ try
+ {
+ await _loop.ConfigureAwait(false);
+ }
+ catch (OperationCanceledException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ _faults.Add($"{ex.GetType().Name}:{ex.Message}");
+ }
+ }
+
+ _cts.Dispose();
+ _cts = null;
+ _loop = null;
+ }
+
+ private async Task LoopAsync(CancellationToken cancellationToken)
+ {
+ var lastCpu = _process.TotalProcessorTime;
+ var lastPause = GC.GetTotalPauseDuration();
+ var lastTicks = Stopwatch.GetTimestamp();
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ try
+ {
+ await Task.Delay(_interval, cancellationToken).ConfigureAwait(false);
+ var nowTicks = Stopwatch.GetTimestamp();
+ _process.Refresh();
+ var cpu = _process.TotalProcessorTime;
+ var pause = GC.GetTotalPauseDuration();
+ var dt = (nowTicks - lastTicks) / (double)Stopwatch.Frequency;
+ var cpuDelta = (cpu - lastCpu).TotalSeconds;
+ var cpuPct = dt > 0
+ ? 100d * cpuDelta / (dt * Environment.ProcessorCount)
+ : 0;
+ var pauseDeltaMs = (pause - lastPause).TotalMilliseconds;
+ ThreadPool.GetAvailableThreads(out var availWorker, out var availIo);
+ ThreadPool.GetMaxThreads(out var maxWorker, out var maxIo);
+ var sample = new TelemetrySample(
+ nowTicks,
+ DateTime.UtcNow,
+ cpuPct,
+ _process.WorkingSet64,
+ _process.PrivateMemorySize64,
+ _process.Threads.Count,
+ GC.CollectionCount(0),
+ GC.CollectionCount(1),
+ GC.CollectionCount(2),
+ pauseDeltaMs,
+ GC.GetTotalMemory(false),
+ maxWorker - availWorker,
+ maxIo - availIo,
+ ThreadPool.PendingWorkItemCount);
+
+ var index = Interlocked.Increment(ref _next) - 1;
+ if (index >= _samples.Length)
+ {
+ Interlocked.Increment(ref _overwrites);
+ }
+
+ _samples[index % _samples.Length] = sample;
+ Volatile.Write(ref _count, Math.Min(index + 1, _samples.Length));
+ lastCpu = cpu;
+ lastPause = pause;
+ lastTicks = nowTicks;
+ }
+ catch (OperationCanceledException)
+ {
+ return;
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ _faults.Add($"{ex.GetType().Name}:{ex.Message}");
+ }
+ }
+ }
+
+ public string Report(long benchOriginTicks)
+ {
+ var n = Volatile.Read(ref _count);
+ var overwrites = Volatile.Read(ref _overwrites);
+ if (n <= 0)
+ {
+ return
+ $"hostTelemetrySamples=0 intervalMs={_interval.TotalMilliseconds:0} "
+ + $"cap={_samples.Length} overwrites={overwrites} faults={_faults.Count} "
+ + "diskCounters=unavailable_no_PerformanceCounter_package";
+ }
+
+ double sumCpu = 0, maxCpu = 0;
+ long maxWs = 0, maxPrivate = 0;
+ double sumPause = 0, maxPause = 0;
+ var maxPending = 0;
+ var maxThreads = 0;
+ for (var i = 0; i < n; i++)
+ {
+ var s = _samples[i];
+ sumCpu += s.CpuPctOfMachine;
+ maxCpu = Math.Max(maxCpu, s.CpuPctOfMachine);
+ maxWs = Math.Max(maxWs, s.WorkingSet);
+ maxPrivate = Math.Max(maxPrivate, s.PrivateBytes);
+ sumPause += s.GcPauseDeltaMs;
+ maxPause = Math.Max(maxPause, s.GcPauseDeltaMs);
+ maxPending = Math.Max(maxPending, (int)s.PendingWorkItems);
+ maxThreads = Math.Max(maxThreads, s.ThreadCount);
+ }
+
+ _ = benchOriginTicks;
+ return
+ $"hostTelemetrySamples={n} intervalMs={_interval.TotalMilliseconds:0} "
+ + $"cap={_samples.Length} overwrites={overwrites} faults={_faults.Count} "
+ + $"cpuPct_mean={sumCpu / n:0.0} max={maxCpu:0.0} "
+ + $"wsPeak={maxWs} privatePeak={maxPrivate} threadPeak={maxThreads} "
+ + $"gcPauseDeltaMs_sum={sumPause:0.000} maxSample={maxPause:0.000} "
+ + $"pendingWorkPeak={maxPending} "
+ + "diskCounters=unavailable_no_PerformanceCounter_package "
+ + "note=gcPauseDelta_is_sample_window_not_per_collection_cause";
+ }
+
+ public string FormatAlignment(long openStartTicks, long openEndTicks, long benchOriginTicks)
+ {
+ var n = Volatile.Read(ref _count);
+ if (n <= 0)
+ {
+ return "align samples=0";
+ }
+
+ TelemetrySample? before = null;
+ TelemetrySample? duringOrAfter = null;
+ for (var i = 0; i < n; i++)
+ {
+ var s = _samples[i];
+ if (s.Ticks <= openStartTicks)
+ {
+ before = s;
+ }
+
+ if (s.Ticks >= openStartTicks && duringOrAfter is null)
+ {
+ duringOrAfter = s;
+ }
+
+ if (s.Ticks >= openEndTicks && duringOrAfter is not null)
+ {
+ duringOrAfter = s;
+ break;
+ }
+ }
+
+ duringOrAfter ??= before;
+ static string Fmt(TelemetrySample? s, long origin)
+ {
+ if (s is null)
+ {
+ return "none";
+ }
+
+ var t = (s.Value.Ticks - origin) / (double)Stopwatch.Frequency;
+ return
+ $"t={t:0.000}s cpuPct={s.Value.CpuPctOfMachine:0.0} ws={s.Value.WorkingSet} "
+ + $"threads={s.Value.ThreadCount} gc01={s.Value.Gen0}/{s.Value.Gen1}/{s.Value.Gen2} "
+ + $"pauseDeltaMs={s.Value.GcPauseDeltaMs:0.000} pending={s.Value.PendingWorkItems} "
+ + $"busyWorkers={s.Value.BusyWorkers}";
+ }
+
+ var openT0 = (openStartTicks - benchOriginTicks) / (double)Stopwatch.Frequency;
+ var openT1 = (openEndTicks - benchOriginTicks) / (double)Stopwatch.Frequency;
+ return
+ $"align openWindow=[{openT0:0.000},{openT1:0.000}]s "
+ + $"before=({Fmt(before, benchOriginTicks)}) "
+ + $"near=({Fmt(duringOrAfter, benchOriginTicks)}) "
+ + $"sampleIntervalMayExceedStall_note=100ms_default_vs_50ms_event";
+ }
+
+ private readonly record struct TelemetrySample(
+ long Ticks,
+ DateTime Utc,
+ double CpuPctOfMachine,
+ long WorkingSet,
+ long PrivateBytes,
+ int ThreadCount,
+ int Gen0,
+ int Gen1,
+ int Gen2,
+ double GcPauseDeltaMs,
+ long ManagedBytes,
+ int BusyWorkers,
+ int BusyIo,
+ long PendingWorkItems);
+ }
+
+ ///
+ /// Low-rate same-volume sidecar (test-only). Runs in a
+ /// separate directory under the bench root while E2E publish continues. Used when elevated
+ /// ETW DiskIO/FileIO capture is unavailable; cannot alone establish root cause.
+ ///
+ private sealed class SameVolumeCreateNewSidecar
+ {
+ private const int EventCap = 4096;
+ private readonly string _root;
+ private readonly double _opsPerSec;
+ private readonly long _originTicks;
+ private readonly string _runId;
+ private readonly ConcurrentBag _openMs = [];
+ private readonly ConcurrentBag _closeMs = [];
+ private readonly ConcurrentBag _totalMs = [];
+ private readonly ConcurrentBag _slowEvents = [];
+ private readonly ConcurrentBag _faults = [];
+ private readonly ConcurrentBag _cleanupWarnings = [];
+ private CancellationTokenSource? _cts;
+ private Task? _loop;
+ private long _ops;
+ private int _stopped;
+
+ public SameVolumeCreateNewSidecar(
+ string root,
+ double opsPerSec,
+ long originTicks,
+ string runId)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(root);
+ ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(opsPerSec, 0);
+ _root = root;
+ _opsPerSec = opsPerSec;
+ _originTicks = originTicks;
+ _runId = runId;
+ }
+
+ public int FaultCount => _faults.Count;
+
+ public void Start()
+ {
+ _cts = new CancellationTokenSource();
+ _loop = Task.Run(() => LoopAsync(_cts.Token));
+ }
+
+ public async Task StopAsync()
+ {
+ if (Interlocked.Exchange(ref _stopped, 1) != 0)
+ {
+ return;
+ }
+
+ if (_cts is null)
+ {
+ return;
+ }
+
+ _cts.Cancel();
+ if (_loop is not null)
+ {
+ try
+ {
+ await _loop.ConfigureAwait(false);
+ }
+ catch (OperationCanceledException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ _faults.Add($"{ex.GetType().Name}:{ex.Message}");
+ }
+ }
+
+ _cts.Dispose();
+ _cts = null;
+ _loop = null;
+ }
+
+ private async Task LoopAsync(CancellationToken cancellationToken)
+ {
+ var interval = TimeSpan.FromSeconds(1d / _opsPerSec);
+ var seq = 0L;
+ while (!cancellationToken.IsCancellationRequested)
+ {
+ var slotStart = Stopwatch.GetTimestamp();
+ try
+ {
+ seq++;
+ var path = Path.Join(
+ _root,
+ $"sidecar-{_runId}-{seq:D8}-{Environment.CurrentManagedThreadId}.tmp");
+ var totalStart = Stopwatch.GetTimestamp();
+ var openStart = totalStart;
+ long openEnd;
+ long closeEnd;
+ using (new FileStream(
+ path,
+ FileMode.CreateNew,
+ FileAccess.Write,
+ FileShare.None,
+ bufferSize: 4096,
+ FileOptions.SequentialScan))
+ {
+ openEnd = Stopwatch.GetTimestamp();
+ }
+
+ closeEnd = Stopwatch.GetTimestamp();
+ var openMs = Ms(openEnd - openStart);
+ var closeMs = Ms(closeEnd - openEnd);
+ var totalMs = Ms(closeEnd - totalStart);
+ _openMs.Add(openMs);
+ _closeMs.Add(closeMs);
+ _totalMs.Add(totalMs);
+ Interlocked.Increment(ref _ops);
+
+ if ((openMs >= 10d || closeMs >= 10d || totalMs >= 10d)
+ && _slowEvents.Count < EventCap)
+ {
+ _slowEvents.Add(new SidecarOpEvent(
+ openMs,
+ closeMs,
+ totalMs,
+ openStart,
+ openEnd,
+ closeEnd,
+ Environment.CurrentManagedThreadId,
+ DateTime.UtcNow,
+ path));
+ }
+
+ try
+ {
+ File.Delete(path);
+ }
+ catch (IOException ex)
+ {
+ // Cleanup-only; do not treat as a sidecar operation fault.
+ _cleanupWarnings.Add($"{ex.GetType().Name}:{ex.Message}");
+ }
+ }
+ catch (OperationCanceledException)
+ {
+ return;
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ _faults.Add($"{ex.GetType().Name}:{ex.Message}");
+ }
+
+ var elapsed = TimeSpan.FromSeconds(
+ (Stopwatch.GetTimestamp() - slotStart) / (double)Stopwatch.Frequency);
+ var remaining = interval - elapsed;
+ if (remaining > TimeSpan.Zero)
+ {
+ try
+ {
+ await Task.Delay(remaining, cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ return;
+ }
+ }
+ }
+ }
+
+ public string Report(double sustainSeconds)
+ {
+ var opens = _openMs.ToArray();
+ var closes = _closeMs.ToArray();
+ var totals = _totalMs.ToArray();
+ var n = opens.Length;
+ var ops = Interlocked.Read(ref _ops);
+ if (n == 0)
+ {
+ return
+ $"sidecar ops={ops} n=0 opsPerSecCfg={_opsPerSec:0.###} "
+ + $"root={_root} runId={_runId} faults={_faults.Count} "
+ + $"cleanupWarnings={_cleanupWarnings.Count} "
+ + "note=same_volume_CreateNew_empty_file_close_delete";
+ }
+
+ Array.Sort(opens);
+ Array.Sort(closes);
+ Array.Sort(totals);
+ var openGe10 = CountGe(opens, 10);
+ var openGe50 = CountGe(opens, 50);
+ var openGe100 = CountGe(opens, 100);
+ var closeGe10 = CountGe(closes, 10);
+ var closeGe50 = CountGe(closes, 50);
+ var totalGe10 = CountGe(totals, 10);
+ var totalGe50 = CountGe(totals, 50);
+ var totalGe100 = CountGe(totals, 100);
+ var rate = sustainSeconds > 0 ? ops / sustainSeconds : 0;
+ return
+ $"sidecar ops={ops} n={n} opsPerSecCfg={_opsPerSec:0.###} "
+ + $"opsPerSecObs={rate:0.###} runId={_runId} faults={_faults.Count} "
+ + $"cleanupWarnings={_cleanupWarnings.Count} "
+ + $"openMs_p50={Percentile(opens, 0.50):0.000} p95={Percentile(opens, 0.95):0.000} "
+ + $"p99={Percentile(opens, 0.99):0.000} max={opens[^1]:0.000} "
+ + $"open_ge10={openGe10} ge50={openGe50} ge100={openGe100} "
+ + $"openGe50_per1000={(n > 0 ? 1000d * openGe50 / n : 0):0.000} "
+ + $"closeMs_p50={Percentile(closes, 0.50):0.000} p95={Percentile(closes, 0.95):0.000} "
+ + $"max={closes[^1]:0.000} close_ge10={closeGe10} ge50={closeGe50} "
+ + $"totalMs_p50={Percentile(totals, 0.50):0.000} p95={Percentile(totals, 0.95):0.000} "
+ + $"max={totals[^1]:0.000} total_ge10={totalGe10} ge50={totalGe50} ge100={totalGe100} "
+ + $"slowEventsRetained={_slowEvents.Count} cap={EventCap} "
+ + "note=same_volume_CreateNew_empty_file_close_delete_not_root_cause_alone";
+ }
+
+ public string CorrelationReport(
+ IReadOnlyList e2eSlow,
+ long benchOriginTicks)
+ {
+ var sidecarSlow = _slowEvents.ToArray();
+ Array.Sort(sidecarSlow, static (a, b) => a.OpenStartTicks.CompareTo(b.OpenStartTicks));
+ // Retain only ≥50 ms opens for burst windows (slow-op ring already ≥50).
+ var e2eOpen = e2eSlow
+ .Where(static e =>
+ e.Line.Contains("stage=open", StringComparison.Ordinal)
+ && e.Line.Contains("ms=", StringComparison.Ordinal))
+ .ToList();
+
+ var windowTicks = (long)(0.060 * Stopwatch.Frequency);
+ var sidecarInE2eWindow = 0;
+ var e2eWindowsWithSidecar = 0;
+ var sidecarAloneGe50 = 0;
+ foreach (var s in sidecarSlow)
+ {
+ if (s.OpenMs < 50d)
+ {
+ continue;
+ }
+
+ sidecarAloneGe50++;
+ var hit = false;
+ foreach (var e in e2eOpen)
+ {
+ // Overlap or within 60 ms of either edge.
+ if (IntervalsNear(
+ s.OpenStartTicks,
+ s.OpenEndTicks,
+ e.OpenStartTicks,
+ e.OpenEndTicks,
+ windowTicks))
+ {
+ hit = true;
+ break;
+ }
+ }
+
+ if (hit)
+ {
+ sidecarInE2eWindow++;
+ }
+ }
+
+ foreach (var e in e2eOpen)
+ {
+ foreach (var s in sidecarSlow)
+ {
+ if (s.OpenMs < 10d)
+ {
+ continue;
+ }
+
+ if (IntervalsNear(
+ s.OpenStartTicks,
+ s.OpenEndTicks,
+ e.OpenStartTicks,
+ e.OpenEndTicks,
+ windowTicks))
+ {
+ e2eWindowsWithSidecar++;
+ break;
+ }
+ }
+ }
+
+ _ = benchOriginTicks;
+ return
+ $"sidecarCorr e2eOpenSlowN={e2eOpen.Count} sidecarSlowRetained={sidecarSlow.Length} "
+ + $"sidecarOpenGe50={sidecarAloneGe50} "
+ + $"sidecarGe50_nearE2eOpen60ms={sidecarInE2eWindow} "
+ + $"e2eOpen_withSidecarGe10_near60ms={e2eWindowsWithSidecar} "
+ + "window=60ms_overlap_or_edge "
+ + "note=shared_volume_coincidence_not_disk_IO_proof";
+ }
+
+ public IEnumerable EnumerateSlowEvents(long benchOriginTicks)
+ {
+ var events = _slowEvents.ToArray();
+ Array.Sort(events, static (a, b) => a.OpenStartTicks.CompareTo(b.OpenStartTicks));
+ foreach (var e in events)
+ {
+ var t0 = (e.OpenStartTicks - benchOriginTicks) / (double)Stopwatch.Frequency;
+ var t1 = (e.OpenEndTicks - benchOriginTicks) / (double)Stopwatch.Frequency;
+ var tClose = (e.CloseEndTicks - benchOriginTicks) / (double)Stopwatch.Frequency;
+ yield return
+ $"sidecarSlow openMs={e.OpenMs:0.000} closeMs={e.CloseMs:0.000} "
+ + $"totalMs={e.TotalMs:0.000} tOpen=[{t0:0.000},{t1:0.000}]s "
+ + $"tCloseEnd={tClose:0.000}s thread={e.ThreadId} utc={e.Utc:O} "
+ + $"file={Path.GetFileName(e.Path)}";
+ }
+ }
+
+ private static bool IntervalsNear(
+ long a0,
+ long a1,
+ long b0,
+ long b1,
+ long windowTicks)
+ {
+ // Overlap
+ if (a0 <= b1 && b0 <= a1)
+ {
+ return true;
+ }
+
+ // Gap between intervals ≤ window
+ if (a1 < b0)
+ {
+ return b0 - a1 <= windowTicks;
+ }
+
+ if (b1 < a0)
+ {
+ return a0 - b1 <= windowTicks;
+ }
+
+ return false;
+ }
+
+ private static int CountGe(double[] sorted, double threshold)
+ {
+ var n = 0;
+ for (var i = 0; i < sorted.Length; i++)
+ {
+ if (sorted[i] >= threshold)
+ {
+ n++;
+ }
+ }
+
+ return n;
+ }
+
+ private static double Ms(long ticks) => ticks * 1000d / Stopwatch.Frequency;
+
+ private readonly record struct SidecarOpEvent(
+ double OpenMs,
+ double CloseMs,
+ double TotalMs,
+ long OpenStartTicks,
+ long OpenEndTicks,
+ long CloseEndTicks,
+ int ThreadId,
+ DateTime Utc,
+ string Path);
+ }
+
+ ///
+ /// Test-only shared token-bucket pacing across TLS client producers. Limits aggregate
+ /// TakeThis start rate without changing server admission or queue semantics.
+ ///
+ private sealed class AggregateAdmissionPacer
+ {
+ private readonly long _intervalTicks;
+ private long _nextSlotTicks;
+
+ public AggregateAdmissionPacer(double admitsPerSecond)
+ {
+ ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(admitsPerSecond, 0);
+ _intervalTicks = (long)(Stopwatch.Frequency / admitsPerSecond);
+ ArgumentOutOfRangeException.ThrowIfLessThan(_intervalTicks, 1);
+ }
+
+ public async Task WaitTurnAsync()
+ {
+ while (true)
+ {
+ var now = Stopwatch.GetTimestamp();
+ var slot = Volatile.Read(ref _nextSlotTicks);
+ var start = Math.Max(now, slot);
+ var next = start + _intervalTicks;
+ if (Interlocked.CompareExchange(ref _nextSlotTicks, next, slot) != slot)
+ {
+ continue;
+ }
+
+ var delayTicks = start - now;
+ if (delayTicks > 0)
+ {
+ var delay = TimeSpan.FromSeconds(delayTicks / (double)Stopwatch.Frequency);
+ if (delay > TimeSpan.Zero)
+ {
+ await Task.Delay(delay).ConfigureAwait(false);
+ }
+ }
+
+ return;
+ }
+ }
+ }
+
+ ///
+ /// Absolute slow-event thresholds (typically w=4 local p95) applied across worker cells.
+ ///
+ private readonly record struct AbsoluteSlowThresholds(
+ double ProcessP95Ms,
+ double OpenP95Ms,
+ double WriteP95Ms,
+ double MoveP95Ms);
+
+ ///
+ /// CPU-only Path rewrite microbench (no spool I/O). Labeled separately from E2E FilesystemNoIndex.
+ /// Uses the same path as production ProcessCore.
+ ///
+ private void MeasurePathRewriteCpuMicrobench(int iterations)
+ {
+ var article = BuildAccepted(42, ArticleBytes);
+ var path = new AcceptedArticlePathProcessor("cache01.usenet.ninja", NullLogger.Instance);
+ path.EnsureReady();
+ // Warm one rewrite so parser/materializer steady-state is represented.
+ if (!path.TryRewriteRented(article, out var warm, out _) || warm is null)
+ {
+ Emit("fspub-path-rewrite-cpu FAILED warm rewrite");
+ return;
+ }
+
+ AcceptedArticlePathProcessor.ReturnRewriteBuffer(warm);
+
+ var samples = new double[iterations];
+ var wall = Stopwatch.StartNew();
+ for (var i = 0; i < iterations; i++)
+ {
+ var started = Stopwatch.GetTimestamp();
+ if (!path.TryRewriteRented(article, out var rented, out _) || rented is null)
+ {
+ Emit($"fspub-path-rewrite-cpu FAILED iteration={i}");
+ return;
+ }
+
+ AcceptedArticlePathProcessor.ReturnRewriteBuffer(rented);
+ samples[i] = Stopwatch.GetElapsedTime(started).TotalMilliseconds;
+ }
+
+ wall.Stop();
+ Array.Sort(samples);
+ var perSec = iterations / Math.Max(wall.Elapsed.TotalSeconds, 0.001);
+ Emit(
+ $"fspub-path-rewrite-cpu iterations={iterations} wallMs={wall.Elapsed.TotalMilliseconds:0} "
+ + $"rewritesPerSec={perSec:0.00} rewriteMs_p50={Percentile(samples, 0.50):0.000} "
+ + $"p95={Percentile(samples, 0.95):0.000} p99={Percentile(samples, 0.99):0.000} "
+ + "scope=AcceptedArticlePathProcessor.TryRewriteRented_only "
+ + "note=no_disk_no_spool_select_no_TLS_not_equivalent_to_E2E_processMs");
+ }
+
+ private static List ParsePositiveIntList(string? raw)
+ {
+ var list = new List();
+ if (string.IsNullOrWhiteSpace(raw))
+ {
+ return list;
+ }
+
+ list.AddRange(
+ raw.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+ .Select(static part => int.TryParse(part, out var value) ? value : 0)
+ .Where(static value => value > 0));
+
+ return list;
+ }
+
+ private enum HostMode
+ {
+ FullIndexed,
+ FilesystemNoIndex,
+ CompletingSink,
+ /// Test-only CompletingSink that busy-waits a calibrated duration with no FS I/O.
+ CompletingSinkCalibratedDelay,
+ }
+
+ private static BenchRoot ResolveBenchRoot()
+ {
+ var configured = Environment.GetEnvironmentVariable(EnvRoot);
+ if (!string.IsNullOrWhiteSpace(configured))
+ {
+ var path = Path.GetFullPath(configured);
+ Directory.CreateDirectory(path);
+ return new BenchRoot(path, IsDedicated: true);
+ }
+
+ // AppContext.BaseDirectory is typically tests/.../bin/x64/Release/net10.0 — five levels up is the repo root.
+ var fromRepo = Path.GetFullPath(Path.Join(
+ AppContext.BaseDirectory,
+ "..",
+ "..",
+ "..",
+ "..",
+ "..",
+ ".artifacts",
+ "perf",
+ "e2e-workdir"));
+ Directory.CreateDirectory(fromRepo);
+ return new BenchRoot(fromRepo, IsDedicated: false);
+ }
+
+ private static SpoolCapacityWaitSignal CreateSignal()
+ {
+ var catalog = new SegmentPoolCatalog();
+ catalog.Publish([new SegmentPoolDevice("block:e2e", [Path.GetTempPath()])]);
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new FlatCounters(),
+ NullLogger.Instance,
+ TimeSpan.FromHours(1),
+ TimeProvider.System,
+ freeSpace: null);
+ return new SpoolCapacityWaitSignal(monitor);
+ }
+
+ private static void InvokeSample(BlockDeviceMonitorService monitor) =>
+ monitor.GetType()
+ .GetMethod("Sample", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)!
+ .Invoke(monitor, null);
+
+ private static async Task WaitAsync(Func condition, TimeSpan timeout)
+ {
+ var deadline = DateTime.UtcNow + timeout;
+ while (!condition())
+ {
+ if (DateTime.UtcNow >= deadline)
+ {
+ throw new TimeoutException("condition not met within " + timeout);
+ }
+
+ await Task.Delay(5).ConfigureAwait(false);
+ }
+ }
+
+ private static WireArticle BuildWireArticle(int index, int bodyBytes)
+ {
+ var messageId = "";
+ var header = Encoding.ASCII.GetBytes(
+ "Path: peer.example\r\nDate: Fri, 23 Aug 2024 07:30:10 +0000\r\nMessage-ID: "
+ + messageId
+ + "\r\nNewsgroups: alt.test\r\nFrom: user@example.test\r\nSubject: e2e\r\n\r\n");
+ var body = new byte[Math.Max(80, bodyBytes - header.Length)];
+ var line = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\r\n"u8;
+ for (var offset = 0; offset < body.Length; offset += line.Length)
+ {
+ var count = Math.Min(line.Length, body.Length - offset);
+ line.Slice(0, count).CopyTo(body.AsSpan(offset, count));
+ }
+
+ body[0] = (byte)(index & 0xFF);
+ // Multiline terminator is recognized only at a line start; the logical article must end with LF.
+ if (body.Length >= 2
+ && (body[^2] != (byte)'\r' || body[^1] != (byte)'\n'))
+ {
+ body[^2] = (byte)'\r';
+ body[^1] = (byte)'\n';
+ }
+
+ var logical = new byte[header.Length + body.Length];
+ header.CopyTo(logical, 0);
+ body.CopyTo(logical, header.Length);
+ var created = ArticleRecordFactory.TryCreate(
+ new NntpArticleParser("cache01.usenet.ninja"),
+ logical,
+ ArticlePathMode.Normalize);
+ Assert.True(created.IsAccepted, created.MaterializeFailure.ToString());
+ var command = Encoding.ASCII.GetBytes("TAKETHIS " + messageId + "\r\n");
+ var stuffed = DotStuff(logical);
+ var wireBytes = command.Length + stuffed.Length + 3; // .\r\n
+ return new WireArticle(created.Record.ArtId, messageId, logical, stuffed, command, wireBytes);
+ }
+
+ private static AcceptedArticle BuildAccepted(int index, int bodyBytes)
+ {
+ var built = BuildWireArticle(index, bodyBytes);
+ var created = ArticleRecordFactory.TryCreate(
+ new NntpArticleParser("cache01.usenet.ninja"),
+ built.Logical,
+ ArticlePathMode.Normalize);
+ Assert.True(created.IsAccepted);
+ return new AcceptedArticle(
+ created.Record.ArtId,
+ created.Record.ArtHash,
+ created.Record.ArtSize,
+ created.Record.ArtData);
+ }
+
+ private static byte[] DotStuff(ReadOnlySpan article)
+ {
+ var length = article.Length;
+ var atLineStart = true;
+ for (var i = 0; i < article.Length; i++)
+ {
+ if (atLineStart && article[i] == (byte)'.')
+ {
+ length++;
+ }
+
+ atLineStart = article[i] == (byte)'\n';
+ }
+
+ var stuffed = new byte[length];
+ var w = 0;
+ atLineStart = true;
+ for (var i = 0; i < article.Length; i++)
+ {
+ if (atLineStart && article[i] == (byte)'.')
+ {
+ stuffed[w++] = (byte)'.';
+ }
+
+ stuffed[w++] = article[i];
+ atLineStart = article[i] == (byte)'\n';
+ }
+
+ return stuffed;
+ }
+
+ private readonly record struct WireArticle(
+ ArticleId ArtId,
+ string MessageId,
+ byte[] Logical,
+ byte[] Stuffed,
+ byte[] Command,
+ long WireBytes);
+
+ ///
+ /// Client-side TAKETHIS timing. is local SSL write of command+body+terminator.
+ /// is wait for the status line (server receive+admit+response, plus network).
+ ///
+ private readonly record struct TakeThisTiming(int Code, double WriteMs, double Wait239Ms);
+
+ private sealed class SequentialTlsClient : IAsyncDisposable
+ {
+ private readonly TcpClient _tcp;
+ private readonly SslStream _ssl;
+ private readonly List _pending = [];
+ private readonly byte[] _readBuffer = new byte[64 * 1024];
+
+ private SequentialTlsClient(TcpClient tcp, SslStream ssl)
+ {
+ _tcp = tcp;
+ _ssl = ssl;
+ }
+
+ public static async Task ConnectAsync(int port)
+ {
+ var tcp = new TcpClient { NoDelay = true };
+ try
+ {
+ tcp.SendBufferSize = 1 << 20;
+ tcp.ReceiveBufferSize = 1 << 20;
+ }
+ catch (SocketException ex)
+ {
+ ExpectedException.Ignore(ex);
+ }
+
+ await tcp.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false);
+ var ssl = new SslStream(tcp.GetStream(), leaveInnerStreamOpen: false);
+ await ssl.AuthenticateAsClientAsync(
+ new SslClientAuthenticationOptions
+ {
+ TargetHost = "localhost",
+ EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13,
+ RemoteCertificateValidationCallback = static (_, _, _, _) => true,
+ },
+ CancellationToken.None)
+ .ConfigureAwait(false);
+ var client = new SequentialTlsClient(tcp, ssl);
+ Assert.Equal(200, await client.ReadStatusAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false));
+ return client;
+ }
+
+ public async Task TakeThisAsync(WireArticle article)
+ {
+ var timing = await TakeThisTimedAsync(article).ConfigureAwait(false);
+ return timing.Code;
+ }
+
+ public async Task TakeThisTimedAsync(WireArticle article)
+ {
+ var writeStarted = Stopwatch.GetTimestamp();
+ await _ssl.WriteAsync(article.Command).ConfigureAwait(false);
+ const int chunk = 64 * 1024;
+ for (var offset = 0; offset < article.Stuffed.Length; offset += chunk)
+ {
+ var length = Math.Min(chunk, article.Stuffed.Length - offset);
+ await _ssl.WriteAsync(article.Stuffed.AsMemory(offset, length)).ConfigureAwait(false);
+ }
+
+ await _ssl.WriteAsync(".\r\n"u8.ToArray()).ConfigureAwait(false);
+ var writeMs = Stopwatch.GetElapsedTime(writeStarted).TotalMilliseconds;
+ var waitStarted = Stopwatch.GetTimestamp();
+ var code = await ReadStatusAsync(TimeSpan.FromMinutes(2)).ConfigureAwait(false);
+ var waitMs = Stopwatch.GetElapsedTime(waitStarted).TotalMilliseconds;
+ return new TakeThisTiming(code, writeMs, waitMs);
+ }
+
+ public async ValueTask DisposeAsync()
+ {
+ try
+ {
+ await _ssl.DisposeAsync().ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is IOException or ObjectDisposedException)
+ {
+ ExpectedException.Ignore(ex);
+ }
+
+ _tcp.Dispose();
+ }
+
+ private async Task ReadStatusAsync(TimeSpan timeout)
+ {
+ using var cts = new CancellationTokenSource(timeout);
+ while (true)
+ {
+ for (var i = 0; i < _pending.Count - 1; i++)
+ {
+ if (_pending[i] == (byte)'\r' && _pending[i + 1] == (byte)'\n')
+ {
+ var line = Encoding.ASCII.GetString(_pending.GetRange(0, i).ToArray());
+ _pending.RemoveRange(0, i + 2);
+ return ((line[0] - '0') * 100) + ((line[1] - '0') * 10) + (line[2] - '0');
+ }
+ }
+
+ var read = await _ssl.ReadAsync(_readBuffer, cts.Token).ConfigureAwait(false);
+ if (read == 0)
+ {
+ throw new EndOfStreamException();
+ }
+
+ for (var i = 0; i < read; i++)
+ {
+ _pending.Add(_readBuffer[i]);
+ }
+ }
+ }
+ }
+
+ private static double Median(IReadOnlyList sorted) => Percentile(sorted, 0.50);
+
+ private static double Percentile(IReadOnlyList sorted, double percentile)
+ {
+ if (sorted.Count == 0)
+ {
+ return 0;
+ }
+
+ var index = (int)Math.Ceiling(sorted.Count * percentile) - 1;
+ if (index < 0)
+ {
+ index = 0;
+ }
+
+ return sorted[Math.Min(index, sorted.Count - 1)];
+ }
+
+ private readonly record struct BenchRoot(string Path, bool IsDedicated);
+
+ private readonly record struct WakeSample(
+ double NotifyToResumeMs,
+ double ResumeToCompleteMs,
+ double NotifyToCompleteMs,
+ double CapacityWaitMs);
+
+ private readonly record struct SampleWake(
+ double EligibleToNotifyMs,
+ double NotifyToResumeMs,
+ double NotifyToCompleteMs);
+
+ private readonly record struct EvictionWake(
+ double FreedToNotifyMs,
+ double NotifyToResumeMs,
+ double NotifyToCompleteMs);
+
+ private readonly record struct CriticalSample(
+ AcceptedArticleProcessResult Result,
+ long Deferred,
+ double ElapsedMs);
+
+ private readonly record struct MultiSpoolSample(
+ long BlockedDeferred,
+ long HealthyStored,
+ double NotifyToCompleteMs);
+
+ private sealed class EndToEndHost : IAsyncDisposable
+ {
+ private readonly StorageNntpListenerService _listener;
+ private readonly AcceptedArticleWorker _worker;
+ private readonly BlockDeviceMonitorService? _monitor;
+ private readonly ArticleIndexService? _index;
+ private readonly TimingSpoolFiles? _timingFiles;
+ private Action? _onCompleted;
+
+ private EndToEndHost(
+ int port,
+ AcceptedArticleQueue queue,
+ StorageNntpListenerService listener,
+ AcceptedArticleWorker worker,
+ BlockDeviceMonitorService? monitor,
+ ArticleIndexService? index,
+ SpoolPersistenceCounters counters,
+ ObservingProcessor observing,
+ TimingSpoolFiles? timingFiles,
+ PublishTailCorrelator? tailCorrelator)
+ {
+ Port = port;
+ Queue = queue;
+ _listener = listener;
+ _worker = worker;
+ _monitor = monitor;
+ _index = index;
+ Counters = counters;
+ Observing = observing;
+ _timingFiles = timingFiles;
+ _tailCorrelator = tailCorrelator;
+ }
+
+ public int Port { get; }
+
+ public AcceptedArticleQueue Queue { get; }
+
+ public SpoolPersistenceCounters Counters { get; }
+
+ public ArticleIndexService? Index => _index;
+
+ public ConcurrentBag CompletedIds => Observing.CompletedIds;
+
+ public int ObservingCompleted => Observing.CompletedCount;
+
+ public ConcurrentBag ProcessLatencyMs => Observing.ProcessLatencyMs;
+
+ public string PublishTimingReport => _timingFiles?.Report() ?? "publishTiming=n/a";
+
+ public string PublishExtremeStallReport(double sustainSeconds) =>
+ _timingFiles?.ExtremeStallReport(sustainSeconds) ?? "extremeStalls=n/a";
+
+ public string SlowOpEventReport(long benchOriginTicks) =>
+ _timingFiles?.SlowOpEventReport(benchOriginTicks) ?? "slowOpEvents=n/a";
+
+ public IReadOnlyList EnumerateSlowOpEvents(long benchOriginTicks) =>
+ _timingFiles?.EnumerateSlowOpEvents(benchOriginTicks) ?? Array.Empty();
+
+ public void SetBenchOrigin(long originTicks, DateTime originUtc) =>
+ _timingFiles?.SetBenchOrigin(originTicks, originUtc);
+
+ public string PublishResidualReport(double processMsP50) =>
+ _timingFiles?.ResidualReport(processMsP50) ?? "publishResidual=n/a";
+
+ public string ConcurrencyReport =>
+ Observing.ConcurrencyReport
+ + " "
+ + (_timingFiles?.ConcurrencyReport() ?? "publishConcurrency=n/a");
+
+ public string TailCorrelationReport =>
+ _tailCorrelator?.Report() ?? "tailCorrelation=n/a";
+
+ public bool TryGetLocalSlowThresholds(out AbsoluteSlowThresholds thresholds)
+ {
+ if (_tailCorrelator is null || !_tailCorrelator.TryGetLocalPercentileThresholds(out thresholds))
+ {
+ thresholds = default;
+ return false;
+ }
+
+ return true;
+ }
+
+ public string TailAbsoluteEventOrderReport(AbsoluteSlowThresholds absolute) =>
+ _tailCorrelator?.AbsoluteEventOrderReport(absolute)
+ ?? "absoluteSlowAnalysis=n/a";
+
+ private ObservingProcessor Observing { get; }
+
+ private readonly PublishTailCorrelator? _tailCorrelator;
+
+ public void AttachCompletionObserver(Action onCompleted) => _onCompleted = onCompleted;
+
+ public void AttachTailQueueSnapshot(Func<(long Accounted, int Queued, long InFlight)> snapshot) =>
+ Observing.AttachTailQueueSnapshot(snapshot);
+
+ public void AttachTailAdmitStartLookup(Func lookup) =>
+ Observing.AttachTailAdmitStartLookup(lookup);
+
+ public Task StopWorkerAsync(CancellationToken cancellationToken) => _worker.StopAsync(cancellationToken);
+
+ public static Task StartAsync(string spoolRoot, string dbDir, int workers) =>
+ StartAsync(spoolRoot, dbDir, workers, HostMode.FullIndexed, DefaultFanoutMaxQueuedBytes);
+
+ public static Task StartAsync(string spoolRoot, string dbDir, int workers, HostMode mode) =>
+ StartAsync(spoolRoot, dbDir, workers, mode, DefaultFanoutMaxQueuedBytes);
+
+ public static async Task StartAsync(
+ string spoolRoot,
+ string dbDir,
+ int workers,
+ HostMode mode,
+ long maxQueuedBytes,
+ TimeSpan? sinkDelay = null,
+ bool enableTailCorrelation = false)
+ {
+ var spool = Path.TrimEndingDirectorySeparator(Path.GetFullPath(spoolRoot));
+ var catalog = ArticleIndexTestSupport.Catalog(spool);
+ var probe = new GenerousProbe();
+ // SampleFreshness is 5s; without a fresh free-space sample every spool is ineligible.
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new FlatCounters(),
+ NullLogger.Instance,
+ TimeSpan.FromMilliseconds(250),
+ TimeProvider.System,
+ probe);
+ InvokeSample(monitor);
+
+ ArticleIndexService? index = null;
+ if (mode == HostMode.FullIndexed)
+ {
+ index = await ArticleIndexTestSupport.StartAsync(dbDir, catalog, ArticleIndexTestSupport.FastSettings())
+ .ConfigureAwait(false);
+ }
+
+ var counters = new SpoolPersistenceCounters();
+ var signal = new SpoolCapacityWaitSignal(monitor);
+ TimingSpoolFiles? timingFiles = null;
+ PublishTailCorrelator? tailCorrelator = enableTailCorrelation && mode == HostMode.FilesystemNoIndex
+ ? new PublishTailCorrelator(capacity: 8192)
+ : null;
+ IAcceptedArticleProcessor inner;
+ if (mode == HostMode.CompletingSink)
+ {
+ inner = new CompletingSinkProcessor();
+ }
+ else if (mode == HostMode.CompletingSinkCalibratedDelay)
+ {
+ var delay = sinkDelay ?? TimeSpan.FromMilliseconds(DefaultCalibratedDelayMs);
+ if (delay <= TimeSpan.Zero)
+ {
+ throw new ArgumentOutOfRangeException(nameof(sinkDelay), delay, "Calibrated sink delay must be positive.");
+ }
+
+ inner = new CompletingSinkCalibratedDelayProcessor(delay);
+ }
+ else
+ {
+ timingFiles = new TimingSpoolFiles(new FileSpoolArticleFiles());
+ inner = new SpoolArticlePersistenceProcessor(
+ "cache01.usenet.ninja",
+ catalog,
+ monitor,
+ new SpoolWriteGate(),
+ new SpoolArticleIdentityGate(),
+ new SpoolHealth(TimeProvider.System),
+ new SpoolPressure(signal),
+ new SpoolArticlePublisher(timingFiles),
+ counters,
+ new StorageServerOptions
+ {
+ DbDir = dbDir,
+ SpoolPersistence = new SpoolPersistenceOptions
+ {
+ MaxInFlightPerSpool = Math.Max(8, workers * 2),
+ },
+ },
+ NullLogger.Instance,
+ TimeProvider.System,
+ index);
+ }
+
+ var observing = new ObservingProcessor(inner, tailCorrelator);
+ // Bound the queue so TLS admission back-pressures instead of building multi-GB backlog on temp volume.
+ // Fan-out may override via VECTORNNTPD_STORAGE_CLIENT_FANOUT_MAX_QUEUED_BYTES (test-only; default 64 MiB).
+ var queue = new AcceptedArticleQueue(maxQueuedBytes, TimeProvider.System);
+ var worker = new AcceptedArticleWorker(
+ queue,
+ observing,
+ NullLogger.Instance,
+ new SpoolPersistenceOptions
+ {
+ MinWorkers = workers,
+ MaxWorkers = workers,
+ ControlIntervalMilliseconds = 100,
+ },
+ new SpoolPressure(signal),
+ counters,
+ TimeProvider.System,
+ signal);
+
+ var port = StorageServerTestOptions.GetFreeTcpPort();
+ var identity = StorageServerTestOptions.CreateValid();
+ identity.BindAddress = ["127.0.0.1"];
+ identity.BindPortTls = port;
+ var acme = StorageServerTestOptions.CreateValidAcme(identity);
+ acme.BindAddress = ["127.0.0.1"];
+ acme.BindPortTls = port;
+ var runtime = StorageServerRuntimeOptionsFactory.Create(identity, acme);
+ var readiness = new AcmeCertificateReadiness();
+ readiness.MarkReady();
+ var listener = new StorageNntpListenerService(
+ runtime,
+ TestListenerCertificates.CreatePublishedProvider(),
+ readiness,
+ NullLogger.Instance,
+ acceptedArticles: queue,
+ assemblyAdmission: new StoreAssemblyAdmission(64));
+
+ await monitor.StartAsync(CancellationToken.None).ConfigureAwait(false);
+ await worker.StartAsync(CancellationToken.None).ConfigureAwait(false);
+ await listener.StartAsync(CancellationToken.None).ConfigureAwait(false);
+
+ var host = new EndToEndHost(
+ port,
+ queue,
+ listener,
+ worker,
+ monitor,
+ index,
+ counters,
+ observing,
+ timingFiles,
+ tailCorrelator);
+ observing.SetObserver((id, ticks) => host._onCompleted?.Invoke(id, ticks));
+ return host;
+ }
+
+ public async ValueTask DisposeAsync()
+ {
+ try
+ {
+ await _listener.DisposeAsync().ConfigureAwait(false);
+ }
+ catch (Exception ex) when (!ExceptionHelpers.IsFatal(ex))
+ {
+ // Listener dispose is best-effort during bench teardown.
+ ExpectedException.Ignore(ex);
+ }
+
+ try
+ {
+ await _worker.StopAsync(CancellationToken.None).ConfigureAwait(false);
+ }
+ catch (InvalidOperationException)
+ {
+ // Drain failure already recorded by the scenario.
+ }
+
+ if (_monitor is not null)
+ {
+ await _monitor.StopAsync(CancellationToken.None).ConfigureAwait(false);
+ }
+
+ if (_index is not null)
+ {
+ await ArticleIndexTestSupport.StopAsync(_index).ConfigureAwait(false);
+ }
+ }
+ }
+
+ private sealed class ObservingProcessor : IAcceptedArticleProcessor
+ {
+ private readonly IAcceptedArticleProcessor _inner;
+ private readonly PublishTailCorrelator? _tailCorrelator;
+ private Action _observer;
+ private Func<(long Accounted, int Queued, long InFlight)>? _queueSnapshot;
+ private Func? _admitStartLookup;
+ private int _completed;
+ private int _processInFlight;
+ private int _processMaxInFlight;
+
+ public ObservingProcessor(IAcceptedArticleProcessor inner, PublishTailCorrelator? tailCorrelator = null)
+ {
+ _inner = inner;
+ _tailCorrelator = tailCorrelator;
+ _observer = static (_, _) => { };
+ }
+
+ public ConcurrentBag CompletedIds { get; } = [];
+
+ public ConcurrentBag ProcessLatencyMs { get; } = [];
+
+ public int CompletedCount => Volatile.Read(ref _completed);
+
+ public string ConcurrencyReport =>
+ $"processInFlightMax={Volatile.Read(ref _processMaxInFlight)}";
+
+ public void SetObserver(Action observer) => _observer = observer;
+
+ public void AttachTailQueueSnapshot(Func<(long Accounted, int Queued, long InFlight)> snapshot) =>
+ _queueSnapshot = snapshot;
+
+ public void AttachTailAdmitStartLookup(Func lookup) =>
+ _admitStartLookup = lookup;
+
+ public void EnsureReady() => _inner.EnsureReady();
+
+ public async ValueTask ProcessAsync(
+ AcceptedArticle article,
+ CancellationToken cancellationToken)
+ {
+ var inFlight = Interlocked.Increment(ref _processInFlight);
+ UpdateMax(ref _processMaxInFlight, inFlight);
+ var started = Stopwatch.GetTimestamp();
+ var stages = _tailCorrelator is not null
+ ? PublishStageAccumulator.Begin(_queueSnapshot)
+ : null;
+
+ try
+ {
+ var result = await _inner.ProcessAsync(article, cancellationToken).ConfigureAwait(false);
+ var processMs = Stopwatch.GetElapsedTime(started).TotalMilliseconds;
+ ProcessLatencyMs.Add(processMs);
+ if (result == AcceptedArticleProcessResult.Completed)
+ {
+ if (stages is not null)
+ {
+ stages.SnapProcessEnd();
+ var admitStart = _admitStartLookup?.Invoke(article.ArtId);
+ var e2eMs = admitStart is long ticks
+ ? Stopwatch.GetElapsedTime(ticks).TotalMilliseconds
+ : double.NaN;
+ _tailCorrelator!.TryAdd(article.ArtId, processMs, stages, e2eMs);
+ }
+
+ Interlocked.Increment(ref _completed);
+ CompletedIds.Add(article.ArtId);
+ _observer(article.ArtId, Stopwatch.GetTimestamp());
+ }
+
+ return result;
+ }
+ finally
+ {
+ PublishStageAccumulator.Clear();
+ Interlocked.Decrement(ref _processInFlight);
+ }
+ }
+
+ private static void UpdateMax(ref int target, int candidate)
+ {
+ var current = Volatile.Read(ref target);
+ while (candidate > current)
+ {
+ var previous = Interlocked.CompareExchange(ref target, candidate, current);
+ if (previous == current)
+ {
+ return;
+ }
+
+ current = previous;
+ }
+ }
+ }
+
+ private sealed class CompletingSinkProcessor : IAcceptedArticleProcessor
+ {
+ public void EnsureReady()
+ {
+ }
+
+ public ValueTask ProcessAsync(
+ AcceptedArticle article,
+ CancellationToken cancellationToken) =>
+ ValueTask.FromResult(AcceptedArticleProcessResult.Completed);
+ }
+
+ ///
+ /// Test-only CompletingSink that holds the worker for using a Stopwatch
+ /// busy-wait (no filesystem I/O). Production publish Write/Move are synchronous on the worker;
+ /// would release the worker and mis-model that occupancy. Windows
+ /// timer coarseness also makes sub-2 ms unreliable. This is not
+ /// equivalent to disk I/O even when elapsed times match.
+ ///
+ private sealed class CompletingSinkCalibratedDelayProcessor : IAcceptedArticleProcessor
+ {
+ private readonly long _delayTicks;
+
+ public CompletingSinkCalibratedDelayProcessor(TimeSpan delay)
+ {
+ ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(delay, TimeSpan.Zero);
+ _delayTicks = (long)(delay.TotalSeconds * Stopwatch.Frequency);
+ }
+
+ public void EnsureReady()
+ {
+ }
+
+ public ValueTask ProcessAsync(
+ AcceptedArticle article,
+ CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var deadline = Stopwatch.GetTimestamp() + _delayTicks;
+ while (Stopwatch.GetTimestamp() < deadline)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ }
+
+ return ValueTask.FromResult(AcceptedArticleProcessResult.Completed);
+ }
+ }
+
+ ///
+ /// Test-only filesystem adapter timings for publication substages.
+ /// WriteAll is instrumented by mirroring WriteNew
+ /// (CreateNew + Write + Flush(false)) so open/write/flush can be separated without production hooks.
+ /// Timings are application-level wall time, not device service time. Stages are exclusive of each other
+ /// within WriteAll; CreateDirectory/Exists/Move are exclusive of WriteAll.
+ ///
+ private sealed class TimingSpoolFiles : ISpoolArticleFiles
+ {
+ private readonly ISpoolArticleFiles _inner;
+ private long _writeTotalTicks;
+ private long _openTicks;
+ private long _writeBytesTicks;
+ private long _flushTicks;
+ private long _moveTicks;
+ private long _directoryTicks;
+ private long _existsTicks;
+ private int _writeCalls;
+ private int _moveCalls;
+ private int _directoryCalls;
+ private int _existsCalls;
+ private int _flushCalls;
+ private int _writeInFlight;
+ private int _writeMaxInFlight;
+ private int _moveInFlight;
+ private int _moveMaxInFlight;
+ private readonly ConcurrentBag _writeTotalMs = [];
+ private readonly ConcurrentBag _openMs = [];
+ private readonly ConcurrentBag _writeBytesMs = [];
+ private readonly ConcurrentBag _flushMs = [];
+ private readonly ConcurrentBag _moveMs = [];
+ private readonly ConcurrentBag _slowOps = [];
+ private const int SlowOpCap = 512;
+ private long _benchOriginTicks;
+ private DateTime _benchOriginUtc;
+
+ public TimingSpoolFiles(ISpoolArticleFiles inner) => _inner = inner;
+
+ public void SetBenchOrigin(long originTicks, DateTime originUtc)
+ {
+ _benchOriginTicks = originTicks;
+ _benchOriginUtc = originUtc;
+ }
+
+ private void RecordSlowOp(
+ string stage,
+ double ms,
+ long startTicks,
+ long endTicks,
+ long accBefore,
+ long accAfter,
+ int queuedBefore,
+ int queuedAfter)
+ {
+ if (ms < 50 || _slowOps.Count >= SlowOpCap)
+ {
+ return;
+ }
+
+ _slowOps.Add(new SlowOpEvent(
+ stage,
+ ms,
+ startTicks,
+ endTicks,
+ Environment.CurrentManagedThreadId,
+ accBefore,
+ accAfter,
+ queuedBefore,
+ queuedAfter,
+ DateTime.UtcNow));
+ }
+
+ public string SlowOpEventReport(long benchOriginTicks)
+ {
+ var events = _slowOps.ToArray();
+ if (events.Length == 0)
+ {
+ return $"slowOpEvents n=0 cap={SlowOpCap}";
+ }
+
+ Array.Sort(events, static (a, b) => a.StartTicks.CompareTo(b.StartTicks));
+ var openN = 0;
+ var moveN = 0;
+ var clusters = 0;
+ var windowTicks = (long)(0.010 * Stopwatch.Frequency);
+ long? prevOpenEnd = null;
+ foreach (var e in events)
+ {
+ if (e.Stage == "open")
+ {
+ openN++;
+ if (prevOpenEnd is long prev && e.StartTicks - prev <= windowTicks)
+ {
+ clusters++;
+ }
+
+ prevOpenEnd = e.EndTicks;
+ }
+ else if (e.Stage == "move")
+ {
+ moveN++;
+ }
+ }
+
+ var first = events[0];
+ var last = events[^1];
+ return
+ $"slowOpEvents n={events.Length} cap={SlowOpCap} openGe50={openN} moveGe50={moveN} "
+ + $"openClusterPairs_le10ms={clusters} "
+ + $"firstT={TicksToSec(first.StartTicks, benchOriginTicks):0.000}s stage={first.Stage} ms={first.Ms:0.000} "
+ + $"lastT={TicksToSec(last.StartTicks, benchOriginTicks):0.000}s stage={last.Stage} ms={last.Ms:0.000} "
+ + "clock=Stopwatch_monotonic_plus_UtcNow_at_record";
+ }
+
+ public IReadOnlyList EnumerateSlowOpEvents(long benchOriginTicks)
+ {
+ var events = _slowOps.ToArray();
+ Array.Sort(events, static (a, b) => a.StartTicks.CompareTo(b.StartTicks));
+ var list = new List(events.Length);
+ foreach (var e in events)
+ {
+ var t0 = TicksToSec(e.StartTicks, benchOriginTicks);
+ var t1 = TicksToSec(e.EndTicks, benchOriginTicks);
+ list.Add(new SlowOpEventView(
+ e.StartTicks,
+ e.EndTicks,
+ $"slowOp stage={e.Stage} ms={e.Ms:0.000} tStart={t0:0.000}s tEnd={t1:0.000}s "
+ + $"thread={e.ThreadId} accBefore={e.AccBefore} accAfter={e.AccAfter} "
+ + $"accDelta={e.AccAfter - e.AccBefore} queuedBefore={e.QueuedBefore} queuedAfter={e.QueuedAfter} "
+ + $"utc={e.Utc:O}"));
+ }
+
+ return list;
+ }
+
+ private static double TicksToSec(long ticks, long origin) =>
+ (ticks - origin) / (double)Stopwatch.Frequency;
+
+ public string ConcurrencyReport() =>
+ $"publishWriteInFlightMax={Volatile.Read(ref _writeMaxInFlight)} "
+ + $"publishMoveInFlightMax={Volatile.Read(ref _moveMaxInFlight)} "
+ + "concurrencyNote=test_only_atomics_around_WriteAll_MoveNew";
+
+ ///