Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,149 +4,29 @@
namespace VectorNNTPD.BackFillerService.ArticleWork
{
/// <summary>
/// Decides whether retained ArtData can be sent as an RFC 3977 multiline block, and builds the
/// stuffed header bytes that are written ahead of the retained body.
/// BackFiller facade over <see cref="CanonicalArticleDeliveryFraming"/> for retained ArtData delivery.
/// </summary>
/// <remarks>
/// <para>
/// Retained ArtData is canonical headers, the header/body separator, and a body whose lines
/// are already dot-stuffed. The retained buffer is never rewritten. A header line that begins
/// with <c>.</c> is stuffed only in the socket image, because a client removes one leading dot
/// per RFC 3977 §3.1.1. The body is the existing slice and is not copied into that image.
/// </para>
/// <para>
/// Every line in the retained buffer must end in CRLF. A bare CR, a bare LF, or NUL cannot be
/// conveyed and is not frameable. A dot-leading line with no <c>CRLF CRLF</c> separator is also
/// not frameable: the header region and the already-stuffed body cannot be told apart, so stuffing
/// the whole buffer could corrupt the body.
/// </para>
/// StorageService and Common callers use <see cref="CanonicalArticleDeliveryFraming"/> directly.
/// This type keeps existing BackFiller call sites and tests stable.
/// </remarks>
internal static class ArticleWorkDeliveryFraming
{
/// <summary>
/// Reports whether <paramref name="artData"/> can be sent as a multiline block without changing the retained bytes.
/// </summary>
/// <param name="artData">Retained article. Not modified. Only this span is examined.</param>
/// <returns>
/// <see langword="true"/> when every line ends in CRLF, the buffer ends in CRLF, NUL is absent,
/// and any dot-leading line sits in a header region before <c>CRLF CRLF</c>.
/// </returns>
/// <inheritdoc cref="CanonicalArticleDeliveryFraming.IsFrameable"/>
internal static bool IsFrameable(ReadOnlySpan<byte> artData) =>
CanonicalArticleFraming.Analyze(artData).IsFrameable;
CanonicalArticleDeliveryFraming.IsFrameable(artData);

/// <summary>
/// Builds the dot-stuffed header image when a header line begins with <c>.</c>.
/// </summary>
/// <param name="artData">Retained article. Not modified.</param>
/// <param name="stuffedHeader">
/// The stuffed header, including the header/body separator, when at least one header line needed a dot.
/// <see langword="null"/> when the caller should send <paramref name="artData"/> unchanged,
/// or when <paramref name="artData"/> is not frameable.
/// The body is not included.
/// </param>
/// <returns>
/// <see langword="true"/> when <paramref name="stuffedHeader"/> holds the stuffed header.
/// <see langword="false"/> when the article is not frameable, has no header/body separator,
/// or no header line begins with <c>.</c>.
/// </returns>
/// <remarks>
/// <see cref="IsFrameable"/> runs before the header is copied. The body stays in <paramref name="artData"/>.
/// </remarks>
/// <inheritdoc cref="CanonicalArticleDeliveryFraming.TryStuffHeaderLines"/>
internal static bool TryStuffHeaderLines(
ReadOnlySpan<byte> artData,
[NotNullWhen(true)] out byte[]? stuffedHeader)
{
stuffedHeader = null;
var framing = CanonicalArticleFraming.Analyze(artData);
if (!framing.IsFrameable)
{
return false;
}
[NotNullWhen(true)] out byte[]? stuffedHeader) =>
CanonicalArticleDeliveryFraming.TryStuffHeaderLines(artData, out stuffedHeader);

return TryCreateStuffedHeader(artData, framing.BodyStart, out stuffedHeader);
}

/// <summary>
/// Copies <paramref name="artData"/> through <paramref name="bodyStart"/> with one extra leading dot
/// on each header line that begins with <c>.</c>.
/// </summary>
/// <param name="artData">Retained article. Not modified. Bytes at and after <paramref name="bodyStart"/> are not read for stuffing.</param>
/// <param name="bodyStart">Index of the first body byte from an earlier framing analysis of this same span.</param>
/// <param name="stuffedHeader">The stuffed header when a dot was inserted. Otherwise <see langword="null"/>.</param>
/// <returns><see langword="true"/> when <paramref name="stuffedHeader"/> was allocated.</returns>
/// <remarks>
/// Does not walk <paramref name="artData"/> past <paramref name="bodyStart"/> and does not decide frameability.
/// The caller uses a body start taken from the same logical bytes.
/// </remarks>
/// <inheritdoc cref="CanonicalArticleDeliveryFraming.TryCreateStuffedHeader"/>
internal static bool TryCreateStuffedHeader(
ReadOnlySpan<byte> artData,
int bodyStart,
[NotNullWhen(true)] out byte[]? stuffedHeader)
{
stuffedHeader = null;
if ((uint)bodyStart > (uint)artData.Length)
{
return false;
}

var header = artData[..bodyStart];
var extra = CountDotLeadingLines(header);
if (extra == 0)
{
return false;
}

stuffedHeader = GC.AllocateUninitializedArray<byte>(header.Length + extra);
WriteDotStuffedLines(header, stuffedHeader);
return true;
}

/// <summary>Counts CRLF lines in <paramref name="region"/> whose first byte is <c>.</c>.</summary>
/// <param name="region">Header region, including the separator.</param>
/// <returns>The number of stuffing dots a header write inserts.</returns>
private static int CountDotLeadingLines(ReadOnlySpan<byte> region)
{
var extra = 0;
var index = 0;
while (index < region.Length)
{
if (region[index] == (byte)'.')
{
extra++;
}

var crlf = region[index..].IndexOf("\r\n"u8);
if (crlf < 0)
{
break;
}

index += crlf + 2;
}

return extra;
}

/// <summary>Writes <paramref name="region"/> with one extra leading dot on each line that begins with <c>.</c>.</summary>
/// <param name="region">Header region whose lines end in CRLF.</param>
/// <param name="destination">Buffer of <paramref name="region"/> length plus the stuffing count.</param>
private static void WriteDotStuffedLines(ReadOnlySpan<byte> region, Span<byte> destination)
{
var written = 0;
var index = 0;
while (index < region.Length)
{
if (region[index] == (byte)'.')
{
destination[written++] = (byte)'.';
}

var crlf = region[index..].IndexOf("\r\n"u8);
var lineLength = crlf + 2;
region.Slice(index, lineLength).CopyTo(destination[written..]);
written += lineLength;
index += lineLength;
}
}
[NotNullWhen(true)] out byte[]? stuffedHeader) =>
CanonicalArticleDeliveryFraming.TryCreateStuffedHeader(artData, bodyStart, out stuffedHeader);
}
}
151 changes: 151 additions & 0 deletions src/VectorNNTPD.Common/Articles/CanonicalArticleDeliveryFraming.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
using System.Diagnostics.CodeAnalysis;

namespace VectorNNTPD.Common.Articles
{
/// <summary>
/// Decides whether CanonicalV1 ArtData can be sent as an RFC 3977 multiline block, and builds the
/// stuffed header bytes that are written ahead of the retained body.
/// </summary>
/// <remarks>
/// <para>
/// CanonicalV1 ArtData is canonical headers, the header/body separator, and a body whose lines
/// are already storage-dot-stuffed. The retained buffer is never rewritten. A header line that begins
/// with <c>.</c> is stuffed only in the socket image, because a client removes one leading dot
/// per RFC 3977 §3.1.1. The body is the existing slice and is not copied into that image.
/// </para>
/// <para>
/// Every line in the retained buffer must end in CRLF. A bare CR, a bare LF, or NUL cannot be
/// conveyed and is not frameable. A dot-leading line with no <c>CRLF CRLF</c> separator is also
/// not frameable: the header region and the already-stuffed body cannot be told apart, so stuffing
/// the whole buffer could corrupt the body.
/// </para>
/// </remarks>
internal static class CanonicalArticleDeliveryFraming
{
/// <summary>
/// Reports whether <paramref name="artData"/> can be sent as a multiline block without changing the retained bytes.
/// </summary>
/// <param name="artData">Retained article. Not modified. Only this span is examined.</param>
/// <returns>
/// <see langword="true"/> when every line ends in CRLF, the buffer ends in CRLF, NUL is absent,
/// and any dot-leading line sits in a header region before <c>CRLF CRLF</c>.
/// </returns>
internal static bool IsFrameable(ReadOnlySpan<byte> artData) =>
CanonicalArticleFraming.Analyze(artData).IsFrameable;

/// <summary>
/// Builds the dot-stuffed header image when a header line begins with <c>.</c>.
/// </summary>
/// <param name="artData">Retained article. Not modified.</param>
/// <param name="stuffedHeader">
/// The stuffed header, including the header/body separator, when at least one header line needed a dot.
/// <see langword="null"/> when the caller should send <paramref name="artData"/> unchanged,
/// or when <paramref name="artData"/> is not frameable.
/// The body is not included.
/// </param>
/// <returns>
/// <see langword="true"/> when <paramref name="stuffedHeader"/> holds the stuffed header.
/// <see langword="false"/> when the article is not frameable, has no header/body separator,
/// or no header line begins with <c>.</c>.
/// </returns>
/// <remarks>
/// <see cref="IsFrameable"/> runs before the header is copied. The body stays in <paramref name="artData"/>.
/// </remarks>
internal static bool TryStuffHeaderLines(
ReadOnlySpan<byte> artData,
[NotNullWhen(true)] out byte[]? stuffedHeader)
{
stuffedHeader = null;
var framing = CanonicalArticleFraming.Analyze(artData);
if (!framing.IsFrameable)
{
return false;
}

return TryCreateStuffedHeader(artData, framing.BodyStart, out stuffedHeader);
}

/// <summary>
/// Copies <paramref name="artData"/> through <paramref name="bodyStart"/> with one extra leading dot
/// on each header line that begins with <c>.</c>.
/// </summary>
/// <param name="artData">Retained article. Not modified. Bytes at and after <paramref name="bodyStart"/> are not read for stuffing.</param>
/// <param name="bodyStart">Index of the first body byte from an earlier framing analysis of this same span.</param>
/// <param name="stuffedHeader">The stuffed header when a dot was inserted. Otherwise <see langword="null"/>.</param>
/// <returns><see langword="true"/> when <paramref name="stuffedHeader"/> was allocated.</returns>
/// <remarks>
/// Does not walk <paramref name="artData"/> past <paramref name="bodyStart"/> and does not decide frameability.
/// The caller uses a body start taken from the same logical bytes.
/// </remarks>
internal static bool TryCreateStuffedHeader(
ReadOnlySpan<byte> artData,
int bodyStart,
[NotNullWhen(true)] out byte[]? stuffedHeader)
{
stuffedHeader = null;
if ((uint)bodyStart > (uint)artData.Length)
{
return false;
}

var header = artData[..bodyStart];
var extra = CountDotLeadingLines(header);
if (extra == 0)
{
return false;
}

stuffedHeader = GC.AllocateUninitializedArray<byte>(header.Length + extra);
WriteDotStuffedLines(header, stuffedHeader);
return true;
}

/// <summary>Counts CRLF lines in <paramref name="region"/> whose first byte is <c>.</c>.</summary>
/// <param name="region">Header region, including the separator.</param>
/// <returns>The number of stuffing dots a header write inserts.</returns>
private static int CountDotLeadingLines(ReadOnlySpan<byte> region)
{
var extra = 0;
var index = 0;
while (index < region.Length)
{
if (region[index] == (byte)'.')
{
extra++;
}

var crlf = region[index..].IndexOf("\r\n"u8);
if (crlf < 0)
{
break;
}

index += crlf + 2;
}

return extra;
}

/// <summary>Writes <paramref name="region"/> with one extra leading dot on each line that begins with <c>.</c>.</summary>
/// <param name="region">Header region whose lines end in CRLF.</param>
/// <param name="destination">Buffer of <paramref name="region"/> length plus the stuffing count.</param>
private static void WriteDotStuffedLines(ReadOnlySpan<byte> region, Span<byte> destination)
{
var written = 0;
var index = 0;
while (index < region.Length)
{
if (region[index] == (byte)'.')
{
destination[written++] = (byte)'.';
}

var crlf = region[index..].IndexOf("\r\n"u8);
var lineLength = crlf + 2;
region.Slice(index, lineLength).CopyTo(destination[written..]);
written += lineLength;
index += lineLength;
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,27 @@ namespace VectorNNTPD.StorageService.Configuration
{

/// <summary>
/// Dedicated segment-pool directories and used-space thresholds for a future eviction loop.
/// <see cref="LowPercent"/>, <see cref="HighPercent"/>, and <see cref="CriticalPercent"/> do not
/// admit or reject article writes. Advertisements are not changed by these thresholds.
/// Dedicated segment-pool directories and filesystem used-space thresholds for capacity policy.
/// Each configured directory has an independent High→Low eviction monitor.
/// <see cref="CriticalPercent"/> excludes a spool from placement (and blocks TAKETHIS when no
/// eligible spool remains). Advertisements are not changed by these thresholds.
/// </summary>
public sealed class SegmentPoolsOptions
{
/// <summary>
/// Highest used-space percent reserved for a future eviction loop.
/// It does not make a spool ineligible for a new write and does not change the RabbitMQ advertisement.
/// Occupancy percent above which a spool is ineligible for new placement (strict greater-than).
/// Equal to this value remains eligible. Global TAKETHIS admission requires at least one
/// valid, fresh, eligible spool. Does not change RabbitMQ advertisements.
/// </summary>
public int CriticalPercent { get; set; } = 85;

/// <summary>
/// Occupancy percent at or above which a future eviction loop would start. Not enforced in this phase.
/// Occupancy percent at or above which that spool's independent LFU eviction monitor starts.
/// </summary>
public int HighPercent { get; set; } = 79;

/// <summary>
/// Occupancy percent at which a future eviction loop would stop. Not enforced in this phase.
/// Occupancy percent at or below which that spool's independent LFU eviction monitor stops.
/// </summary>
public int LowPercent { get; set; } = 75;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ namespace VectorNNTPD.StorageService.Configuration
/// <para>
/// At 10 Gbps the payload rate is 1.25 GB/s. An average 750 KiB article
/// (<c>750 × 1024 = 768_000</c> bytes) is about 1,628 articles/s. The default
/// 1 GiB queue holds about 1,398 such articles. STORE admission reserves that
/// article's META <c>ArtSize</c> before WINDOW credit is granted, so a full queue
/// does not receive the body. <c>TryCommit</c> moves the reservation into the queue
/// without adding the bytes twice. A dequeued article leaves the budget.
/// 1 GiB queue holds about 1,398 such articles. Admission reserves <c>ArtSize</c>
/// before <c>239</c>. <c>TryCommit</c> moves the reservation into the queue without
/// adding the bytes twice. A dequeued article stays in the budget as in-flight until
/// the worker calls <c>ReleaseProcessed</c> (capacity-blocked holds included), so
/// sustained critical storage cannot refill the queue past <see cref="MaxQueuedBytes"/>.
/// TAKETHIS may assemble a body before reservation; concurrent assemblies are capped by
/// listener <c>MaxActiveConnections</c> and are outside this byte budget until reserved.
/// Path rewrite keeps a rented second buffer until publish returns, then returns it
/// to a pool capped at eight arrays per size. Live memory is still the original plus
/// that buffer. The pool retains at most eight arrays of each size that has been used.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,13 @@ public sealed class StorageServerListenerOptions
/// </summary>
public int IoProgressTimeoutSeconds { get; set; } = 60;

/// <summary>
/// Overall deadline in seconds for receiving one TAKETHIS article (headers and body through
/// the multiline terminator). Unlike <see cref="IoProgressTimeoutSeconds"/>, this does not
/// reset when the peer drips another byte. Default is 300 seconds.
/// </summary>
public int ArticleReceiveTimeoutSeconds { get; set; } = 300;

/// <summary>Maximum concurrently active accepted connections.</summary>
public int MaxActiveConnections { get; set; } = 1024;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,11 @@ private static void ValidateListener(StorageServerOptions options, List<string>
failures.Add("StorageServer:Listener:IoProgressTimeoutSeconds must be between 1 and 600.");
}

if (listener.ArticleReceiveTimeoutSeconds is < 30 or > 3600)
{
failures.Add("StorageServer:Listener:ArticleReceiveTimeoutSeconds must be between 30 and 3600.");
}

if (listener.MaxActiveConnections < 1)
{
failures.Add("StorageServer:Listener:MaxActiveConnections must be between 1 and 2147483647.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,6 @@ public sealed record StorageServerListenerRuntimeOptions(
int ParserAccumulationMaxBytes,
TimeSpan TlsHandshakeTimeout,
TimeSpan IoProgressTimeout,
int MaxActiveConnections);
int MaxActiveConnections,
TimeSpan ArticleReceiveTimeout);
}
Loading
Loading