diff --git a/src/VectorNNTPD.Common/Articles/Processing/NntpArticleCanonicalMaterializer.cs b/src/VectorNNTPD.Common/Articles/Processing/NntpArticleCanonicalMaterializer.cs
index cff65672..1397bf00 100644
--- a/src/VectorNNTPD.Common/Articles/Processing/NntpArticleCanonicalMaterializer.cs
+++ b/src/VectorNNTPD.Common/Articles/Processing/NntpArticleCanonicalMaterializer.cs
@@ -4,6 +4,49 @@
namespace VectorNNTPD.Common.Articles.Processing
{
+ ///
+ /// How treats the body of an accepted parse result.
+ ///
+ ///
+ /// Three representations must not be confused:
+ ///
+ /// -
+ ///
+ /// NNTP transport stuffing (wire): an extra leading . on body lines during multiline transfer.
+ /// Removed by the protocol reader before any materialization.
+ ///
+ ///
+ /// -
+ ///
+ /// Destuffed article content: logical body after transport destuff. First ingest materialization
+ /// uses .
+ ///
+ ///
+ /// -
+ ///
+ /// CanonicalV1 storage body: after one storage stuffing pass (one leading . on each body
+ /// line whose first content byte is .). Queued ArtData and spool bytes use this form.
+ /// Rematerializing Path/Date on that form requires .
+ ///
+ ///
+ ///
+ /// Choosing the wrong mode silently corrupts dot-leading body lines (under-stuff or double-stuff).
+ ///
+ internal enum CanonicalBodyMode
+ {
+ ///
+ /// Source body is destuffed article content (not CanonicalV1 yet).
+ /// Apply exactly one CanonicalV1 storage stuffing pass.
+ ///
+ StuffDestuffedBody = 0,
+
+ ///
+ /// Source body is already CanonicalV1 storage-stuffed (for example queued ArtData).
+ /// Copy the body unchanged. Do not use for destuffed ingress bytes or NNTP wire bytes.
+ ///
+ PreserveCanonicalBody = 1,
+ }
+
///
/// Classifies why canonical Date/Path materialization could not produce a retainable article.
///
@@ -146,9 +189,10 @@ internal static NntpArticleCanonicalMaterializeResult Accepted(byte[] articleByt
///
///
/// Rewrites only the selected date-header value and the Path header (or inserts Path).
- /// Other header bytes are copied exactly. Body lines whose first content byte is . gain one
- /// leading dot through . yEnc is not decoded.
- /// The NNTP terminator .CRLF is not stored.
+ /// Other header bytes are copied exactly. By default, body lines whose first content byte is .
+ /// gain one leading dot through when the source body is destuffed.
+ /// copies an already-stuffed CanonicalV1 body unchanged.
+ /// yEnc is not decoded. The NNTP terminator .CRLF is not stored.
///
internal static class NntpArticleCanonicalMaterializer
{
@@ -188,13 +232,20 @@ internal static NntpArticleCanonicalMaterializeResult Materialize(
/// The caller returns it to this same pool after every consumer has finished.
/// When null, the result owns a new exact-size array.
///
+ ///
+ /// (default): source body is destuffed article content;
+ /// apply one CanonicalV1 storage stuffing pass. Preserves historical ingest behaviour.
+ /// : source body is already CanonicalV1;
+ /// copy it unchanged while rewriting Path/Date. Wrong mode silently corrupts dot-leading lines.
+ ///
/// A result that owns one exact-size canonical buffer on success, or a rented buffer, or a failure classification.
internal static NntpArticleCanonicalMaterializeResult Materialize(
in NntpArticleParseResult parseResult,
ArticlePathMode pathMode,
int maxArticleBytes,
ReadOnlySpan organizationalTrackerHost,
- ArrayPool? rentFrom = null)
+ ArrayPool? rentFrom = null,
+ CanonicalBodyMode bodyMode = CanonicalBodyMode.StuffDestuffedBody)
{
if (!parseResult.IsAccepted)
{
@@ -294,7 +345,11 @@ internal static NntpArticleCanonicalMaterializeResult Materialize(
int finalLength;
try
{
- extraDots = ArticleBodyWireStuffer.CountExtraDots(source[bodyStart..]);
+ // Queued CanonicalV1 ArtData is already storage-stuffed. Re-stuffing it would corrupt
+ // every body line that begins with '.' (for example "..leading" becomes "...leading").
+ extraDots = bodyMode == CanonicalBodyMode.PreserveCanonicalBody
+ ? 0
+ : ArticleBodyWireStuffer.CountExtraDots(source[bodyStart..]);
finalLength = checked(destinationLength + extraDots);
}
catch (OverflowException)
@@ -350,6 +405,8 @@ internal static NntpArticleCanonicalMaterializeResult Materialize(
written += headerTail;
}
+ // PreserveCanonicalBody already forced extraDots to 0 above, so a body copy covers
+ // both "no dots to stuff" and "already CanonicalV1; do not restuff".
if (extraDots == 0)
{
source[bodyStart..].CopyTo(dest[written..]);
diff --git a/src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs b/src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs
index ad85155d..af32291c 100644
--- a/src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs
+++ b/src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs
@@ -118,7 +118,13 @@ internal bool TryRewrite(AcceptedArticle article, out ReadOnlyMemory rewri
return false;
}
- var materialize = NntpArticleCanonicalMaterializer.Materialize(in parse, ArticlePathMode.Traverse);
+ var materialize = NntpArticleCanonicalMaterializer.Materialize(
+ in parse,
+ ArticlePathMode.Traverse,
+ ArticleResourceLimits.MaxArticleBytes,
+ ArticlePathCanonicalizer.OrganizationalTrackerHost,
+ rentFrom: null,
+ CanonicalBodyMode.PreserveCanonicalBody);
if (!materialize.IsAccepted || materialize.ArticleBytes is null)
{
AcceptedArticleWorkerLogMessages.PathRejected(
@@ -168,12 +174,14 @@ internal bool TryRewriteRented(AcceptedArticle article, out byte[]? buffer, out
return false;
}
+ // ArtData is already CanonicalV1 from ingest Normalize; only Path (and Date) must change.
var materialize = NntpArticleCanonicalMaterializer.Materialize(
in parse,
ArticlePathMode.Traverse,
ArticleResourceLimits.MaxArticleBytes,
ArticlePathCanonicalizer.OrganizationalTrackerHost,
- RewritePool);
+ RewritePool,
+ CanonicalBodyMode.PreserveCanonicalBody);
if (!materialize.IsAccepted || materialize.ArticleBytes is null)
{
AcceptedArticleWorkerLogMessages.PathRejected(
diff --git a/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs
index 89a081ad..7a3d70b8 100644
--- a/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs
+++ b/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs
@@ -20,6 +20,50 @@ public sealed class NntpArticleCanonicalMaterializerTests
///
private const string OrganizationalTrackerPrefix = "news.usenet.ninja!";
+ ///
+ /// Rematerializing CanonicalV1 ArtData with
+ /// must not add another stuffing layer to dot-leading body lines.
+ ///
+ [Fact]
+ public void Materialize_PreserveCanonicalBody_DoesNotRestuffAlreadyStuffedBody()
+ {
+ var parser = new NntpArticleParser(LocalFqdn);
+ var destuffed = BuildArticle(
+ [
+ "Path: peer.example",
+ "Date: Fri, 23 Aug 2024 07:30:10 +0000",
+ "Message-ID: ",
+ "Newsgroups: alt.test",
+ "From: user@example.test",
+ "Subject: dots",
+ ],
+ "Hello\r\n.leading\r\n..leading\r\n");
+ var first = ArticleRecordFactory.TryCreate(parser, destuffed, ArticlePathMode.Normalize);
+ Assert.True(first.IsAccepted);
+ var stuffedBody = Encoding.ASCII.GetString(GetBodyBytes(first.Record.ArtData.Span));
+ Assert.Equal("Hello\r\n..leading\r\n...leading\r\n", stuffedBody);
+
+ var parse = parser.Parse(first.Record.ArtData);
+ Assert.True(parse.IsAccepted);
+ var restuffed = NntpArticleCanonicalMaterializer.Materialize(in parse, ArticlePathMode.Traverse);
+ Assert.True(restuffed.IsAccepted);
+ Assert.Equal(
+ "Hello\r\n...leading\r\n....leading\r\n",
+ Encoding.ASCII.GetString(GetBodyBytes(restuffed.ArticleBytes.AsSpan(0, restuffed.CanonicalLength))));
+
+ var preserved = NntpArticleCanonicalMaterializer.Materialize(
+ in parse,
+ ArticlePathMode.Traverse,
+ ArticleResourceLimits.MaxArticleBytes,
+ ArticlePathCanonicalizer.OrganizationalTrackerHost,
+ rentFrom: null,
+ CanonicalBodyMode.PreserveCanonicalBody);
+ Assert.True(preserved.IsAccepted);
+ Assert.Equal(
+ stuffedBody,
+ Encoding.ASCII.GetString(GetBodyBytes(preserved.ArticleBytes.AsSpan(0, preserved.CanonicalLength))));
+ }
+
///
/// A missing Path is inserted as the tracker plus the local identity, the Date is rewritten to canonical UTC, and the body is unchanged.
///
diff --git a/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs b/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs
index 17d476e8..5a97c76b 100644
--- a/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs
+++ b/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs
@@ -47,7 +47,8 @@ public async Task Worker_applies_the_path_hop_and_discards_the_rewritten_article
[Fact]
public async Task Worker_prepends_the_hop_again_when_it_is_already_present()
{
- var inbound = Build("", "kept-body\r\n", CacheFqdn);
+ // Queued ArtData already carries this server's hop (e.g. loop/reinject). Normalize keeps it.
+ var inbound = Build("", "kept-body\r\n", pathAlreadyHasCacheHop: true);
var inboundText = Encoding.ASCII.GetString(inbound.Record.ArtData.Span);
Assert.Equal(1, CountHop(inboundText, CacheFqdn));
var queue = new AcceptedArticleQueue();
@@ -133,18 +134,14 @@ public async Task Worker_continues_after_a_rejected_article_and_a_thrown_article
}
[Fact]
- public void Path_rewrite_matches_the_canonical_factory_bytes()
+ public void Path_rewrite_preserves_canonical_body_and_prepends_cache_hop()
{
var inbound = Build("", "body-line\r\nsecond\r\n");
var processor = new AcceptedArticlePathProcessor(CacheFqdn, NullLogger.Instance);
Assert.True(processor.TryRewrite(ToQueued(inbound), out var rewritten));
- var created = ArticleRecordFactory.TryCreate(
- new NntpArticleParser(CacheFqdn),
- inbound.Record.ArtData,
- ArticlePathMode.Traverse);
- Assert.True(created.IsAccepted);
- Assert.True(created.Record.ArtData.Span.SequenceEqual(rewritten.Span));
- Assert.Equal(inbound.Record.ArtId, created.Record.ArtId);
+ Assert.Equal(inbound.Record.ArtId, ArticleId.FromMessageId(MessageIdValue(rewritten.Span)));
+ Assert.Equal(Body(Encoding.ASCII.GetString(inbound.Record.ArtData.Span)), Body(Encoding.ASCII.GetString(rewritten.Span)));
+ Assert.StartsWith($"Path: {CacheFqdn}!", HeaderLine(rewritten.Span, "Path"), StringComparison.Ordinal);
var wrong = new AcceptedArticle(
ArticleId.FromMessageId(""u8),
@@ -154,6 +151,31 @@ public void Path_rewrite_matches_the_canonical_factory_bytes()
Assert.False(processor.TryRewrite(wrong, out _));
}
+ [Fact]
+ public void Path_rewrite_does_not_double_stuff_dot_leading_body_lines()
+ {
+ // Destuffed logical body (post-TAKETHIS unstuff). Normalize produces CanonicalV1 stuffed ArtData.
+ var inbound = Build(
+ "",
+ "Hello\r\n.leading\r\n..leading\r\n...leading\r\n");
+ var queuedBody = Body(Encoding.ASCII.GetString(inbound.Record.ArtData.Span));
+ Assert.Equal("Hello\r\n..leading\r\n...leading\r\n....leading\r\n", queuedBody);
+
+ var processor = new AcceptedArticlePathProcessor(CacheFqdn, NullLogger.Instance);
+ Assert.True(processor.TryRewriteRented(ToQueued(inbound), out var rented, out var length));
+ try
+ {
+ var rewritten = rented.AsSpan(0, length);
+ Assert.Equal(queuedBody, Body(Encoding.ASCII.GetString(rewritten)));
+ Assert.StartsWith($"Path: {CacheFqdn}!", HeaderLine(rewritten, "Path"), StringComparison.Ordinal);
+ Assert.DoesNotContain(".....\r\n", Encoding.ASCII.GetString(rewritten), StringComparison.Ordinal);
+ }
+ finally
+ {
+ AcceptedArticlePathProcessor.ReturnRewriteBuffer(rented!);
+ }
+ }
+
[Fact]
public void Rented_path_rewrite_matches_the_exact_buffer_and_is_returned()
{
@@ -213,11 +235,33 @@ private static string Body(string article)
return article[(split + 4)..];
}
- private static Built Build(string messageId, string body, string ingressFqdn = "nntpd01.usenet.ninja")
+ private static string HeaderLine(ReadOnlySpan article, string name)
+ {
+ var text = Encoding.ASCII.GetString(article);
+ var prefix = name + ": ";
+ var start = text.IndexOf(prefix, StringComparison.Ordinal);
+ Assert.True(start >= 0, name);
+ var end = text.IndexOf("\r\n", start, StringComparison.Ordinal);
+ Assert.True(end > start);
+ return text[start..end];
+ }
+
+ private static ReadOnlySpan MessageIdValue(ReadOnlySpan article)
{
- var parser = new NntpArticleParser(ingressFqdn);
+ var line = HeaderLine(article, "Message-ID");
+ Assert.StartsWith("Message-ID: ", line, StringComparison.Ordinal);
+ return Encoding.ASCII.GetBytes(line["Message-ID: ".Length..]);
+ }
+
+ private static Built Build(string messageId, string body, bool pathAlreadyHasCacheHop = false)
+ {
+ // Match StorageNntpFeedSession: parser identity is the storage FQDN; ingest uses Normalize.
+ var parser = new NntpArticleParser(CacheFqdn);
+ var path = pathAlreadyHasCacheHop
+ ? CacheFqdn + "!peer.example"
+ : "peer.example";
var text = new StringBuilder()
- .Append("Path: peer.example\r\n")
+ .Append("Path: ").Append(path).Append("\r\n")
.Append("Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n")
.Append("Message-ID: ").Append(messageId).Append("\r\n")
.Append("Newsgroups: alt.test\r\n")
@@ -225,7 +269,10 @@ private static Built Build(string messageId, string body, string ingressFqdn = "
.Append("Subject: vatp\r\n")
.Append("\r\n")
.Append(body);
- var created = ArticleRecordFactory.TryCreate(parser, Encoding.ASCII.GetBytes(text.ToString()));
+ var created = ArticleRecordFactory.TryCreate(
+ parser,
+ Encoding.ASCII.GetBytes(text.ToString()),
+ ArticlePathMode.Normalize);
Assert.True(created.IsAccepted, created.MaterializeFailure.ToString());
return new Built(created.Record);
}
diff --git a/tests/VectorNNTPD.StorageService.Tests/Listener/StorageNntpQueueIngressTests.cs b/tests/VectorNNTPD.StorageService.Tests/Listener/StorageNntpQueueIngressTests.cs
index c3129338..256d2bad 100644
--- a/tests/VectorNNTPD.StorageService.Tests/Listener/StorageNntpQueueIngressTests.cs
+++ b/tests/VectorNNTPD.StorageService.Tests/Listener/StorageNntpQueueIngressTests.cs
@@ -7,6 +7,9 @@
using VectorNNTPD.StorageService.Ingress;
using VectorNNTPD.StorageService.Listener;
using VectorNNTPD.StorageService;
+using VectorNNTPD.StorageService.Storage.SegmentPools;
+using VectorNNTPD.StorageService.Storage.SpoolPersistence;
+using VectorNNTPD.StorageService.Storage.SpoolShards;
namespace VectorNNTPD.StorageService.Tests.Listener
{
@@ -213,6 +216,90 @@ public async Task Dot_stuffed_article_enqueues_with_matching_art_id()
Assert.Contains(".middle\r\n", Encoding.ASCII.GetString(delivery.Article.ArtData.Span), StringComparison.Ordinal);
}
+ [Fact]
+ public async Task TakeThis_dot_bodies_persist_canonical_body_with_storage_path_hop()
+ {
+ // Wire TAKETHIS (transport-stuffed) → queue CanonicalV1 → worker Traverse → spool file.
+ var inbound = Build(
+ "",
+ "Hello\r\n.leading\r\n..leading\r\n...leading\r\n");
+ var expectedBody = Body(Encoding.ASCII.GetString(inbound.Record.ArtData.Span));
+ Assert.Equal("Hello\r\n..leading\r\n...leading\r\n....leading\r\n", expectedBody);
+
+ var root = Path.Join(Path.GetTempPath(), "vectornntp-wire-spool-" + Guid.NewGuid().ToString("n"));
+ Directory.CreateDirectory(root);
+ var queue = new AcceptedArticleQueue();
+ AcceptedArticleWorker? worker = null;
+ BlockDeviceMonitorService? monitor = null;
+ try
+ {
+ var probe = new StaticFreeSpaceProbe();
+ probe.Set(root, total: 100_000_000_000, available: 80_000_000_000);
+ var catalog = new SegmentPoolCatalog();
+ catalog.Publish([new SegmentPoolDevice("block:a", [root])]);
+ monitor = new BlockDeviceMonitorService(
+ catalog,
+ new RisingCounters(),
+ NullLogger.Instance,
+ TimeSpan.FromMilliseconds(20),
+ TimeProvider.System,
+ probe);
+ await monitor.StartAsync(CancellationToken.None);
+ await WaitUntilAsync(() => monitor.Observations.Space.ContainsKey(root));
+
+ var processor = new SpoolArticlePersistenceProcessor(
+ CacheFqdn,
+ catalog,
+ monitor,
+ new SpoolWriteGate(),
+ new SpoolArticleIdentityGate(),
+ new SpoolHealth(TimeProvider.System),
+ new SpoolPressure(),
+ new SpoolArticlePublisher(new FileSpoolArticleFiles()),
+ new SpoolPersistenceCounters(),
+ new StorageServerOptions { SpoolPersistence = new SpoolPersistenceOptions() },
+ NullLogger.Instance,
+ TimeProvider.System);
+ worker = new AcceptedArticleWorker(
+ queue,
+ processor,
+ NullLogger.Instance,
+ new SpoolPersistenceOptions { MinWorkers = 1, MaxWorkers = 1 },
+ new SpoolPressure(),
+ new SpoolPersistenceCounters(),
+ TimeProvider.System);
+ await worker.StartAsync(CancellationToken.None);
+
+ await using var harness = await Harness.StartAsync(queue);
+ Assert.Equal(239, await harness.TakeThisAsync(inbound).WaitAsync(Safety));
+
+ var spoolPath = SpoolArticlePath.CombineArticle(root, inbound.Record.ArtId);
+ await WaitUntilAsync(() => File.Exists(spoolPath));
+ var stored = Encoding.ASCII.GetString(File.ReadAllBytes(spoolPath));
+ Assert.StartsWith($"Path: {CacheFqdn}!", HeaderLine(stored, "Path"), StringComparison.Ordinal);
+ Assert.Equal(expectedBody, Body(stored));
+ Assert.DoesNotContain(".....\r\n", stored, StringComparison.Ordinal);
+ }
+ finally
+ {
+ if (worker is not null)
+ {
+ queue.Complete();
+ await worker.StopAsync(CancellationToken.None);
+ }
+
+ if (monitor is not null)
+ {
+ await monitor.StopAsync(CancellationToken.None);
+ }
+
+ if (Directory.Exists(root))
+ {
+ Directory.Delete(root, recursive: true);
+ }
+ }
+ }
+
[Fact]
public async Task Pipelined_second_takeThis_with_large_residue_still_enqueues()
{
@@ -291,6 +378,16 @@ private static string Body(string article)
return article[(split + 4)..];
}
+ private static string HeaderLine(string article, string name)
+ {
+ var prefix = name + ": ";
+ var start = article.IndexOf(prefix, StringComparison.Ordinal);
+ Assert.True(start >= 0, name);
+ var end = article.IndexOf("\r\n", start, StringComparison.Ordinal);
+ Assert.True(end > start);
+ return article[start..end];
+ }
+
private static byte[] EncodeTakeThis(Built built)
{
var messageId = built.Record.MessageId.ToArray();
@@ -526,5 +623,39 @@ public ValueTask DisposeAsync()
return ValueTask.CompletedTask;
}
}
+
+ private sealed class StaticFreeSpaceProbe : ISpoolFreeSpaceProbe
+ {
+ private readonly Dictionary _space = new(StringComparer.Ordinal);
+
+ public void Set(string path, long total, long available) => _space[path] = (total, available);
+
+ public bool TryRead(string spoolRoot, out long totalBytes, out long availableBytes)
+ {
+ if (!_space.TryGetValue(spoolRoot, out var sample))
+ {
+ totalBytes = 0;
+ availableBytes = 0;
+ return false;
+ }
+
+ totalBytes = sample.Total;
+ availableBytes = sample.Available;
+ return true;
+ }
+ }
+
+ private sealed class RisingCounters : IBlockDeviceCounterSource
+ {
+ private ulong _value = 1000;
+
+ public bool TryRead(string deviceId, out BlockDeviceCounters counters, out string? error)
+ {
+ _value += 1000;
+ counters = new BlockDeviceCounters(_value, _value, _value, _value, _value, _value, LatencyAvailable: true);
+ error = null;
+ return true;
+ }
+ }
}
}
diff --git a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs
index 9bd0f5d7..79ca4b5b 100644
--- a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs
+++ b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs
@@ -499,8 +499,15 @@ public void Rewrite_component_times_are_reported()
});
var materializeTicks = Time(count, i =>
{
+ // ArtData is already CanonicalV1; match the worker Path rewrite body mode.
var parsed = parser.Parse(articles[i].ArtData);
- var materialized = NntpArticleCanonicalMaterializer.Materialize(in parsed, ArticlePathMode.Traverse);
+ var materialized = NntpArticleCanonicalMaterializer.Materialize(
+ in parsed,
+ ArticlePathMode.Traverse,
+ ArticleResourceLimits.MaxArticleBytes,
+ ArticlePathCanonicalizer.OrganizationalTrackerHost,
+ rentFrom: null,
+ CanonicalBodyMode.PreserveCanonicalBody);
Assert.True(materialized.IsAccepted);
});
var hashTicks = Time(count, i =>
diff --git a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs
index ef574982..adb55264 100644
--- a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs
+++ b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs
@@ -14,6 +14,86 @@ namespace VectorNNTPD.StorageService.Tests.Storage
public sealed class SpoolPersistenceIntegrationTests
{
+ [Fact]
+ public async Task Processor_preserves_canonical_dot_stuffed_body_through_path_traverse_and_spool_write()
+ {
+ var root = NewRoot();
+ try
+ {
+ var probe = new StaticFreeSpaceProbe();
+ probe.Set(root, total: 100_000_000_000, available: 80_000_000_000);
+ var catalog = new SegmentPoolCatalog();
+ catalog.Publish([new SegmentPoolDevice("block:a", [root])]);
+ var monitor = new BlockDeviceMonitorService(
+ catalog,
+ new RisingCounters(),
+ NullLogger.Instance,
+ TimeSpan.FromMilliseconds(20),
+ TimeProvider.System,
+ probe);
+ await monitor.StartAsync(CancellationToken.None);
+ try
+ {
+ await WaitAsync(() => monitor.Observations.Space.ContainsKey(root));
+ var processor = new SpoolArticlePersistenceProcessor(
+ "cache01.usenet.ninja",
+ catalog,
+ monitor,
+ new SpoolWriteGate(),
+ new SpoolArticleIdentityGate(),
+ new SpoolHealth(TimeProvider.System),
+ new SpoolPressure(),
+ new SpoolArticlePublisher(new FileSpoolArticleFiles()),
+ new SpoolPersistenceCounters(),
+ new StorageServerOptions { SpoolPersistence = new SpoolPersistenceOptions() },
+ NullLogger.Instance,
+ TimeProvider.System);
+
+ // Logical destuffed body (after TAKETHIS unstuff). Normalize → CanonicalV1 stuffed ArtData.
+ var destuffed =
+ "Path: peer.example\r\n" +
+ "Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n" +
+ "Message-ID: \r\n" +
+ "Newsgroups: alt.test\r\n" +
+ "From: user@example.test\r\n" +
+ "Subject: dots\r\n" +
+ "\r\n" +
+ "Hello\r\n" +
+ ".leading\r\n" +
+ "..leading\r\n" +
+ "...leading\r\n";
+ var created = ArticleRecordFactory.TryCreate(
+ new NntpArticleParser("cache01.usenet.ninja"),
+ Encoding.ASCII.GetBytes(destuffed),
+ ArticlePathMode.Normalize);
+ Assert.True(created.IsAccepted, created.MaterializeFailure.ToString());
+ var queuedBody = BodyOf(created.Record.ArtData.Span);
+ Assert.Equal("Hello\r\n..leading\r\n...leading\r\n....leading\r\n", queuedBody);
+
+ var article = new AcceptedArticle(
+ created.Record.ArtId,
+ created.Record.ArtHash,
+ created.Record.ArtSize,
+ created.Record.ArtData);
+ Assert.Equal(AcceptedArticleProcessResult.Completed, await processor.ProcessAsync(article, CancellationToken.None));
+
+ var path = SpoolArticlePath.CombineArticle(root, article.ArtId);
+ var stored = Encoding.ASCII.GetString(File.ReadAllBytes(path));
+ Assert.StartsWith("Path: cache01.usenet.ninja!", HeaderLine(stored, "Path"), StringComparison.Ordinal);
+ Assert.Equal(queuedBody, BodyOf(Encoding.ASCII.GetBytes(stored)));
+ Assert.DoesNotContain(".....\r\n", stored, StringComparison.Ordinal);
+ }
+ finally
+ {
+ await monitor.StopAsync(CancellationToken.None);
+ }
+ }
+ finally
+ {
+ Directory.Delete(root, recursive: true);
+ }
+ }
+
[Fact]
public async Task Processor_writes_the_rewritten_article_to_the_roomier_spool()
{
@@ -471,11 +551,29 @@ private static AcceptedArticle Build(string messageId)
var text = "Path: peer.example\r\nDate: Fri, 23 Aug 2024 07:30:10 +0000\r\nMessage-ID: "
+ messageId
+ "\r\nNewsgroups: alt.test\r\nFrom: user@example.test\r\nSubject: vatp\r\n\r\nbody-line\r\n";
- var created = ArticleRecordFactory.TryCreate(parser, Encoding.ASCII.GetBytes(text));
+ var created = ArticleRecordFactory.TryCreate(parser, Encoding.ASCII.GetBytes(text), ArticlePathMode.Normalize);
Assert.True(created.IsAccepted, created.MaterializeFailure.ToString());
return new AcceptedArticle(created.Record.ArtId, created.Record.ArtHash, created.Record.ArtSize, created.Record.ArtData);
}
+ private static string BodyOf(ReadOnlySpan article)
+ {
+ var text = Encoding.ASCII.GetString(article);
+ var split = text.IndexOf("\r\n\r\n", StringComparison.Ordinal);
+ Assert.True(split >= 0);
+ return text[(split + 4)..];
+ }
+
+ private static string HeaderLine(string article, string name)
+ {
+ var prefix = name + ": ";
+ var start = article.IndexOf(prefix, StringComparison.Ordinal);
+ Assert.True(start >= 0, name);
+ var end = article.IndexOf("\r\n", start, StringComparison.Ordinal);
+ Assert.True(end > start);
+ return article[start..end];
+ }
+
private static async Task WaitAsync(Func condition)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(2);