diff --git a/src/VectorNNTPD.Common/Articles/Processing/NntpArticleCanonicalMaterializer.cs b/src/VectorNNTPD.Common/Articles/Processing/NntpArticleCanonicalMaterializer.cs index cff65672..1397bf00 100644 --- a/src/VectorNNTPD.Common/Articles/Processing/NntpArticleCanonicalMaterializer.cs +++ b/src/VectorNNTPD.Common/Articles/Processing/NntpArticleCanonicalMaterializer.cs @@ -4,6 +4,49 @@ namespace VectorNNTPD.Common.Articles.Processing { + /// + /// How treats the body of an accepted parse result. + /// + /// + /// Three representations must not be confused: + /// + /// + /// + /// NNTP transport stuffing (wire): an extra leading . on body lines during multiline transfer. + /// Removed by the protocol reader before any materialization. + /// + /// + /// + /// + /// Destuffed article content: logical body after transport destuff. First ingest materialization + /// uses . + /// + /// + /// + /// + /// CanonicalV1 storage body: after one storage stuffing pass (one leading . on each body + /// line whose first content byte is .). Queued ArtData and spool bytes use this form. + /// Rematerializing Path/Date on that form requires . + /// + /// + /// + /// Choosing the wrong mode silently corrupts dot-leading body lines (under-stuff or double-stuff). + /// + internal enum CanonicalBodyMode + { + /// + /// Source body is destuffed article content (not CanonicalV1 yet). + /// Apply exactly one CanonicalV1 storage stuffing pass. + /// + StuffDestuffedBody = 0, + + /// + /// Source body is already CanonicalV1 storage-stuffed (for example queued ArtData). + /// Copy the body unchanged. Do not use for destuffed ingress bytes or NNTP wire bytes. + /// + PreserveCanonicalBody = 1, + } + /// /// Classifies why canonical Date/Path materialization could not produce a retainable article. /// @@ -146,9 +189,10 @@ internal static NntpArticleCanonicalMaterializeResult Accepted(byte[] articleByt /// /// /// Rewrites only the selected date-header value and the Path header (or inserts Path). - /// Other header bytes are copied exactly. Body lines whose first content byte is . gain one - /// leading dot through . yEnc is not decoded. - /// The NNTP terminator .CRLF is not stored. + /// Other header bytes are copied exactly. By default, body lines whose first content byte is . + /// gain one leading dot through when the source body is destuffed. + /// copies an already-stuffed CanonicalV1 body unchanged. + /// yEnc is not decoded. The NNTP terminator .CRLF is not stored. /// internal static class NntpArticleCanonicalMaterializer { @@ -188,13 +232,20 @@ internal static NntpArticleCanonicalMaterializeResult Materialize( /// The caller returns it to this same pool after every consumer has finished. /// When null, the result owns a new exact-size array. /// + /// + /// (default): source body is destuffed article content; + /// apply one CanonicalV1 storage stuffing pass. Preserves historical ingest behaviour. + /// : source body is already CanonicalV1; + /// copy it unchanged while rewriting Path/Date. Wrong mode silently corrupts dot-leading lines. + /// /// A result that owns one exact-size canonical buffer on success, or a rented buffer, or a failure classification. internal static NntpArticleCanonicalMaterializeResult Materialize( in NntpArticleParseResult parseResult, ArticlePathMode pathMode, int maxArticleBytes, ReadOnlySpan organizationalTrackerHost, - ArrayPool? rentFrom = null) + ArrayPool? rentFrom = null, + CanonicalBodyMode bodyMode = CanonicalBodyMode.StuffDestuffedBody) { if (!parseResult.IsAccepted) { @@ -294,7 +345,11 @@ internal static NntpArticleCanonicalMaterializeResult Materialize( int finalLength; try { - extraDots = ArticleBodyWireStuffer.CountExtraDots(source[bodyStart..]); + // Queued CanonicalV1 ArtData is already storage-stuffed. Re-stuffing it would corrupt + // every body line that begins with '.' (for example "..leading" becomes "...leading"). + extraDots = bodyMode == CanonicalBodyMode.PreserveCanonicalBody + ? 0 + : ArticleBodyWireStuffer.CountExtraDots(source[bodyStart..]); finalLength = checked(destinationLength + extraDots); } catch (OverflowException) @@ -350,6 +405,8 @@ internal static NntpArticleCanonicalMaterializeResult Materialize( written += headerTail; } + // PreserveCanonicalBody already forced extraDots to 0 above, so a body copy covers + // both "no dots to stuff" and "already CanonicalV1; do not restuff". if (extraDots == 0) { source[bodyStart..].CopyTo(dest[written..]); diff --git a/src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs b/src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs index ad85155d..af32291c 100644 --- a/src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs +++ b/src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs @@ -118,7 +118,13 @@ internal bool TryRewrite(AcceptedArticle article, out ReadOnlyMemory rewri return false; } - var materialize = NntpArticleCanonicalMaterializer.Materialize(in parse, ArticlePathMode.Traverse); + var materialize = NntpArticleCanonicalMaterializer.Materialize( + in parse, + ArticlePathMode.Traverse, + ArticleResourceLimits.MaxArticleBytes, + ArticlePathCanonicalizer.OrganizationalTrackerHost, + rentFrom: null, + CanonicalBodyMode.PreserveCanonicalBody); if (!materialize.IsAccepted || materialize.ArticleBytes is null) { AcceptedArticleWorkerLogMessages.PathRejected( @@ -168,12 +174,14 @@ internal bool TryRewriteRented(AcceptedArticle article, out byte[]? buffer, out return false; } + // ArtData is already CanonicalV1 from ingest Normalize; only Path (and Date) must change. var materialize = NntpArticleCanonicalMaterializer.Materialize( in parse, ArticlePathMode.Traverse, ArticleResourceLimits.MaxArticleBytes, ArticlePathCanonicalizer.OrganizationalTrackerHost, - RewritePool); + RewritePool, + CanonicalBodyMode.PreserveCanonicalBody); if (!materialize.IsAccepted || materialize.ArticleBytes is null) { AcceptedArticleWorkerLogMessages.PathRejected( diff --git a/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs index 89a081ad..7a3d70b8 100644 --- a/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Articles/Processing/NntpArticleCanonicalMaterializerTests.cs @@ -20,6 +20,50 @@ public sealed class NntpArticleCanonicalMaterializerTests /// private const string OrganizationalTrackerPrefix = "news.usenet.ninja!"; + /// + /// Rematerializing CanonicalV1 ArtData with + /// must not add another stuffing layer to dot-leading body lines. + /// + [Fact] + public void Materialize_PreserveCanonicalBody_DoesNotRestuffAlreadyStuffedBody() + { + var parser = new NntpArticleParser(LocalFqdn); + var destuffed = BuildArticle( + [ + "Path: peer.example", + "Date: Fri, 23 Aug 2024 07:30:10 +0000", + "Message-ID: ", + "Newsgroups: alt.test", + "From: user@example.test", + "Subject: dots", + ], + "Hello\r\n.leading\r\n..leading\r\n"); + var first = ArticleRecordFactory.TryCreate(parser, destuffed, ArticlePathMode.Normalize); + Assert.True(first.IsAccepted); + var stuffedBody = Encoding.ASCII.GetString(GetBodyBytes(first.Record.ArtData.Span)); + Assert.Equal("Hello\r\n..leading\r\n...leading\r\n", stuffedBody); + + var parse = parser.Parse(first.Record.ArtData); + Assert.True(parse.IsAccepted); + var restuffed = NntpArticleCanonicalMaterializer.Materialize(in parse, ArticlePathMode.Traverse); + Assert.True(restuffed.IsAccepted); + Assert.Equal( + "Hello\r\n...leading\r\n....leading\r\n", + Encoding.ASCII.GetString(GetBodyBytes(restuffed.ArticleBytes.AsSpan(0, restuffed.CanonicalLength)))); + + var preserved = NntpArticleCanonicalMaterializer.Materialize( + in parse, + ArticlePathMode.Traverse, + ArticleResourceLimits.MaxArticleBytes, + ArticlePathCanonicalizer.OrganizationalTrackerHost, + rentFrom: null, + CanonicalBodyMode.PreserveCanonicalBody); + Assert.True(preserved.IsAccepted); + Assert.Equal( + stuffedBody, + Encoding.ASCII.GetString(GetBodyBytes(preserved.ArticleBytes.AsSpan(0, preserved.CanonicalLength)))); + } + /// /// A missing Path is inserted as the tracker plus the local identity, the Date is rewritten to canonical UTC, and the body is unchanged. /// diff --git a/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs b/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs index 17d476e8..5a97c76b 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs @@ -47,7 +47,8 @@ public async Task Worker_applies_the_path_hop_and_discards_the_rewritten_article [Fact] public async Task Worker_prepends_the_hop_again_when_it_is_already_present() { - var inbound = Build("", "kept-body\r\n", CacheFqdn); + // Queued ArtData already carries this server's hop (e.g. loop/reinject). Normalize keeps it. + var inbound = Build("", "kept-body\r\n", pathAlreadyHasCacheHop: true); var inboundText = Encoding.ASCII.GetString(inbound.Record.ArtData.Span); Assert.Equal(1, CountHop(inboundText, CacheFqdn)); var queue = new AcceptedArticleQueue(); @@ -133,18 +134,14 @@ public async Task Worker_continues_after_a_rejected_article_and_a_thrown_article } [Fact] - public void Path_rewrite_matches_the_canonical_factory_bytes() + public void Path_rewrite_preserves_canonical_body_and_prepends_cache_hop() { var inbound = Build("", "body-line\r\nsecond\r\n"); var processor = new AcceptedArticlePathProcessor(CacheFqdn, NullLogger.Instance); Assert.True(processor.TryRewrite(ToQueued(inbound), out var rewritten)); - var created = ArticleRecordFactory.TryCreate( - new NntpArticleParser(CacheFqdn), - inbound.Record.ArtData, - ArticlePathMode.Traverse); - Assert.True(created.IsAccepted); - Assert.True(created.Record.ArtData.Span.SequenceEqual(rewritten.Span)); - Assert.Equal(inbound.Record.ArtId, created.Record.ArtId); + Assert.Equal(inbound.Record.ArtId, ArticleId.FromMessageId(MessageIdValue(rewritten.Span))); + Assert.Equal(Body(Encoding.ASCII.GetString(inbound.Record.ArtData.Span)), Body(Encoding.ASCII.GetString(rewritten.Span))); + Assert.StartsWith($"Path: {CacheFqdn}!", HeaderLine(rewritten.Span, "Path"), StringComparison.Ordinal); var wrong = new AcceptedArticle( ArticleId.FromMessageId(""u8), @@ -154,6 +151,31 @@ public void Path_rewrite_matches_the_canonical_factory_bytes() Assert.False(processor.TryRewrite(wrong, out _)); } + [Fact] + public void Path_rewrite_does_not_double_stuff_dot_leading_body_lines() + { + // Destuffed logical body (post-TAKETHIS unstuff). Normalize produces CanonicalV1 stuffed ArtData. + var inbound = Build( + "", + "Hello\r\n.leading\r\n..leading\r\n...leading\r\n"); + var queuedBody = Body(Encoding.ASCII.GetString(inbound.Record.ArtData.Span)); + Assert.Equal("Hello\r\n..leading\r\n...leading\r\n....leading\r\n", queuedBody); + + var processor = new AcceptedArticlePathProcessor(CacheFqdn, NullLogger.Instance); + Assert.True(processor.TryRewriteRented(ToQueued(inbound), out var rented, out var length)); + try + { + var rewritten = rented.AsSpan(0, length); + Assert.Equal(queuedBody, Body(Encoding.ASCII.GetString(rewritten))); + Assert.StartsWith($"Path: {CacheFqdn}!", HeaderLine(rewritten, "Path"), StringComparison.Ordinal); + Assert.DoesNotContain(".....\r\n", Encoding.ASCII.GetString(rewritten), StringComparison.Ordinal); + } + finally + { + AcceptedArticlePathProcessor.ReturnRewriteBuffer(rented!); + } + } + [Fact] public void Rented_path_rewrite_matches_the_exact_buffer_and_is_returned() { @@ -213,11 +235,33 @@ private static string Body(string article) return article[(split + 4)..]; } - private static Built Build(string messageId, string body, string ingressFqdn = "nntpd01.usenet.ninja") + private static string HeaderLine(ReadOnlySpan article, string name) + { + var text = Encoding.ASCII.GetString(article); + var prefix = name + ": "; + var start = text.IndexOf(prefix, StringComparison.Ordinal); + Assert.True(start >= 0, name); + var end = text.IndexOf("\r\n", start, StringComparison.Ordinal); + Assert.True(end > start); + return text[start..end]; + } + + private static ReadOnlySpan MessageIdValue(ReadOnlySpan article) { - var parser = new NntpArticleParser(ingressFqdn); + var line = HeaderLine(article, "Message-ID"); + Assert.StartsWith("Message-ID: ", line, StringComparison.Ordinal); + return Encoding.ASCII.GetBytes(line["Message-ID: ".Length..]); + } + + private static Built Build(string messageId, string body, bool pathAlreadyHasCacheHop = false) + { + // Match StorageNntpFeedSession: parser identity is the storage FQDN; ingest uses Normalize. + var parser = new NntpArticleParser(CacheFqdn); + var path = pathAlreadyHasCacheHop + ? CacheFqdn + "!peer.example" + : "peer.example"; var text = new StringBuilder() - .Append("Path: peer.example\r\n") + .Append("Path: ").Append(path).Append("\r\n") .Append("Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n") .Append("Message-ID: ").Append(messageId).Append("\r\n") .Append("Newsgroups: alt.test\r\n") @@ -225,7 +269,10 @@ private static Built Build(string messageId, string body, string ingressFqdn = " .Append("Subject: vatp\r\n") .Append("\r\n") .Append(body); - var created = ArticleRecordFactory.TryCreate(parser, Encoding.ASCII.GetBytes(text.ToString())); + var created = ArticleRecordFactory.TryCreate( + parser, + Encoding.ASCII.GetBytes(text.ToString()), + ArticlePathMode.Normalize); Assert.True(created.IsAccepted, created.MaterializeFailure.ToString()); return new Built(created.Record); } diff --git a/tests/VectorNNTPD.StorageService.Tests/Listener/StorageNntpQueueIngressTests.cs b/tests/VectorNNTPD.StorageService.Tests/Listener/StorageNntpQueueIngressTests.cs index c3129338..256d2bad 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Listener/StorageNntpQueueIngressTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Listener/StorageNntpQueueIngressTests.cs @@ -7,6 +7,9 @@ using VectorNNTPD.StorageService.Ingress; using VectorNNTPD.StorageService.Listener; using VectorNNTPD.StorageService; +using VectorNNTPD.StorageService.Storage.SegmentPools; +using VectorNNTPD.StorageService.Storage.SpoolPersistence; +using VectorNNTPD.StorageService.Storage.SpoolShards; namespace VectorNNTPD.StorageService.Tests.Listener { @@ -213,6 +216,90 @@ public async Task Dot_stuffed_article_enqueues_with_matching_art_id() Assert.Contains(".middle\r\n", Encoding.ASCII.GetString(delivery.Article.ArtData.Span), StringComparison.Ordinal); } + [Fact] + public async Task TakeThis_dot_bodies_persist_canonical_body_with_storage_path_hop() + { + // Wire TAKETHIS (transport-stuffed) → queue CanonicalV1 → worker Traverse → spool file. + var inbound = Build( + "", + "Hello\r\n.leading\r\n..leading\r\n...leading\r\n"); + var expectedBody = Body(Encoding.ASCII.GetString(inbound.Record.ArtData.Span)); + Assert.Equal("Hello\r\n..leading\r\n...leading\r\n....leading\r\n", expectedBody); + + var root = Path.Join(Path.GetTempPath(), "vectornntp-wire-spool-" + Guid.NewGuid().ToString("n")); + Directory.CreateDirectory(root); + var queue = new AcceptedArticleQueue(); + AcceptedArticleWorker? worker = null; + BlockDeviceMonitorService? monitor = null; + try + { + var probe = new StaticFreeSpaceProbe(); + probe.Set(root, total: 100_000_000_000, available: 80_000_000_000); + var catalog = new SegmentPoolCatalog(); + catalog.Publish([new SegmentPoolDevice("block:a", [root])]); + monitor = new BlockDeviceMonitorService( + catalog, + new RisingCounters(), + NullLogger.Instance, + TimeSpan.FromMilliseconds(20), + TimeProvider.System, + probe); + await monitor.StartAsync(CancellationToken.None); + await WaitUntilAsync(() => monitor.Observations.Space.ContainsKey(root)); + + var processor = new SpoolArticlePersistenceProcessor( + CacheFqdn, + catalog, + monitor, + new SpoolWriteGate(), + new SpoolArticleIdentityGate(), + new SpoolHealth(TimeProvider.System), + new SpoolPressure(), + new SpoolArticlePublisher(new FileSpoolArticleFiles()), + new SpoolPersistenceCounters(), + new StorageServerOptions { SpoolPersistence = new SpoolPersistenceOptions() }, + NullLogger.Instance, + TimeProvider.System); + worker = new AcceptedArticleWorker( + queue, + processor, + NullLogger.Instance, + new SpoolPersistenceOptions { MinWorkers = 1, MaxWorkers = 1 }, + new SpoolPressure(), + new SpoolPersistenceCounters(), + TimeProvider.System); + await worker.StartAsync(CancellationToken.None); + + await using var harness = await Harness.StartAsync(queue); + Assert.Equal(239, await harness.TakeThisAsync(inbound).WaitAsync(Safety)); + + var spoolPath = SpoolArticlePath.CombineArticle(root, inbound.Record.ArtId); + await WaitUntilAsync(() => File.Exists(spoolPath)); + var stored = Encoding.ASCII.GetString(File.ReadAllBytes(spoolPath)); + Assert.StartsWith($"Path: {CacheFqdn}!", HeaderLine(stored, "Path"), StringComparison.Ordinal); + Assert.Equal(expectedBody, Body(stored)); + Assert.DoesNotContain(".....\r\n", stored, StringComparison.Ordinal); + } + finally + { + if (worker is not null) + { + queue.Complete(); + await worker.StopAsync(CancellationToken.None); + } + + if (monitor is not null) + { + await monitor.StopAsync(CancellationToken.None); + } + + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } + } + [Fact] public async Task Pipelined_second_takeThis_with_large_residue_still_enqueues() { @@ -291,6 +378,16 @@ private static string Body(string article) return article[(split + 4)..]; } + private static string HeaderLine(string article, string name) + { + var prefix = name + ": "; + var start = article.IndexOf(prefix, StringComparison.Ordinal); + Assert.True(start >= 0, name); + var end = article.IndexOf("\r\n", start, StringComparison.Ordinal); + Assert.True(end > start); + return article[start..end]; + } + private static byte[] EncodeTakeThis(Built built) { var messageId = built.Record.MessageId.ToArray(); @@ -526,5 +623,39 @@ public ValueTask DisposeAsync() return ValueTask.CompletedTask; } } + + private sealed class StaticFreeSpaceProbe : ISpoolFreeSpaceProbe + { + private readonly Dictionary _space = new(StringComparer.Ordinal); + + public void Set(string path, long total, long available) => _space[path] = (total, available); + + public bool TryRead(string spoolRoot, out long totalBytes, out long availableBytes) + { + if (!_space.TryGetValue(spoolRoot, out var sample)) + { + totalBytes = 0; + availableBytes = 0; + return false; + } + + totalBytes = sample.Total; + availableBytes = sample.Available; + return true; + } + } + + private sealed class RisingCounters : IBlockDeviceCounterSource + { + private ulong _value = 1000; + + public bool TryRead(string deviceId, out BlockDeviceCounters counters, out string? error) + { + _value += 1000; + counters = new BlockDeviceCounters(_value, _value, _value, _value, _value, _value, LatencyAvailable: true); + error = null; + return true; + } + } } } diff --git a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs index 9bd0f5d7..79ca4b5b 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs @@ -499,8 +499,15 @@ public void Rewrite_component_times_are_reported() }); var materializeTicks = Time(count, i => { + // ArtData is already CanonicalV1; match the worker Path rewrite body mode. var parsed = parser.Parse(articles[i].ArtData); - var materialized = NntpArticleCanonicalMaterializer.Materialize(in parsed, ArticlePathMode.Traverse); + var materialized = NntpArticleCanonicalMaterializer.Materialize( + in parsed, + ArticlePathMode.Traverse, + ArticleResourceLimits.MaxArticleBytes, + ArticlePathCanonicalizer.OrganizationalTrackerHost, + rentFrom: null, + CanonicalBodyMode.PreserveCanonicalBody); Assert.True(materialized.IsAccepted); }); var hashTicks = Time(count, i => diff --git a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs index ef574982..adb55264 100644 --- a/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs +++ b/tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs @@ -14,6 +14,86 @@ namespace VectorNNTPD.StorageService.Tests.Storage public sealed class SpoolPersistenceIntegrationTests { + [Fact] + public async Task Processor_preserves_canonical_dot_stuffed_body_through_path_traverse_and_spool_write() + { + var root = NewRoot(); + try + { + var probe = new StaticFreeSpaceProbe(); + probe.Set(root, total: 100_000_000_000, available: 80_000_000_000); + var catalog = new SegmentPoolCatalog(); + catalog.Publish([new SegmentPoolDevice("block:a", [root])]); + var monitor = new BlockDeviceMonitorService( + catalog, + new RisingCounters(), + NullLogger.Instance, + TimeSpan.FromMilliseconds(20), + TimeProvider.System, + probe); + await monitor.StartAsync(CancellationToken.None); + try + { + await WaitAsync(() => monitor.Observations.Space.ContainsKey(root)); + var processor = new SpoolArticlePersistenceProcessor( + "cache01.usenet.ninja", + catalog, + monitor, + new SpoolWriteGate(), + new SpoolArticleIdentityGate(), + new SpoolHealth(TimeProvider.System), + new SpoolPressure(), + new SpoolArticlePublisher(new FileSpoolArticleFiles()), + new SpoolPersistenceCounters(), + new StorageServerOptions { SpoolPersistence = new SpoolPersistenceOptions() }, + NullLogger.Instance, + TimeProvider.System); + + // Logical destuffed body (after TAKETHIS unstuff). Normalize → CanonicalV1 stuffed ArtData. + var destuffed = + "Path: peer.example\r\n" + + "Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n" + + "Message-ID: \r\n" + + "Newsgroups: alt.test\r\n" + + "From: user@example.test\r\n" + + "Subject: dots\r\n" + + "\r\n" + + "Hello\r\n" + + ".leading\r\n" + + "..leading\r\n" + + "...leading\r\n"; + var created = ArticleRecordFactory.TryCreate( + new NntpArticleParser("cache01.usenet.ninja"), + Encoding.ASCII.GetBytes(destuffed), + ArticlePathMode.Normalize); + Assert.True(created.IsAccepted, created.MaterializeFailure.ToString()); + var queuedBody = BodyOf(created.Record.ArtData.Span); + Assert.Equal("Hello\r\n..leading\r\n...leading\r\n....leading\r\n", queuedBody); + + var article = new AcceptedArticle( + created.Record.ArtId, + created.Record.ArtHash, + created.Record.ArtSize, + created.Record.ArtData); + Assert.Equal(AcceptedArticleProcessResult.Completed, await processor.ProcessAsync(article, CancellationToken.None)); + + var path = SpoolArticlePath.CombineArticle(root, article.ArtId); + var stored = Encoding.ASCII.GetString(File.ReadAllBytes(path)); + Assert.StartsWith("Path: cache01.usenet.ninja!", HeaderLine(stored, "Path"), StringComparison.Ordinal); + Assert.Equal(queuedBody, BodyOf(Encoding.ASCII.GetBytes(stored))); + Assert.DoesNotContain(".....\r\n", stored, StringComparison.Ordinal); + } + finally + { + await monitor.StopAsync(CancellationToken.None); + } + } + finally + { + Directory.Delete(root, recursive: true); + } + } + [Fact] public async Task Processor_writes_the_rewritten_article_to_the_roomier_spool() { @@ -471,11 +551,29 @@ private static AcceptedArticle Build(string messageId) var text = "Path: peer.example\r\nDate: Fri, 23 Aug 2024 07:30:10 +0000\r\nMessage-ID: " + messageId + "\r\nNewsgroups: alt.test\r\nFrom: user@example.test\r\nSubject: vatp\r\n\r\nbody-line\r\n"; - var created = ArticleRecordFactory.TryCreate(parser, Encoding.ASCII.GetBytes(text)); + var created = ArticleRecordFactory.TryCreate(parser, Encoding.ASCII.GetBytes(text), ArticlePathMode.Normalize); Assert.True(created.IsAccepted, created.MaterializeFailure.ToString()); return new AcceptedArticle(created.Record.ArtId, created.Record.ArtHash, created.Record.ArtSize, created.Record.ArtData); } + private static string BodyOf(ReadOnlySpan article) + { + var text = Encoding.ASCII.GetString(article); + var split = text.IndexOf("\r\n\r\n", StringComparison.Ordinal); + Assert.True(split >= 0); + return text[(split + 4)..]; + } + + private static string HeaderLine(string article, string name) + { + var prefix = name + ": "; + var start = article.IndexOf(prefix, StringComparison.Ordinal); + Assert.True(start >= 0, name); + var end = article.IndexOf("\r\n", start, StringComparison.Ordinal); + Assert.True(end > start); + return article[start..end]; + } + private static async Task WaitAsync(Func condition) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(2);