From af0952999118a5cdcfd49d80cbcb95f8d87f820e Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Tue, 20 Oct 2026 06:40:51 +0200 Subject: [PATCH 1/2] Fix body dot-stuffing for bare CR line endings Recognize CR, LF, and CRLF as line boundaries when locating dot-leading body lines, matching parser line accounting. Add regression coverage for bare CR, LF, and repeated CR without changing the exclusive-CRLF delivery requirement. --- docs/common/ArticleRecord.md | 2 +- .../Processing/ArticleBodyWireStuffer.cs | 45 +++++-- .../Processing/ArticleBodyWireStufferTests.cs | 111 +++++++++++++++++- 3 files changed, 143 insertions(+), 15 deletions(-) diff --git a/docs/common/ArticleRecord.md b/docs/common/ArticleRecord.md index 3c15b3f7..ba30fed3 100644 --- a/docs/common/ArticleRecord.md +++ b/docs/common/ArticleRecord.md @@ -214,7 +214,7 @@ Semantics (tests: `Parse_BodyLineCount_MatchesParserLineWalk`): Terminators: CR, LF, or CRLF (`FindLineTerminator` / `AdvancePastTerminator`). The last unterminated fragment counts as one line. -Materialization copies body lines and inserts one leading `.` on each line whose first content byte is `.`. That does not add a line, so the destuffed body line count stays valid for canonical ArtData. Overview `:lines` can use `ArtLines` without another body scan. `:bytes` is the stuffed `ArtSize`. +Materialization copies body lines and inserts one leading `.` on each line whose first content byte is `.`. `ArticleBodyWireStuffer` uses the same CR, LF, and CRLF line ends as that walk, so a bare CR before a dot-leading line is stuffed. That does not add a line, so the destuffed body line count stays valid for canonical ArtData. Overview `:lines` can use `ArtLines` without another body scan. `:bytes` is the stuffed `ArtSize`. Delivery framing still requires exclusive CRLF; a retained article that used bare CR or bare LF is not frameable. --- diff --git a/src/VectorNNTPD.Common/Articles/Processing/ArticleBodyWireStuffer.cs b/src/VectorNNTPD.Common/Articles/Processing/ArticleBodyWireStuffer.cs index 89122f70..db3a4de2 100644 --- a/src/VectorNNTPD.Common/Articles/Processing/ArticleBodyWireStuffer.cs +++ b/src/VectorNNTPD.Common/Articles/Processing/ArticleBodyWireStuffer.cs @@ -12,9 +12,11 @@ namespace VectorNNTPD.Common.Articles.Processing /// /// /// Headers, including a header line that begins with ., and the header/body separator are copied unchanged. - /// A body line whose first content byte is . gains one leading .. Content excludes a CR that - /// immediately precedes LF. The NNTP multiline terminator .CRLF is not appended. + /// A body line whose first content byte is . gains one leading .. Line ends are CR, LF, or CRLF, + /// matching the parser's body walk. A CR that immediately precedes LF is one terminator, not two. + /// The NNTP multiline terminator .CRLF is not appended. /// When no body line needs a dot, the original array is returned. + /// Delivery framing still requires exclusive CRLF and may reject a retained article that used bare CR or bare LF. /// /// internal static class ArticleBodyWireStuffer @@ -102,7 +104,7 @@ internal static int CountExtraDots(ReadOnlySpan body) } extra++; - SkipThroughLineFeed(body, ref index); + SkipThroughLineTerminator(body, ref index); } return extra; @@ -156,7 +158,7 @@ internal static int WriteStuffed(ReadOnlySpan body, Span destination destination[written++] = (byte)'.'; var start = index; - SkipThroughLineFeed(body, ref index); + SkipThroughLineTerminator(body, ref index); var line = index - start; body.Slice(start, line).CopyTo(destination[written..]); written += line; @@ -183,23 +185,40 @@ private static void SkipPlainLines(ReadOnlySpan body, ref int index) { while (index < body.Length && body[index] != (byte)'.') { - SkipThroughLineFeed(body, ref index); + SkipThroughLineTerminator(body, ref index); } } - /// Advances to the first byte after this line's LF, or to the buffer length. + /// + /// Advances to the first byte after this line's CR, LF, or CRLF, or to the buffer length. + /// /// Logical body. /// A byte inside the current line on entry. - private static void SkipThroughLineFeed(ReadOnlySpan body, ref int index) + /// + /// Matches the parser's AdvancePastTerminator rules so a bare CR ends a line for stuffing. + /// A CR immediately followed by LF is consumed as one terminator. + /// + private static void SkipThroughLineTerminator(ReadOnlySpan body, ref int index) { - while (index < body.Length && body[index] != (byte)'\n') + while (index < body.Length) { - index++; - } + var b = body[index++]; + if (b == (byte)'\n') + { + return; + } - if (index < body.Length) - { - index++; + if (b != (byte)'\r') + { + continue; + } + + if (index < body.Length && body[index] == (byte)'\n') + { + index++; + } + + return; } } } diff --git a/tests/VectorNNTPD.Common.Tests/Articles/Processing/ArticleBodyWireStufferTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/Processing/ArticleBodyWireStufferTests.cs index 1c08b31d..fd278baf 100644 --- a/tests/VectorNNTPD.Common.Tests/Articles/Processing/ArticleBodyWireStufferTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Articles/Processing/ArticleBodyWireStufferTests.cs @@ -1,3 +1,5 @@ +using VectorNNTPD.Common.Articles; +using VectorNNTPD.Common.Articles.Parsing; using VectorNNTPD.Common.Articles.Processing; using System.Text; @@ -31,7 +33,7 @@ public void Header_dot_is_not_stuffed_and_an_unstuffed_body_is_the_same_array() Assert.Same(logical, wire); } - /// Parameterized body fragments keep the one-dot-per-line-start rule across CRLF, LF, and mixed endings. + /// Parameterized body fragments keep the one-dot-per-line-start rule across CRLF, LF, CR, and mixed endings. /// Logical body bytes after the header separator. /// Expected stuffed body including any added dots. [Theory] @@ -46,6 +48,11 @@ public void Header_dot_is_not_stuffed_and_an_unstuffed_body_is_the_same_array() [InlineData("...\r\n", "....\r\n")] [InlineData("a.b\r\n", "a.b\r\n")] [InlineData("plain\r\n.\r\n..x\n...y\r\n.last", "plain\r\n..\r\n...x\n....y\r\n..last")] + [InlineData("hello\r\n.secret\r\n", "hello\r\n..secret\r\n")] + [InlineData("hello\r.secret\n", "hello\r..secret\n")] + [InlineData("hello\r\r.secret\r\n", "hello\r\r..secret\r\n")] + [InlineData("hello\n.secret\r\n", "hello\n..secret\r\n")] + [InlineData(".\r\n.\n.\r", "..\r\n..\n..\r")] public void Logical_body_lines_keep_one_stuffed_dot_at_the_line_start(string body, string stuffed) { var logical = Encoding.ASCII.GetBytes("From: a@b.test\r\n\r\n" + body); @@ -54,6 +61,49 @@ public void Logical_body_lines_keep_one_stuffed_dot_at_the_line_start(string bod Assert.Equal(Encoding.ASCII.GetBytes("From: a@b.test\r\n\r\n" + stuffed), wire); } + /// Issue #168: a bare CR ends the preceding line so the following dot-leading line is stuffed. + [Fact] + public void Issue168_BareCrBeforeDotLeadingLine_IsStuffed() + { + AssertStuffedBody("hello\r.secret\n", "hello\r..secret\n"); + } + + /// A CRLF before a dot-leading line still stuffs exactly once. + [Fact] + public void DotFollowingCrlf_IsStuffed() + { + AssertStuffedBody("hello\r\n.secret\r\n", "hello\r\n..secret\r\n"); + } + + /// A bare LF before a dot-leading line still stuffs exactly once. + [Fact] + public void DotFollowingBareLf_IsStuffed() + { + AssertStuffedBody("hello\n.secret\r\n", "hello\n..secret\r\n"); + } + + /// Consecutive bare CR bytes each end a line, so a later dot-leading line is stuffed. + [Fact] + public void ConsecutiveBareCr_BeforeDotLeadingLine_IsStuffed() + { + AssertStuffedBody("hello\r\r.secret\r\n", "hello\r\r..secret\r\n"); + } + + /// A body that begins with a dot is stuffed, including consecutive dot-leading lines. + [Fact] + public void BodyBeginningWithDot_AndConsecutiveDotLines_AreStuffed() + { + AssertStuffedBody(".\r\n.second\r\n", "..\r\n..second\r\n"); + } + + /// An empty body and a CRLF-only body need no stuffing and keep their exact bytes. + [Fact] + public void EmptyAndCrlfOnlyBodies_RemainUnchanged() + { + AssertStuffedBody(string.Empty, string.Empty); + AssertStuffedBody("\r\n", "\r\n"); + } + /// LF header separators and mixed line endings affect only the body region after the first blank line. [Fact] public void Lf_separator_and_mixed_endings_stuff_only_the_body() @@ -63,5 +113,64 @@ public void Lf_separator_and_mixed_endings_stuff_only_the_body() Assert.Equal(Encoding.ASCII.GetBytes("From: a@b.test\n\n..\n...x\r\n"), wire); } + + /// + /// Materializing the Issue #168 bare-CR body stores the stuffed image and remains not frameable for NNTP delivery. + /// + [Fact] + public void Materialize_WhenBodyUsesBareCrBeforeDot_StoresStuffedBytesAndIsNotFrameable() + { + var parser = new NntpArticleParser("bf01.usenet.ninja"); + var article = Encoding.ASCII.GetBytes( + "Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n" + + "Message-ID: \r\n" + + "Newsgroups: alt.test\r\n" + + "From: user@example.test\r\n" + + "\r\n" + + "hello\r.secret\n"); + + var created = ArticleRecordFactory.TryCreate(parser, article); + + Assert.True(created.IsAccepted, created.ParseFailure.ToString()); + Assert.Equal( + Encoding.ASCII.GetBytes("hello\r..secret\n"), + created.Record.ArtData.Span[created.Record.Framing.BodyStart..].ToArray()); + Assert.False(created.Record.Framing.IsFrameable); + Assert.False(CanonicalArticleFraming.Analyze(created.Record.ArtData.Span).IsFrameable); + } + + /// + /// A CRLF body with a dot-leading line is stuffed and remains frameable. + /// + [Fact] + public void Materialize_WhenBodyUsesCrlfBeforeDot_StoresStuffedBytesAndIsFrameable() + { + var parser = new NntpArticleParser("bf01.usenet.ninja"); + var article = Encoding.ASCII.GetBytes( + "Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n" + + "Message-ID: \r\n" + + "Newsgroups: alt.test\r\n" + + "From: user@example.test\r\n" + + "\r\n" + + "hello\r\n.secret\r\n"); + + var created = ArticleRecordFactory.TryCreate(parser, article); + + Assert.True(created.IsAccepted, created.ParseFailure.ToString()); + Assert.Equal( + Encoding.ASCII.GetBytes("hello\r\n..secret\r\n"), + created.Record.ArtData.Span[created.Record.Framing.BodyStart..].ToArray()); + Assert.True(created.Record.Framing.IsFrameable); + } + + /// Asserts stuffing of a body fragment after a fixed CRLF header separator. + /// Logical body. + /// Expected stuffed body. + private static void AssertStuffedBody(string body, string stuffed) + { + var logical = Encoding.ASCII.GetBytes("From: a@b.test\r\n\r\n" + body); + var wire = ArticleBodyWireStuffer.StuffBody(logical); + Assert.Equal(Encoding.ASCII.GetBytes("From: a@b.test\r\n\r\n" + stuffed), wire); + } } } From 16604bc9ec392dd59db0a2a1f6b10a3ab9508877 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Tue, 20 Oct 2026 06:51:54 +0200 Subject: [PATCH 2/2] Isolate replenish cancellation shutdown regression test Use responsive servers in the never-firing replenish timer test so it measures cancellation independently of stalled QUIT writes. Retain explicit timer-armed synchronization and bounded cleanup. --- .../Nntp/NntpShutdownDeadlineTests.cs | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs index a5aadd3b..2ab96224 100644 --- a/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs +++ b/tests/VectorNNTPD.BackFillerService.Tests/Nntp/NntpShutdownDeadlineTests.cs @@ -788,33 +788,54 @@ public async Task Responsive_replacement_still_sends_quit() /// /// A replenish delay that never fires does not hold replacement after the shutdown token is cancelled. /// + /// + /// Responsive servers keep this case on the replenish-timer invariant alone. + /// Stalled QUIT and shared-budget coverage stay in + /// and the other replacement drain tests. + /// Completing under is well below the fifteen-second replenish interval. + /// [Fact] public async Task Replacement_does_not_wait_for_a_replenish_delay_that_ignores_its_timer() { const int sessions = 2; var factory = new ScriptedNntpTransportFactory(); - var servers = EnqueueStalledQuitServers(factory, sessions); + var servers = EnqueueReadyServers(factory, sessions); var clock = new NeverFiringTimeProvider(); var pool = NntpSessionPoolTests.CreatePool( factory, max: sessions, time: clock, shutdownGrace: SharedBudget); + var dispose = Task.CompletedTask; try { await pool.EnsureDesiredSessionsAsync(CancellationToken.None); await WaitUntilAsync(() => clock.TimersCreated >= 1); - var elapsed = await MeasureAsync(() => pool.DrainAndDisposeAsync(CancellationToken.None), servers); - Report("replacement-replenish", SharedBudget, sessions, providers: 1, QuitStarts(servers), elapsed); + var started = Stopwatch.GetTimestamp(); + dispose = pool.DrainAndDisposeAsync(CancellationToken.None); + await dispose.WaitAsync(SharedShutdownCeiling); + var elapsed = Stopwatch.GetElapsedTime(started); + Report("replacement-replenish", SharedBudget, sessions, providers: 1, quitStarted: 0, elapsed); AssertWithinCeiling(elapsed); + Assert.True( + elapsed < TimeSpan.FromSeconds(15), + $"elapsed {elapsed.TotalMilliseconds:0} ms must stay below the replenish interval"); Assert.All(servers, static server => Assert.Equal(1, server.ClientStreamDisposeCount)); Assert.Equal(0, pool.LiveSessionCount); } finally { - Release(servers); + try + { + await dispose.WaitAsync(TimeSpan.FromSeconds(5)); + } + catch (TimeoutException) + { + // Observed so an abandoned drain cannot outlive the test. The original failure stands. + } + await pool.DisposeAsync(); } }