Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,49 @@

namespace VectorNNTPD.Common.Articles.Processing
{
/// <summary>
/// How <see cref="NntpArticleCanonicalMaterializer"/> treats the body of an accepted parse result.
/// </summary>
/// <remarks>
/// Three representations must not be confused:
/// <list type="bullet">
/// <item>
/// <description>
/// NNTP transport stuffing (wire): an extra leading <c>.</c> on body lines during multiline transfer.
/// Removed by the protocol reader before any materialization.
/// </description>
/// </item>
/// <item>
/// <description>
/// Destuffed article content: logical body after transport destuff. First ingest materialization
/// uses <see cref="StuffDestuffedBody"/>.
/// </description>
/// </item>
/// <item>
/// <description>
/// CanonicalV1 storage body: after one storage stuffing pass (one leading <c>.</c> on each body
/// line whose first content byte is <c>.</c>). Queued <c>ArtData</c> and spool bytes use this form.
/// Rematerializing Path/Date on that form requires <see cref="PreserveCanonicalBody"/>.
/// </description>
/// </item>
/// </list>
/// Choosing the wrong mode silently corrupts dot-leading body lines (under-stuff or double-stuff).
/// </remarks>
internal enum CanonicalBodyMode
{
/// <summary>
/// Source body is destuffed article content (not CanonicalV1 yet).
/// Apply exactly one CanonicalV1 storage stuffing pass.
/// </summary>
StuffDestuffedBody = 0,

/// <summary>
/// Source body is already CanonicalV1 storage-stuffed (for example queued <c>ArtData</c>).
/// Copy the body unchanged. Do not use for destuffed ingress bytes or NNTP wire bytes.
/// </summary>
PreserveCanonicalBody = 1,
}

/// <summary>
/// Classifies why canonical Date/Path materialization could not produce a retainable article.
/// </summary>
Expand Down Expand Up @@ -146,9 +189,10 @@ internal static NntpArticleCanonicalMaterializeResult Accepted(byte[] articleByt
/// </summary>
/// <remarks>
/// Rewrites only the selected date-header value and the Path header (or inserts Path).
/// Other header bytes are copied exactly. Body lines whose first content byte is <c>.</c> gain one
/// leading dot through <see cref="ArticleBodyWireStuffer"/>. yEnc is not decoded.
/// The NNTP terminator <c>.CRLF</c> is not stored.
/// Other header bytes are copied exactly. By default, body lines whose first content byte is <c>.</c>
/// gain one leading dot through <see cref="ArticleBodyWireStuffer"/> when the source body is destuffed.
/// <see cref="CanonicalBodyMode.PreserveCanonicalBody"/> copies an already-stuffed CanonicalV1 body unchanged.
/// yEnc is not decoded. The NNTP terminator <c>.CRLF</c> is not stored.
/// </remarks>
internal static class NntpArticleCanonicalMaterializer
{
Expand Down Expand Up @@ -188,13 +232,20 @@ internal static NntpArticleCanonicalMaterializeResult Materialize(
/// The caller returns it to this same pool after every consumer has finished.
/// When null, the result owns a new exact-size array.
/// </param>
/// <param name="bodyMode">
/// <see cref="CanonicalBodyMode.StuffDestuffedBody"/> (default): source body is destuffed article content;
/// apply one CanonicalV1 storage stuffing pass. Preserves historical ingest behaviour.
/// <see cref="CanonicalBodyMode.PreserveCanonicalBody"/>: source body is already CanonicalV1;
/// copy it unchanged while rewriting Path/Date. Wrong mode silently corrupts dot-leading lines.
/// </param>
/// <returns>A result that owns one exact-size canonical buffer on success, or a rented buffer, or a failure classification.</returns>
internal static NntpArticleCanonicalMaterializeResult Materialize(
in NntpArticleParseResult parseResult,
ArticlePathMode pathMode,
int maxArticleBytes,
ReadOnlySpan<byte> organizationalTrackerHost,
ArrayPool<byte>? rentFrom = null)
ArrayPool<byte>? rentFrom = null,
CanonicalBodyMode bodyMode = CanonicalBodyMode.StuffDestuffedBody)
{
if (!parseResult.IsAccepted)
{
Expand Down Expand Up @@ -294,7 +345,11 @@ internal static NntpArticleCanonicalMaterializeResult Materialize(
int finalLength;
try
{
extraDots = ArticleBodyWireStuffer.CountExtraDots(source[bodyStart..]);
// Queued CanonicalV1 ArtData is already storage-stuffed. Re-stuffing it would corrupt
// every body line that begins with '.' (for example "..leading" becomes "...leading").
extraDots = bodyMode == CanonicalBodyMode.PreserveCanonicalBody
? 0
: ArticleBodyWireStuffer.CountExtraDots(source[bodyStart..]);
finalLength = checked(destinationLength + extraDots);
}
catch (OverflowException)
Expand Down Expand Up @@ -350,6 +405,8 @@ internal static NntpArticleCanonicalMaterializeResult Materialize(
written += headerTail;
}

// PreserveCanonicalBody already forced extraDots to 0 above, so a body copy covers
// both "no dots to stuff" and "already CanonicalV1; do not restuff".
if (extraDots == 0)
{
source[bodyStart..].CopyTo(dest[written..]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,13 @@ internal bool TryRewrite(AcceptedArticle article, out ReadOnlyMemory<byte> rewri
return false;
}

var materialize = NntpArticleCanonicalMaterializer.Materialize(in parse, ArticlePathMode.Traverse);
var materialize = NntpArticleCanonicalMaterializer.Materialize(
in parse,
ArticlePathMode.Traverse,
ArticleResourceLimits.MaxArticleBytes,
ArticlePathCanonicalizer.OrganizationalTrackerHost,
rentFrom: null,
CanonicalBodyMode.PreserveCanonicalBody);
if (!materialize.IsAccepted || materialize.ArticleBytes is null)
{
AcceptedArticleWorkerLogMessages.PathRejected(
Expand Down Expand Up @@ -168,12 +174,14 @@ internal bool TryRewriteRented(AcceptedArticle article, out byte[]? buffer, out
return false;
}

// ArtData is already CanonicalV1 from ingest Normalize; only Path (and Date) must change.
var materialize = NntpArticleCanonicalMaterializer.Materialize(
in parse,
ArticlePathMode.Traverse,
ArticleResourceLimits.MaxArticleBytes,
ArticlePathCanonicalizer.OrganizationalTrackerHost,
RewritePool);
RewritePool,
CanonicalBodyMode.PreserveCanonicalBody);
if (!materialize.IsAccepted || materialize.ArticleBytes is null)
{
AcceptedArticleWorkerLogMessages.PathRejected(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,50 @@ public sealed class NntpArticleCanonicalMaterializerTests
/// </summary>
private const string OrganizationalTrackerPrefix = "news.usenet.ninja!";

/// <summary>
/// Rematerializing CanonicalV1 ArtData with <see cref="CanonicalBodyMode.PreserveCanonicalBody"/>
/// must not add another stuffing layer to dot-leading body lines.
/// </summary>
[Fact]
public void Materialize_PreserveCanonicalBody_DoesNotRestuffAlreadyStuffedBody()
{
var parser = new NntpArticleParser(LocalFqdn);
var destuffed = BuildArticle(
[
"Path: peer.example",
"Date: Fri, 23 Aug 2024 07:30:10 +0000",
"Message-ID: <preserve-body@example.test>",
"Newsgroups: alt.test",
"From: user@example.test",
"Subject: dots",
],
"Hello\r\n.leading\r\n..leading\r\n");
var first = ArticleRecordFactory.TryCreate(parser, destuffed, ArticlePathMode.Normalize);
Assert.True(first.IsAccepted);
var stuffedBody = Encoding.ASCII.GetString(GetBodyBytes(first.Record.ArtData.Span));
Assert.Equal("Hello\r\n..leading\r\n...leading\r\n", stuffedBody);

var parse = parser.Parse(first.Record.ArtData);
Assert.True(parse.IsAccepted);
var restuffed = NntpArticleCanonicalMaterializer.Materialize(in parse, ArticlePathMode.Traverse);
Assert.True(restuffed.IsAccepted);
Assert.Equal(
"Hello\r\n...leading\r\n....leading\r\n",
Encoding.ASCII.GetString(GetBodyBytes(restuffed.ArticleBytes.AsSpan(0, restuffed.CanonicalLength))));

var preserved = NntpArticleCanonicalMaterializer.Materialize(
in parse,
ArticlePathMode.Traverse,
ArticleResourceLimits.MaxArticleBytes,
ArticlePathCanonicalizer.OrganizationalTrackerHost,
rentFrom: null,
CanonicalBodyMode.PreserveCanonicalBody);
Assert.True(preserved.IsAccepted);
Assert.Equal(
stuffedBody,
Encoding.ASCII.GetString(GetBodyBytes(preserved.ArticleBytes.AsSpan(0, preserved.CanonicalLength))));
}

/// <summary>
/// A missing Path is inserted as the tracker plus the local identity, the Date is rewritten to canonical UTC, and the body is unchanged.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ public async Task Worker_applies_the_path_hop_and_discards_the_rewritten_article
[Fact]
public async Task Worker_prepends_the_hop_again_when_it_is_already_present()
{
var inbound = Build("<worker-again@seg.test>", "kept-body\r\n", CacheFqdn);
// Queued ArtData already carries this server's hop (e.g. loop/reinject). Normalize keeps it.
var inbound = Build("<worker-again@seg.test>", "kept-body\r\n", pathAlreadyHasCacheHop: true);
var inboundText = Encoding.ASCII.GetString(inbound.Record.ArtData.Span);
Assert.Equal(1, CountHop(inboundText, CacheFqdn));
var queue = new AcceptedArticleQueue();
Expand Down Expand Up @@ -133,18 +134,14 @@ public async Task Worker_continues_after_a_rejected_article_and_a_thrown_article
}

[Fact]
public void Path_rewrite_matches_the_canonical_factory_bytes()
public void Path_rewrite_preserves_canonical_body_and_prepends_cache_hop()
{
var inbound = Build("<rewrite-match@seg.test>", "body-line\r\nsecond\r\n");
var processor = new AcceptedArticlePathProcessor(CacheFqdn, NullLogger.Instance);
Assert.True(processor.TryRewrite(ToQueued(inbound), out var rewritten));
var created = ArticleRecordFactory.TryCreate(
new NntpArticleParser(CacheFqdn),
inbound.Record.ArtData,
ArticlePathMode.Traverse);
Assert.True(created.IsAccepted);
Assert.True(created.Record.ArtData.Span.SequenceEqual(rewritten.Span));
Assert.Equal(inbound.Record.ArtId, created.Record.ArtId);
Assert.Equal(inbound.Record.ArtId, ArticleId.FromMessageId(MessageIdValue(rewritten.Span)));
Assert.Equal(Body(Encoding.ASCII.GetString(inbound.Record.ArtData.Span)), Body(Encoding.ASCII.GetString(rewritten.Span)));
Assert.StartsWith($"Path: {CacheFqdn}!", HeaderLine(rewritten.Span, "Path"), StringComparison.Ordinal);

var wrong = new AcceptedArticle(
ArticleId.FromMessageId("<other@seg.test>"u8),
Expand All @@ -154,6 +151,31 @@ public void Path_rewrite_matches_the_canonical_factory_bytes()
Assert.False(processor.TryRewrite(wrong, out _));
}

[Fact]
public void Path_rewrite_does_not_double_stuff_dot_leading_body_lines()
{
// Destuffed logical body (post-TAKETHIS unstuff). Normalize produces CanonicalV1 stuffed ArtData.
var inbound = Build(
"<rewrite-dots@seg.test>",
"Hello\r\n.leading\r\n..leading\r\n...leading\r\n");
var queuedBody = Body(Encoding.ASCII.GetString(inbound.Record.ArtData.Span));
Assert.Equal("Hello\r\n..leading\r\n...leading\r\n....leading\r\n", queuedBody);

var processor = new AcceptedArticlePathProcessor(CacheFqdn, NullLogger.Instance);
Assert.True(processor.TryRewriteRented(ToQueued(inbound), out var rented, out var length));
try
{
var rewritten = rented.AsSpan(0, length);
Assert.Equal(queuedBody, Body(Encoding.ASCII.GetString(rewritten)));
Assert.StartsWith($"Path: {CacheFqdn}!", HeaderLine(rewritten, "Path"), StringComparison.Ordinal);
Assert.DoesNotContain(".....\r\n", Encoding.ASCII.GetString(rewritten), StringComparison.Ordinal);
}
finally
{
AcceptedArticlePathProcessor.ReturnRewriteBuffer(rented!);
}
}

[Fact]
public void Rented_path_rewrite_matches_the_exact_buffer_and_is_returned()
{
Expand Down Expand Up @@ -213,19 +235,44 @@ private static string Body(string article)
return article[(split + 4)..];
}

private static Built Build(string messageId, string body, string ingressFqdn = "nntpd01.usenet.ninja")
private static string HeaderLine(ReadOnlySpan<byte> article, string name)
{
var text = Encoding.ASCII.GetString(article);
var prefix = name + ": ";
var start = text.IndexOf(prefix, StringComparison.Ordinal);
Assert.True(start >= 0, name);
var end = text.IndexOf("\r\n", start, StringComparison.Ordinal);
Assert.True(end > start);
return text[start..end];
}

private static ReadOnlySpan<byte> MessageIdValue(ReadOnlySpan<byte> article)
{
var parser = new NntpArticleParser(ingressFqdn);
var line = HeaderLine(article, "Message-ID");
Assert.StartsWith("Message-ID: ", line, StringComparison.Ordinal);
return Encoding.ASCII.GetBytes(line["Message-ID: ".Length..]);
}

private static Built Build(string messageId, string body, bool pathAlreadyHasCacheHop = false)
{
// Match StorageNntpFeedSession: parser identity is the storage FQDN; ingest uses Normalize.
var parser = new NntpArticleParser(CacheFqdn);
var path = pathAlreadyHasCacheHop
? CacheFqdn + "!peer.example"
: "peer.example";
var text = new StringBuilder()
.Append("Path: peer.example\r\n")
.Append("Path: ").Append(path).Append("\r\n")
.Append("Date: Fri, 23 Aug 2024 07:30:10 +0000\r\n")
.Append("Message-ID: ").Append(messageId).Append("\r\n")
.Append("Newsgroups: alt.test\r\n")
.Append("From: user@example.test\r\n")
.Append("Subject: vatp\r\n")
.Append("\r\n")
.Append(body);
var created = ArticleRecordFactory.TryCreate(parser, Encoding.ASCII.GetBytes(text.ToString()));
var created = ArticleRecordFactory.TryCreate(
parser,
Encoding.ASCII.GetBytes(text.ToString()),
ArticlePathMode.Normalize);
Assert.True(created.IsAccepted, created.MaterializeFailure.ToString());
return new Built(created.Record);
}
Expand Down
Loading
Loading