Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/common/ArticleRecord.md
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ Headers are split on CR, LF, or CRLF. A zero-length line is the header/body sepa
| Header | Rule |
|---|---|
| Message-ID | Required, not duplicate, unfold + `IsValidMessageId` (no strip in this path) |
| Newsgroups | Required, not duplicate, length 1…4096, comma tokens, SP/HTAB ignored, plausible group chars |
| Newsgroups | Required, not duplicate, length 1…4096. A CRLF followed by SP or HTAB unfolds to one SP before comma-token checks. Bare CR, bare LF, and CRLF without that whitespace are `InvalidNewsgroups`. SP and HTAB are allowed before the first name, after the last name, and immediately beside a comma. Whitespace inside a name is `InvalidNewsgroups`. |
| Date-family | At least one resolvable candidate (see §8) |
| Path | At most one; may be absent; invalid tokens/controls/length → `InvalidPath` |
| From | Optional; if present, unfold, must contain `@` not at ends, printable ASCII, non-empty |
Expand Down Expand Up @@ -451,7 +451,7 @@ public readonly struct ArticleFieldTable

`Locate(artData, selectedDateHeaderName)` walks canonical headers (same CR/LF/CRLF and folding rules as the parser) and records the **first** occurrence of each name. `Date` is the header whose known name equals the parser’s winning `SelectedDateHeaderName`.

Ranges include folded continuation bytes as stored. Absent fields use `ArticleByteRange.Absent`. Newsgroups is that value range (ArtGroups). Membership checks tokenize `,` / skip SP-HTAB later; the record does not allocate `List<string>` (`TryCreate_DoesNotCreateGroupStringCollection`).
Ranges include folded continuation bytes as stored. Absent fields use `ArticleByteRange.Absent`. Newsgroups is that value range (ArtGroups). Membership checks tokenize `,` and skip SP, HTAB, CR, and LF around each token; the record does not allocate `List<string>` (`TryCreate_DoesNotCreateGroupStringCollection`).

Parser offsets into destuffed source must not be kept after Date/Path rewrite.

Expand Down
71 changes: 67 additions & 4 deletions src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -756,7 +756,10 @@ private static bool TryValidateMessageId(
/// <returns>
/// <see langword="false"/> for an empty token (leading, trailing, or doubled comma), a byte other than SP, HTAB, comma,
/// or <see cref="IsPlausibleNewsgroupChar"/>, or a value whose last token has no plausible character.
/// SP and HTAB are skipped and do not separate tokens.
/// A CRLF followed by SP or HTAB is unfolded to one SP before that check. A bare CR, a bare LF, or a CRLF that is not
/// followed by SP or HTAB is <see cref="NntpArticleParseFailureCode.InvalidNewsgroups"/>.
/// SP and HTAB are allowed before the first name, after the last name, and immediately beside a comma.
/// The same bytes inside a name are <see cref="NntpArticleParseFailureCode.InvalidNewsgroups"/>.
/// </returns>
private static bool TryValidateNewsgroups(
ReadOnlySpan<byte> articleSpan,
Expand Down Expand Up @@ -797,10 +800,19 @@ private static bool TryValidateNewsgroups(
}

var value = articleSpan.Slice(newsgroupsHeader.ValueOffset, newsgroupsHeader.ValueLength);
Span<byte> unfolded = stackalloc byte[value.Length];
if (!HasOnlyCrlfFolds(value)
|| !NntpArticleHeaderValueUnfolder.TryUnfold(value, unfolded, out var unfoldedLength))
{
failureCode = NntpArticleParseFailureCode.InvalidNewsgroups;
return false;
}

ReadOnlySpan<byte> semantic = unfolded[..unfoldedLength];
var tokenHasChar = false;
for (var i = 0; i < value.Length; i++)
for (var i = 0; i < semantic.Length; i++)
{
var b = value[i];
var b = semantic[i];
if (b == (byte)',')
{
if (!tokenHasChar)
Expand All @@ -815,7 +827,24 @@ private static bool TryValidateNewsgroups(

if (b is (byte)' ' or (byte)'\t')
{
continue;
if (!tokenHasChar)
{
continue;
}

var next = i + 1;
while (next < semantic.Length && semantic[next] is (byte)' ' or (byte)'\t')
{
next++;
}

if (next >= semantic.Length || semantic[next] == (byte)',')
{
continue;
}

failureCode = NntpArticleParseFailureCode.InvalidNewsgroups;
return false;
}

if (!IsPlausibleNewsgroupChar(b))
Expand All @@ -836,6 +865,40 @@ private static bool TryValidateNewsgroups(
return true;
}

/// <summary>
/// Reports whether every line break in a Newsgroups value is a CRLF fold.
/// </summary>
/// <param name="value">Raw Newsgroups value, including continuation bytes.</param>
/// <returns>
/// <see langword="false"/> when a CR is not followed by LF or a LF appears without a preceding CR.
/// The following SP or HTAB is checked by <see cref="NntpArticleHeaderValueUnfolder.TryUnfold"/>.
/// </returns>
private static bool HasOnlyCrlfFolds(ReadOnlySpan<byte> value)
{
for (var i = 0; i < value.Length; i++)
{
var b = value[i];
if (b == (byte)'\n')
{
return false;
}

if (b != (byte)'\r')
{
continue;
}

if ((uint)(i + 1) >= (uint)value.Length || value[i + 1] != (byte)'\n')
{
return false;
}

i++;
}

return true;
}

/// <summary>
/// Checks every From header. Absence is success. Duplicates are each checked and are not themselves a failure.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,10 @@ public void EncodeDecode_RoundTrip_IsEquivalent()
}

/// <summary>
/// A folded Newsgroups header is rejected with <see cref="NntpArticleParseFailureCode.InvalidNewsgroups"/> before an overview payload is produced.
/// A CRLF-folded Newsgroups header is accepted, and overview stores each comma-separated group.
/// </summary>
[Fact]
public void FoldedNewsgroups_AreRejectedByParser_BeforeOverviewEncode()
public void FoldedNewsgroups_AreAccepted_AndOverviewStoresEachGroup()
{
var destuffed =
"Path: peer.example\r\n" +
Expand All @@ -225,8 +225,9 @@ public void FoldedNewsgroups_AreRejectedByParser_BeforeOverviewEncode()
new NntpArticleParser(LocalFqdn),
Encoding.ASCII.GetBytes(destuffed));

Assert.False(created.IsAccepted);
Assert.Equal(NntpArticleParseFailureCode.InvalidNewsgroups, created.ParseFailure);
Assert.True(created.IsAccepted);
var overview = OverviewArticleV1Codec.Decode(OverviewArticleV1Codec.Encode(created.Record));
Assert.Equal(["group.one", "group.two", "group.three"], overview.Newsgroups);
}

/// <summary>Multi-line body containing a token that must not appear in the overview payload.</summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1214,6 +1214,175 @@ public void Parse_WhenNewsgroupsContainsEmptyToken_Rejects()
Assert.Equal(NntpArticleParseFailureCode.InvalidNewsgroups, result.FailureCode);
}

/// <summary>
/// A single-line Newsgroups value remains one stored group.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsIsSingleLine_AcceptsThatGroup()
{
AssertNewsgroupsAccepted("Newsgroups: alt.test", "alt.test");
}

/// <summary>
/// CRLF plus a space is folding whitespace and does not join the group names.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsFoldsOnCrlfSpace_AcceptsSeparateGroups()
{
ArticleRecord record = AssertNewsgroupsAccepted(
"Newsgroups: alt.test,\r\n alt.example",
"alt.test",
"alt.example");
string stored = Encoding.ASCII.GetString(record.Newsgroups);
Assert.Contains("alt.test", stored, StringComparison.Ordinal);
Assert.Contains("alt.example", stored, StringComparison.Ordinal);
Assert.Contains("\r\n", stored, StringComparison.Ordinal);
Assert.DoesNotContain("alt.testalt.example", stored, StringComparison.Ordinal);
}

/// <summary>
/// CRLF plus a tab is the same fold as CRLF plus a space.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsFoldsOnCrlfTab_AcceptsSeparateGroups()
{
AssertNewsgroupsAccepted(
"Newsgroups: alt.test,\r\n\talt.example",
"alt.test",
"alt.example");
}

/// <summary>
/// Several CRLF folds in one value stay separate groups.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsHasMultipleCrlfFolds_AcceptsEachGroup()
{
AssertNewsgroupsAccepted(
"Newsgroups: alt.test,\r\n alt.example,\r\n alt.misc",
"alt.test",
"alt.example",
"alt.misc");
}

/// <summary>
/// Folding whitespace on either side of a comma remains a separator.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsFoldsAroundComma_AcceptsSeparateGroups()
{
AssertNewsgroupsAccepted(
"Newsgroups: alt.test\r\n ,\r\n alt.example",
"alt.test",
"alt.example");
}

/// <summary>
/// Spaces and tabs without a line break still separate only at commas.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsUsesSpaceOrTabWithoutCrlf_AcceptsSeparateGroups()
{
AssertNewsgroupsAccepted("Newsgroups: alt.test, alt.example", "alt.test", "alt.example");
AssertNewsgroupsAccepted("Newsgroups: alt.test,\talt.example", "alt.test", "alt.example");
}

/// <summary>
/// A bare CR is not folding whitespace.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsContainsBareCr_RejectsInvalidNewsgroups()
{
AssertNewsgroupsRejected("Newsgroups: alt.test,\r alt.example", NntpArticleParseFailureCode.InvalidNewsgroups);
}

/// <summary>
/// A bare LF is not folding whitespace.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsContainsBareLf_RejectsInvalidNewsgroups()
{
AssertNewsgroupsRejected("Newsgroups: alt.test,\n alt.example", NntpArticleParseFailureCode.InvalidNewsgroups);
}

/// <summary>
/// A CRLF that is not followed by SP or HTAB is not a Newsgroups fold.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsCrlfIsNotAContinuation_Rejects()
{
AssertNewsgroupsRejected(
"Newsgroups: alt.test,\r\nalt.example",
NntpArticleParseFailureCode.MissingHeaderBodySeparator);
}

/// <summary>
/// An illegal group character is still rejected after a legal fold.
/// </summary>
[Fact]
public void Parse_WhenFoldedNewsgroupsContainsIllegalCharacter_Rejects()
{
AssertNewsgroupsRejected(
"Newsgroups: alt.test,\r\n alt.example!",
NntpArticleParseFailureCode.InvalidNewsgroups);
}

/// <summary>
/// A fold does not hide an empty comma token.
/// </summary>
[Fact]
public void Parse_WhenFoldedNewsgroupsContainsEmptyToken_Rejects()
{
AssertNewsgroupsRejected(
"Newsgroups: alt.test,\r\n ,alt.example",
NntpArticleParseFailureCode.InvalidNewsgroups);
}

/// <summary>
/// Folding whitespace immediately before a comma remains a separator.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsFoldsBeforeComma_AcceptsSeparateGroups()
{
AssertNewsgroupsAccepted(
"Newsgroups: alt.test\r\n ,alt.example",
"alt.test",
"alt.example");
}

/// <summary>
/// A fold inside a group name is rejected.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsFoldSplitsGroupName_Rejects()
{
AssertNewsgroupsRejected(
"Newsgroups: alt.\r\n test",
NntpArticleParseFailureCode.InvalidNewsgroups);
}

/// <summary>
/// A space inside a group name is rejected.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsSpaceSplitsGroupName_Rejects()
{
AssertNewsgroupsRejected(
"Newsgroups: alt.te st",
NntpArticleParseFailureCode.InvalidNewsgroups);
}

/// <summary>
/// A tab inside a group name is rejected.
/// </summary>
[Fact]
public void Parse_WhenNewsgroupsTabSplitsGroupName_Rejects()
{
AssertNewsgroupsRejected(
"Newsgroups: alt.\t test",
NntpArticleParseFailureCode.InvalidNewsgroups);
}

/// <summary>
/// Verifies date resolver integration uses fallback candidate headers when primary Date is absent.
/// </summary>
Expand Down Expand Up @@ -2275,6 +2444,64 @@ private static NntpArticleHeaderEntry FindHeaderEntry(IReadOnlyList<NntpArticleH
throw new InvalidOperationException($"Header '{knownName}' was not found in parse output.");
}

/// <summary>
/// Accepts one article and returns the record whose overview groups equal <paramref name="groups"/>.
/// </summary>
/// <param name="newsgroupsHeader">Newsgroups header line. It may contain a CRLF fold.</param>
/// <param name="groups">Group names in header order.</param>
/// <returns>The accepted record.</returns>
private static ArticleRecord AssertNewsgroupsAccepted(string newsgroupsHeader, params string[] groups)
{
NntpArticleParser parser = new(LocalFqdn);
byte[] article = BuildArticle(
headers:
[
"Date: Fri, 23 Aug 2024 07:30:10 +0000",
"Message-ID: <ng-fold@example.test>",
newsgroupsHeader,
"From: user@example.test",
],
body: "body\r\n");

NntpArticleParseResult result = parser.Parse(article);
Assert.True(result.IsAccepted);
Assert.Equal(NntpArticleParseFailureCode.None, result.FailureCode);

ArticleRecordCreateResult created = ArticleRecordFactory.TryCreate(parser, article);
Assert.True(created.IsAccepted);
OverviewArticleV1 overview = OverviewArticleV1Codec.Decode(OverviewArticleV1Codec.Encode(created.Record));
Assert.Equal(groups, overview.Newsgroups);
return created.Record;
}

/// <summary>
/// Parses one article and asserts the Newsgroups failure code.
/// </summary>
/// <param name="newsgroupsHeader">Newsgroups header line. It may contain a broken fold.</param>
/// <param name="failureCode">Expected parser failure.</param>
private static void AssertNewsgroupsRejected(string newsgroupsHeader, NntpArticleParseFailureCode failureCode)
{
NntpArticleParser parser = new(LocalFqdn);
byte[] article = BuildArticle(
headers:
[
"Date: Fri, 23 Aug 2024 07:30:10 +0000",
"Message-ID: <ng-fold-bad@example.test>",
newsgroupsHeader,
"From: user@example.test",
],
body: "body\r\n");

NntpArticleParseResult result = parser.Parse(article);
Assert.False(result.IsAccepted);
Assert.Equal(failureCode, result.FailureCode);

ArticleRecordCreateResult created = ArticleRecordFactory.TryCreate(parser, article);
Assert.False(created.IsAccepted);
Assert.Equal(failureCode, created.ParseFailure);
Assert.Equal(0, created.Record.ArtSize);
}

/// <summary>
/// Parses two <c>Date</c> lines and asserts the article is rejected with no canonical instant.
/// </summary>
Expand Down
Loading