From e18eccad989cd9c8cff7bb4104ee877bc60d75a8 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Mon, 19 Oct 2026 21:42:29 +0200 Subject: [PATCH] Fix validation of folded Newsgroups values Unfold valid CRLF continuations before validating Newsgroups, allow whitespace around group separators, and reject whitespace inside group names. Add parser and overview regression coverage. --- docs/common/ArticleRecord.md | 4 +- .../Articles/Parsing/NntpArticleParser.cs | 71 +++++- .../OverviewDb/OverviewArticleV1CodecTests.cs | 9 +- .../Parsing/NntpArticleParserTests.cs | 227 ++++++++++++++++++ 4 files changed, 301 insertions(+), 10 deletions(-) diff --git a/docs/common/ArticleRecord.md b/docs/common/ArticleRecord.md index 6ff16093..adec2e1c 100644 --- a/docs/common/ArticleRecord.md +++ b/docs/common/ArticleRecord.md @@ -323,7 +323,7 @@ Headers are split on CR, LF, or CRLF. A zero-length line is the header/body sepa | Header | Rule | |---|---| | Message-ID | Required, not duplicate, unfold + `IsValidMessageId` (no strip in this path) | -| Newsgroups | Required, not duplicate, length 1…4096, comma tokens, SP/HTAB ignored, plausible group chars | +| Newsgroups | Required, not duplicate, length 1…4096. A CRLF followed by SP or HTAB unfolds to one SP before comma-token checks. Bare CR, bare LF, and CRLF without that whitespace are `InvalidNewsgroups`. SP and HTAB are allowed before the first name, after the last name, and immediately beside a comma. Whitespace inside a name is `InvalidNewsgroups`. | | Date-family | At least one resolvable candidate (see §8) | | Path | At most one; may be absent; invalid tokens/controls/length → `InvalidPath` | | From | Optional; if present, unfold, must contain `@` not at ends, printable ASCII, non-empty | @@ -451,7 +451,7 @@ public readonly struct ArticleFieldTable `Locate(artData, selectedDateHeaderName)` walks canonical headers (same CR/LF/CRLF and folding rules as the parser) and records the **first** occurrence of each name. `Date` is the header whose known name equals the parser’s winning `SelectedDateHeaderName`. -Ranges include folded continuation bytes as stored. Absent fields use `ArticleByteRange.Absent`. Newsgroups is that value range (ArtGroups). Membership checks tokenize `,` / skip SP-HTAB later; the record does not allocate `List` (`TryCreate_DoesNotCreateGroupStringCollection`). +Ranges include folded continuation bytes as stored. Absent fields use `ArticleByteRange.Absent`. Newsgroups is that value range (ArtGroups). Membership checks tokenize `,` and skip SP, HTAB, CR, and LF around each token; the record does not allocate `List` (`TryCreate_DoesNotCreateGroupStringCollection`). Parser offsets into destuffed source must not be kept after Date/Path rewrite. diff --git a/src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs b/src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs index 7bed660b..0be0f51e 100644 --- a/src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs +++ b/src/VectorNNTPD.Common/Articles/Parsing/NntpArticleParser.cs @@ -756,7 +756,10 @@ private static bool TryValidateMessageId( /// /// for an empty token (leading, trailing, or doubled comma), a byte other than SP, HTAB, comma, /// or , or a value whose last token has no plausible character. - /// SP and HTAB are skipped and do not separate tokens. + /// A CRLF followed by SP or HTAB is unfolded to one SP before that check. A bare CR, a bare LF, or a CRLF that is not + /// followed by SP or HTAB is . + /// SP and HTAB are allowed before the first name, after the last name, and immediately beside a comma. + /// The same bytes inside a name are . /// private static bool TryValidateNewsgroups( ReadOnlySpan articleSpan, @@ -797,10 +800,19 @@ private static bool TryValidateNewsgroups( } var value = articleSpan.Slice(newsgroupsHeader.ValueOffset, newsgroupsHeader.ValueLength); + Span unfolded = stackalloc byte[value.Length]; + if (!HasOnlyCrlfFolds(value) + || !NntpArticleHeaderValueUnfolder.TryUnfold(value, unfolded, out var unfoldedLength)) + { + failureCode = NntpArticleParseFailureCode.InvalidNewsgroups; + return false; + } + + ReadOnlySpan semantic = unfolded[..unfoldedLength]; var tokenHasChar = false; - for (var i = 0; i < value.Length; i++) + for (var i = 0; i < semantic.Length; i++) { - var b = value[i]; + var b = semantic[i]; if (b == (byte)',') { if (!tokenHasChar) @@ -815,7 +827,24 @@ private static bool TryValidateNewsgroups( if (b is (byte)' ' or (byte)'\t') { - continue; + if (!tokenHasChar) + { + continue; + } + + var next = i + 1; + while (next < semantic.Length && semantic[next] is (byte)' ' or (byte)'\t') + { + next++; + } + + if (next >= semantic.Length || semantic[next] == (byte)',') + { + continue; + } + + failureCode = NntpArticleParseFailureCode.InvalidNewsgroups; + return false; } if (!IsPlausibleNewsgroupChar(b)) @@ -836,6 +865,40 @@ private static bool TryValidateNewsgroups( return true; } + /// + /// Reports whether every line break in a Newsgroups value is a CRLF fold. + /// + /// Raw Newsgroups value, including continuation bytes. + /// + /// when a CR is not followed by LF or a LF appears without a preceding CR. + /// The following SP or HTAB is checked by . + /// + private static bool HasOnlyCrlfFolds(ReadOnlySpan value) + { + for (var i = 0; i < value.Length; i++) + { + var b = value[i]; + if (b == (byte)'\n') + { + return false; + } + + if (b != (byte)'\r') + { + continue; + } + + if ((uint)(i + 1) >= (uint)value.Length || value[i + 1] != (byte)'\n') + { + return false; + } + + i++; + } + + return true; + } + /// /// Checks every From header. Absence is success. Duplicates are each checked and are not themselves a failure. /// diff --git a/tests/VectorNNTPD.Common.Tests/Articles/OverviewDb/OverviewArticleV1CodecTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/OverviewDb/OverviewArticleV1CodecTests.cs index 2965128c..b3a22337 100644 --- a/tests/VectorNNTPD.Common.Tests/Articles/OverviewDb/OverviewArticleV1CodecTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Articles/OverviewDb/OverviewArticleV1CodecTests.cs @@ -206,10 +206,10 @@ public void EncodeDecode_RoundTrip_IsEquivalent() } /// - /// A folded Newsgroups header is rejected with before an overview payload is produced. + /// A CRLF-folded Newsgroups header is accepted, and overview stores each comma-separated group. /// [Fact] - public void FoldedNewsgroups_AreRejectedByParser_BeforeOverviewEncode() + public void FoldedNewsgroups_AreAccepted_AndOverviewStoresEachGroup() { var destuffed = "Path: peer.example\r\n" + @@ -225,8 +225,9 @@ public void FoldedNewsgroups_AreRejectedByParser_BeforeOverviewEncode() new NntpArticleParser(LocalFqdn), Encoding.ASCII.GetBytes(destuffed)); - Assert.False(created.IsAccepted); - Assert.Equal(NntpArticleParseFailureCode.InvalidNewsgroups, created.ParseFailure); + Assert.True(created.IsAccepted); + var overview = OverviewArticleV1Codec.Decode(OverviewArticleV1Codec.Encode(created.Record)); + Assert.Equal(["group.one", "group.two", "group.three"], overview.Newsgroups); } /// Multi-line body containing a token that must not appear in the overview payload. diff --git a/tests/VectorNNTPD.Common.Tests/Articles/Parsing/NntpArticleParserTests.cs b/tests/VectorNNTPD.Common.Tests/Articles/Parsing/NntpArticleParserTests.cs index 964cd424..838d7848 100644 --- a/tests/VectorNNTPD.Common.Tests/Articles/Parsing/NntpArticleParserTests.cs +++ b/tests/VectorNNTPD.Common.Tests/Articles/Parsing/NntpArticleParserTests.cs @@ -1214,6 +1214,175 @@ public void Parse_WhenNewsgroupsContainsEmptyToken_Rejects() Assert.Equal(NntpArticleParseFailureCode.InvalidNewsgroups, result.FailureCode); } + /// + /// A single-line Newsgroups value remains one stored group. + /// + [Fact] + public void Parse_WhenNewsgroupsIsSingleLine_AcceptsThatGroup() + { + AssertNewsgroupsAccepted("Newsgroups: alt.test", "alt.test"); + } + + /// + /// CRLF plus a space is folding whitespace and does not join the group names. + /// + [Fact] + public void Parse_WhenNewsgroupsFoldsOnCrlfSpace_AcceptsSeparateGroups() + { + ArticleRecord record = AssertNewsgroupsAccepted( + "Newsgroups: alt.test,\r\n alt.example", + "alt.test", + "alt.example"); + string stored = Encoding.ASCII.GetString(record.Newsgroups); + Assert.Contains("alt.test", stored, StringComparison.Ordinal); + Assert.Contains("alt.example", stored, StringComparison.Ordinal); + Assert.Contains("\r\n", stored, StringComparison.Ordinal); + Assert.DoesNotContain("alt.testalt.example", stored, StringComparison.Ordinal); + } + + /// + /// CRLF plus a tab is the same fold as CRLF plus a space. + /// + [Fact] + public void Parse_WhenNewsgroupsFoldsOnCrlfTab_AcceptsSeparateGroups() + { + AssertNewsgroupsAccepted( + "Newsgroups: alt.test,\r\n\talt.example", + "alt.test", + "alt.example"); + } + + /// + /// Several CRLF folds in one value stay separate groups. + /// + [Fact] + public void Parse_WhenNewsgroupsHasMultipleCrlfFolds_AcceptsEachGroup() + { + AssertNewsgroupsAccepted( + "Newsgroups: alt.test,\r\n alt.example,\r\n alt.misc", + "alt.test", + "alt.example", + "alt.misc"); + } + + /// + /// Folding whitespace on either side of a comma remains a separator. + /// + [Fact] + public void Parse_WhenNewsgroupsFoldsAroundComma_AcceptsSeparateGroups() + { + AssertNewsgroupsAccepted( + "Newsgroups: alt.test\r\n ,\r\n alt.example", + "alt.test", + "alt.example"); + } + + /// + /// Spaces and tabs without a line break still separate only at commas. + /// + [Fact] + public void Parse_WhenNewsgroupsUsesSpaceOrTabWithoutCrlf_AcceptsSeparateGroups() + { + AssertNewsgroupsAccepted("Newsgroups: alt.test, alt.example", "alt.test", "alt.example"); + AssertNewsgroupsAccepted("Newsgroups: alt.test,\talt.example", "alt.test", "alt.example"); + } + + /// + /// A bare CR is not folding whitespace. + /// + [Fact] + public void Parse_WhenNewsgroupsContainsBareCr_RejectsInvalidNewsgroups() + { + AssertNewsgroupsRejected("Newsgroups: alt.test,\r alt.example", NntpArticleParseFailureCode.InvalidNewsgroups); + } + + /// + /// A bare LF is not folding whitespace. + /// + [Fact] + public void Parse_WhenNewsgroupsContainsBareLf_RejectsInvalidNewsgroups() + { + AssertNewsgroupsRejected("Newsgroups: alt.test,\n alt.example", NntpArticleParseFailureCode.InvalidNewsgroups); + } + + /// + /// A CRLF that is not followed by SP or HTAB is not a Newsgroups fold. + /// + [Fact] + public void Parse_WhenNewsgroupsCrlfIsNotAContinuation_Rejects() + { + AssertNewsgroupsRejected( + "Newsgroups: alt.test,\r\nalt.example", + NntpArticleParseFailureCode.MissingHeaderBodySeparator); + } + + /// + /// An illegal group character is still rejected after a legal fold. + /// + [Fact] + public void Parse_WhenFoldedNewsgroupsContainsIllegalCharacter_Rejects() + { + AssertNewsgroupsRejected( + "Newsgroups: alt.test,\r\n alt.example!", + NntpArticleParseFailureCode.InvalidNewsgroups); + } + + /// + /// A fold does not hide an empty comma token. + /// + [Fact] + public void Parse_WhenFoldedNewsgroupsContainsEmptyToken_Rejects() + { + AssertNewsgroupsRejected( + "Newsgroups: alt.test,\r\n ,alt.example", + NntpArticleParseFailureCode.InvalidNewsgroups); + } + + /// + /// Folding whitespace immediately before a comma remains a separator. + /// + [Fact] + public void Parse_WhenNewsgroupsFoldsBeforeComma_AcceptsSeparateGroups() + { + AssertNewsgroupsAccepted( + "Newsgroups: alt.test\r\n ,alt.example", + "alt.test", + "alt.example"); + } + + /// + /// A fold inside a group name is rejected. + /// + [Fact] + public void Parse_WhenNewsgroupsFoldSplitsGroupName_Rejects() + { + AssertNewsgroupsRejected( + "Newsgroups: alt.\r\n test", + NntpArticleParseFailureCode.InvalidNewsgroups); + } + + /// + /// A space inside a group name is rejected. + /// + [Fact] + public void Parse_WhenNewsgroupsSpaceSplitsGroupName_Rejects() + { + AssertNewsgroupsRejected( + "Newsgroups: alt.te st", + NntpArticleParseFailureCode.InvalidNewsgroups); + } + + /// + /// A tab inside a group name is rejected. + /// + [Fact] + public void Parse_WhenNewsgroupsTabSplitsGroupName_Rejects() + { + AssertNewsgroupsRejected( + "Newsgroups: alt.\t test", + NntpArticleParseFailureCode.InvalidNewsgroups); + } + /// /// Verifies date resolver integration uses fallback candidate headers when primary Date is absent. /// @@ -2275,6 +2444,64 @@ private static NntpArticleHeaderEntry FindHeaderEntry(IReadOnlyList + /// Accepts one article and returns the record whose overview groups equal . + /// + /// Newsgroups header line. It may contain a CRLF fold. + /// Group names in header order. + /// The accepted record. + private static ArticleRecord AssertNewsgroupsAccepted(string newsgroupsHeader, params string[] groups) + { + NntpArticleParser parser = new(LocalFqdn); + byte[] article = BuildArticle( + headers: + [ + "Date: Fri, 23 Aug 2024 07:30:10 +0000", + "Message-ID: ", + newsgroupsHeader, + "From: user@example.test", + ], + body: "body\r\n"); + + NntpArticleParseResult result = parser.Parse(article); + Assert.True(result.IsAccepted); + Assert.Equal(NntpArticleParseFailureCode.None, result.FailureCode); + + ArticleRecordCreateResult created = ArticleRecordFactory.TryCreate(parser, article); + Assert.True(created.IsAccepted); + OverviewArticleV1 overview = OverviewArticleV1Codec.Decode(OverviewArticleV1Codec.Encode(created.Record)); + Assert.Equal(groups, overview.Newsgroups); + return created.Record; + } + + /// + /// Parses one article and asserts the Newsgroups failure code. + /// + /// Newsgroups header line. It may contain a broken fold. + /// Expected parser failure. + private static void AssertNewsgroupsRejected(string newsgroupsHeader, NntpArticleParseFailureCode failureCode) + { + NntpArticleParser parser = new(LocalFqdn); + byte[] article = BuildArticle( + headers: + [ + "Date: Fri, 23 Aug 2024 07:30:10 +0000", + "Message-ID: ", + newsgroupsHeader, + "From: user@example.test", + ], + body: "body\r\n"); + + NntpArticleParseResult result = parser.Parse(article); + Assert.False(result.IsAccepted); + Assert.Equal(failureCode, result.FailureCode); + + ArticleRecordCreateResult created = ArticleRecordFactory.TryCreate(parser, article); + Assert.False(created.IsAccepted); + Assert.Equal(failureCode, created.ParseFailure); + Assert.Equal(0, created.Record.ArtSize); + } + /// /// Parses two Date lines and asserts the article is rejected with no canonical instant. ///